def mpi_initialize(self, file_names): self.integration_pickle_names = file_names self.t1 = time.time() assert self.params.backend == 'FS' # for prototyping rank=0 marshalling from xfel.merging.database.merging_database_fs import manager db_mgr = manager(self.params) db_mgr.initialize_db(self.miller_set.indices()) self.master_db_mgr = db_mgr
def run(self, comm, timing=False): rank = comm.Get_rank() size = comm.Get_size() from time import time as tt # set things up if rank == 0: if timing: print("SETUP START RANK=%d TIME=%f" % (rank, tt())) self.initialize() self.validate() self.read_models() scaler_master = self.scaler_class(miller_set=self.miller_set, i_model=self.i_model, params=self.params, log=self.out) scaler_master.mpi_initialize(self.frame_files) transmitted_info = dict(file_names=self.frame_files, miller_set=self.miller_set, model=self.i_model, params=self.params) if timing: print("SETUP END RANK=%d TIME=%f" % (rank, tt())) else: if timing: print("SETUP START RANK=%d TIME=%f" % (rank, tt())) transmitted_info = None if timing: print("SETUP END RANK=%d TIME=%f" % (rank, tt())) if timing: print("BROADCAST START RANK=%d TIME=%f" % (rank, tt())) transmitted_info = comm.bcast(transmitted_info, root=0) if timing: print("BROADCAST END RANK=%d TIME=%f" % (rank, tt())) # now actually do the work if timing: print("SCALER_WORKER_SETUP START RANK=%d TIME=%f" % (rank, tt())) scaler_worker = self.scaler_class(transmitted_info["miller_set"], transmitted_info["model"], transmitted_info["params"], log=sys.stdout) if timing: print("SCALER_WORKER_SETUP END RANK=%d TIME=%f" % (rank, tt())) assert scaler_worker.params.backend == 'FS' # only option that makes sense from xfel.merging.database.merging_database_fs import manager2 as manager db_mgr = manager(scaler_worker.params) file_names = [ transmitted_info["file_names"][i] for i in range(len(transmitted_info["file_names"])) if i % size == rank ] if timing: print("SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt())) scaler_worker._scale_all_serial(file_names, db_mgr) if timing: print("SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt())) scaler_worker.finished_db_mgr = db_mgr # might want to clean up a bit before returning del scaler_worker.log del scaler_worker.params del scaler_worker.miller_set del scaler_worker.i_model del scaler_worker.reverse_lookup # gather reports and all add together if timing: print("GATHER START RANK=%d TIME=%f" % (rank, tt())) reports = comm.gather(scaler_worker, root=0) if timing: print("GATHER END RANK=%d TIME=%f" % (rank, tt())) if rank == 0: print("Processing reports from %d ranks" % (len(reports))) ireport = 0 for item in reports: if timing: print("SCALER_MASTER_ADD START RANK=%d TIME=%f" % (rank, tt())) scaler_master._add_all_frames(item) if timing: print("SCALER_MASTER_ADD END RANK=%d TIME=%f" % (rank, tt())) print("processing %d calls from report %d" % (len(item.finished_db_mgr.sequencer), ireport)) ireport += 1 for call_instance in item.finished_db_mgr.sequencer: if call_instance["call"] == "insert_frame": if timing: print( "SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f" % (rank, tt())) frame_id_zero_base = scaler_master.master_db_mgr.insert_frame( **call_instance["data"]) if timing: print( "SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f" % (rank, tt())) elif call_instance["call"] == "insert_observation": if timing: print( "SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f" % (rank, tt())) call_instance["data"]['frame_id_0_base'] = [ frame_id_zero_base ] * len(call_instance["data"]['frame_id_0_base']) scaler_master.master_db_mgr.insert_observation( **call_instance["data"]) if timing: print( "SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f" % (rank, tt())) if timing: print("SCALER_MASTER_FINALISE START RANK=%d TIME=%f" % (rank, tt())) scaler_master.master_db_mgr.join( ) # database written, finalize the manager scaler_master.mpi_finalize() if timing: print("SCALER_MASTER_FINALISE END RANK=%d TIME=%f" % (rank, tt())) return self.finalize(scaler_master)
def run(self, comm, timing=False): rank = comm.Get_rank() size = comm.Get_size() from time import time as tt # set things up if rank == 0: self.initialize() self.validate(comm) self.read_models() timing = self.params.mpi.logging if timing: print "~SETUP START RANK=%d TIME=%f;" % (rank, tt()) scaler_master = self.scaler_class(miller_set=self.miller_set, i_model=self.i_model, params=self.params, log=self.out) scaler_master.mpi_initialize(self.frame_files) transmitted_info = dict(file_names=self.frame_files, miller_set=self.miller_set, model=self.i_model, params=self.params) if timing: print "~SETUP END RANK=%d TIME=%f;" % (rank, tt()) else: if timing: print "~SETUP START RANK=%d TIME=%f;" % (rank, tt()) transmitted_info = None if timing: print "~SETUP END RANK=%d TIME=%f;" % (rank, tt()) if timing: print "~BROADCAST START RANK=%d TIME=%f;" % (rank, tt()) transmitted_info = comm.bcast(transmitted_info, root=0) if timing: print "~BROADCAST END RANK=%d TIME=%f;" % (rank, tt()) # now actually do the work if timing: print "~SCALER_WORKER_SETUP START RANK=%d TIME=%f;" % (rank, tt()) scaler_worker = self.scaler_class(transmitted_info["miller_set"], transmitted_info["model"], transmitted_info["params"], log=sys.stdout) if timing: print "~SCALER_WORKER_SETUP END RANK=%d TIME=%f;" % (rank, tt()) assert scaler_worker.params.backend == 'FS' # only option that makes sense from xfel.merging.database.merging_database_fs import manager2 as manager db_mgr = manager(scaler_worker.params) # Use client-server distribution of work to the available MPI ranks. # Each free rank requests a TAR ID and proceeds to process it. if scaler_worker.params.mpi.cs == True: tar_file_names = transmitted_info["file_names"] if timing: print "~SCALER_WORKERS START RANK=%d TIME=%f;" % (rank, tt()) if rank == 0: for ix in range(len(tar_file_names)): rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send(ix, dest=rankreq) for rankreq in range(size - 1): rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send('endrun', dest=rankreq) scaler_worker.finished_db_mgr = db_mgr else: while True: comm.send(rank, dest=0) idx = comm.recv(source=0) if idx == 'endrun': scaler_worker.finished_db_mgr = db_mgr break if timing: print "~SCALER_WORKER START=%d RANK=%d TIME=%f;" % ( idx, rank, tt()) scaler_worker._scale_all_serial([ tar_file_names[idx], ], db_mgr) if timing: print "~SCALER_WORKER END=%d RANK=%d TIME=%f;" % ( idx, rank, tt()) if timing: print "~SCALER_WORKERS END RANK=%d TIME=%f;" % (rank, tt()) # Distribute chunks of TAR files to each MPI rank. # The files are equidistributed across all the available ranks. else: file_names = [ transmitted_info["file_names"][i] for i in range(len(transmitted_info["file_names"])) if i % size == rank ] if timing: print "SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt()) scaler_worker._scale_all_serial(file_names, db_mgr) if timing: print "SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt()) scaler_worker.finished_db_mgr = db_mgr # might want to clean up a bit before returning del scaler_worker.log del scaler_worker.params del scaler_worker.miller_set del scaler_worker.i_model del scaler_worker.reverse_lookup # gather reports and all add together if timing: print "~GATHER START RANK=%d TIME=%f;" % (rank, tt()) reports = comm.gather(scaler_worker, root=0) if timing: print "~GATHER END RANK=%d TIME=%f;" % (rank, tt()) if rank == 0: print "Processing reports from %d ranks" % (len(reports)) ireport = 0 for item in reports: if timing: print "~SCALER_MASTER_ADD START RANK=%d TIME=%f;" % (rank, tt()) scaler_master._add_all_frames(item) if timing: print "~SCALER_MASTER_ADD END RANK=%d TIME=%f;" % (rank, tt()) print "processing %d calls from report %d" % (len( item.finished_db_mgr.sequencer), ireport) ireport += 1 for call_instance in item.finished_db_mgr.sequencer: if call_instance["call"] == "insert_frame": if timing: print "~SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f;" % ( rank, tt()) frame_id_zero_base = scaler_master.master_db_mgr.insert_frame( **call_instance["data"]) if timing: print "~SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f;" % ( rank, tt()) elif call_instance["call"] == "insert_observation": if timing: print "~SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f;" % ( rank, tt()) call_instance["data"]['frame_id_0_base'] = [ frame_id_zero_base ] * len(call_instance["data"]['frame_id_0_base']) scaler_master.master_db_mgr.insert_observation( **call_instance["data"]) if timing: print "~SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f;" % ( rank, tt()) if timing: print "~SCALER_MASTER_FINALISE START RANK=%d TIME=%f;" % (rank, tt()) scaler_master.master_db_mgr.join( ) # database written, finalize the manager scaler_master.mpi_finalize() if timing: print "~SCALER_MASTER_FINALISE END RANK=%d TIME=%f;" % (rank, tt()) return self.finalize(scaler_master)
def run(self, comm, timing=True): from mpi4py import MPI rank = comm.Get_rank() size = comm.Get_size() merge_op = MPI.Op.Create(self.mpi_merge_op, commute=True) # set things up if rank == 0: if timing: print "SETUP START RANK=%d TIME=%f" % (rank, tt()) self.initialize() self.validate() self.read_models() scaler_master = self.scaler_class(miller_set=self.miller_set, i_model=self.i_model, params=self.params, log=self.out) scaler_master.mpi_initialize(self.frame_files) transmitted_info = dict(file_names=self.frame_files, miller_set=self.miller_set, model=self.i_model, params=self.params) if timing: print "SETUP END RANK=%d TIME=%f" % (rank, tt()) else: if timing: print "SETUP START RANK=%d TIME=%f" % (rank, tt()) transmitted_info = None if timing: print "SETUP END RANK=%d TIME=%f" % (rank, tt()) if timing: print "BROADCAST START RANK=%d TIME=%f" % (rank, tt()) transmitted_info = comm.bcast(transmitted_info, root=0) if timing: print "BROADCAST END RANK=%d TIME=%f" % (rank, tt()) # now actually do the work if timing: print "SCALER_WORKER_SETUP START RANK=%d TIME=%f" % (rank, tt()) scaler_worker = self.scaler_class(transmitted_info["miller_set"], transmitted_info["model"], transmitted_info["params"], log=sys.stdout) if timing: print "SCALER_WORKER_SETUP END RANK=%d TIME=%f" % (rank, tt()) assert scaler_worker.params.backend == 'FS' # only option that makes sense from xfel.merging.database.merging_database_fs import manager2 as manager db_mgr = manager(scaler_worker.params) tar_file_names = transmitted_info["file_names"] if timing: print "SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt()) if rank == 0: for ix in range(len(tar_file_names)): if timing: print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % ( ix, rank, tt()) rankreq = comm.recv(source=MPI.ANY_SOURCE) if timing: print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % ( ix, rank, tt()) if timing: print "SCALER_WORKER_SEND START=%d RANK=%d,%d TIME=%f" % ( ix, rank, rankreq, tt()) comm.send(ix, dest=rankreq) if timing: print "SCALER_WORKER_SEND END=%d RANK=%d,%d TIME=%f" % ( ix, rank, rankreq, tt()) for rankreq in range(size - 1): if timing: print "SCALER_WORKER_RECV_KILL START RANK=%d TIME=%f" % ( rank, tt()) rankreq = comm.recv(source=MPI.ANY_SOURCE) if timing: print "SCALER_WORKER_RECV_KILL END RANK=%d TIME=%f" % ( rank, tt()) if timing: print "SCALER_WORKER_SEND_KILL START RANK=%d,%d TIME=%f" % ( rank, rankreq, tt()) comm.send('endrun', dest=rankreq) if timing: print "SCALER_WORKER_SEND_KILL END RANK=%d,%d TIME=%f" % ( rank, rankreq, tt()) scaler_worker.finished_db_mgr = db_mgr else: while True: if timing: print "SCALER_WORKER_RANKSEND START RANK=%d TIME=%f" % ( rank, tt()) comm.send(rank, dest=0) if timing: print "SCALER_WORKER_RANKSEND END RANK=%d TIME=%f" % (rank, tt()) if timing: print "SCALER_WORKER_IDXRECV START RANK=%d TIME=%f" % ( rank, tt()) idx = comm.recv(source=0) if timing: print "SCALER_WORKER_IDXRECV END RANK=%d TIME=%f" % (rank, tt()) if idx == 'endrun': scaler_worker.finished_db_mgr = db_mgr break if timing: print "SCALER_WORKER START=%s RANK=%d TIME=%f" % (str( tar_file_names[idx]), rank, tt()) scaler_worker._scale_all_serial([ tar_file_names[idx], ], db_mgr) if timing: print "SCALER_WORKER END=%s RANK=%d TIME=%f" % (str( tar_file_names[idx]), rank, tt()) if timing: print "SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt()) # might want to clean up a bit before returning del scaler_worker.log #del scaler_worker.params del scaler_worker.miller_set del scaler_worker.i_model del scaler_worker.reverse_lookup scaler_worker.myRank = rank if timing: print "SCALER_WORKERS_REDUCE START RANK=%d TIME=%f" % (rank, tt()) scaler_workers = comm.reduce(scaler_worker, op=merge_op, root=0) if timing: print "SCALER_WORKERS_REDUCE END RANK=%d TIME=%f" % (rank, tt()) MPI.Finalize() if rank == 0: if timing: print "SCALER_MASTER_ADD START RANK=%d TIME=%f" % (rank, tt()) scaler_master._add_all_frames(scaler_workers) if timing: print "SCALER_MASTER_ADD END RANK=%d TIME=%f" % (rank, tt()) for call_instance in scaler_workers.finished_db_mgr.sequencer: if call_instance["call"] == "insert_frame": if timing: print "SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f" % ( rank, tt()) frame_id_zero_base = scaler_master.master_db_mgr.insert_frame( **call_instance["data"]) if timing: print "SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f" % ( rank, tt()) elif call_instance["call"] == "insert_observation": if timing: print "SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f" % ( rank, tt()) call_instance["data"]['frame_id_0_base'] = [ frame_id_zero_base ] * len(call_instance["data"]['frame_id_0_base']) scaler_master.master_db_mgr.insert_observation( **call_instance["data"]) if timing: print "SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f" % ( rank, tt()) if timing: print "SCALER_MASTER_FINALISE START RANK=%d TIME=%f" % (rank, tt()) scaler_master.master_db_mgr.join( ) # database written, finalize the manager scaler_master.mpi_finalize() if timing: print "SCALER_MASTER_FINALISE END RANK=%d TIME=%f" % (rank, tt()) return self.finalize(scaler_master)
print "BROADCAST START RANK=%d TIME=%f" % (rank, tt()) transmitted_info = comm.bcast(transmitted_info, root=0) print "BROADCAST END RANK=%d TIME=%f" % (rank, tt()) # now actually do the work print "SCALER_WORKER_SETUP START RANK=%d TIME=%f" % (rank, tt()) scaler_worker = scaling_manager_mpi(transmitted_info["miller_set"], transmitted_info["model"], transmitted_info["params"], log=sys.stdout) print "SCALER_WORKER_SETUP END RANK=%d TIME=%f" % (rank, tt()) assert scaler_worker.params.backend == 'FS' # only option that makes sense from xfel.merging.database.merging_database_fs import manager2 as manager db_mgr = manager(scaler_worker.params) tar_file_names = transmitted_info["file_names"][0] print "SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt()) if rank == 0: for ix in xrange(len(tar_file_names)): print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % (ix, rank, tt()) rankreq = comm.recv(source=MPI.ANY_SOURCE) print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % (ix, rank, tt()) print "SCALER_WORKER_SEND START=%d RANK=%d,%d TIME=%f" % ( ix, rank, rankreq, tt()) comm.send(ix, dest=rankreq) print "SCALER_WORKER_SEND END=%d RANK=%d,%d TIME=%f" % ( ix, rank, rankreq, tt())
def run(self,comm,timing=False): from mpi4py import MPI rank = comm.Get_rank() size = comm.Get_size() from time import time as tt merge_op = MPI.Op.Create(self.mpi_merge_op, commute=True) # set things up if rank == 0: if timing: print "SETUP START RANK=%d TIME=%f"%(rank,tt()) self.initialize() self.validate() self.read_models() scaler_master = self.scaler_class( miller_set=self.miller_set, i_model=self.i_model, params=self.params, log=self.out) scaler_master.mpi_initialize(self.frame_files) transmitted_info = dict(file_names=self.frame_files, miller_set=self.miller_set, model = self.i_model, params = self.params ) if timing: print "SETUP END RANK=%d TIME=%f"%(rank,tt()) else: if timing: print "SETUP START RANK=%d TIME=%f"%(rank,tt()) transmitted_info = None if timing: print "SETUP END RANK=%d TIME=%f"%(rank,tt()) if timing: print "BROADCAST START RANK=%d TIME=%f"%(rank,tt()) transmitted_info = comm.bcast(transmitted_info, root = 0) if timing: print "BROADCAST END RANK=%d TIME=%f"%(rank,tt()) # now actually do the work if timing: print "SCALER_WORKER_SETUP START RANK=%d TIME=%f"%(rank,tt()) scaler_worker = self.scaler_class(transmitted_info["miller_set"], transmitted_info["model"], transmitted_info["params"], log = sys.stdout) if timing: print "SCALER_WORKER_SETUP END RANK=%d TIME=%f"%(rank,tt()) assert scaler_worker.params.backend == 'FS' # only option that makes sense from xfel.merging.database.merging_database_fs import manager2 as manager db_mgr = manager(scaler_worker.params) file_names = [transmitted_info["file_names"][i] for i in xrange(len(transmitted_info["file_names"])) if i%size == rank] if timing: print "SCALER_WORKERS START RANK=%d TIME=%f"%(rank, tt()) scaler_worker._scale_all_serial(file_names, db_mgr) if timing: print "SCALER_WORKERS END RANK=%d TIME=%f"%(rank, tt()) scaler_worker.finished_db_mgr = db_mgr # might want to clean up a bit before returning del scaler_worker.log #del scaler_worker.params del scaler_worker.miller_set del scaler_worker.i_model del scaler_worker.reverse_lookup if timing: print "SCALER_WORKERS_REDUCE START RANK=%d TIME=%f"%(rank, tt()) scaler_workers = comm.reduce(scaler_worker, op=merge_op, root=0) if timing: print "SCALER_WORKERS_REDUCE END RANK=%d TIME=%f"%(rank, tt()) MPI.Finalize() if rank == 0: if timing: print "SCALER_MASTER_ADD START RANK=%d TIME=%f"%(rank, tt()) scaler_master._add_all_frames(scaler_workers) if timing: print "SCALER_MASTER_ADD END RANK=%d TIME=%f"%(rank, tt()) for call_instance in scaler_workers.finished_db_mgr.sequencer: if call_instance["call"] == "insert_frame": if timing: print "SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f"%(rank, tt()) frame_id_zero_base = scaler_master.master_db_mgr.insert_frame(**call_instance["data"]) if timing: print "SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f"%(rank, tt()) elif call_instance["call"] == "insert_observation": if timing: print "SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f"%(rank, tt()) call_instance["data"]['frame_id_0_base'] = [frame_id_zero_base] * len(call_instance["data"]['frame_id_0_base']) scaler_master.master_db_mgr.insert_observation(**call_instance["data"]) if timing: print "SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f"%(rank, tt()) if timing: print "SCALER_MASTER_FINALISE START RANK=%d TIME=%f"%(rank, tt()) scaler_master.master_db_mgr.join() # database written, finalize the manager scaler_master.mpi_finalize() if timing: print "SCALER_MASTER_FINALISE END RANK=%d TIME=%f"%(rank, tt()) return self.finalize(scaler_master)