def mpi_initialize(self, file_names):
self.integration_pickle_names = file_names
self.t1 = time.time()
assert self.params.backend == 'FS' # for prototyping rank=0 marshalling
from xfel.merging.database.merging_database_fs import manager
db_mgr = manager(self.params)
db_mgr.initialize_db(self.miller_set.indices())
self.master_db_mgr = db_mgr
def run(self, comm, timing=False):
rank = comm.Get_rank()
size = comm.Get_size()
from time import time as tt
# set things up
if rank == 0:
if timing: print("SETUP START RANK=%d TIME=%f" % (rank, tt()))
self.initialize()
self.validate()
self.read_models()
scaler_master = self.scaler_class(miller_set=self.miller_set,
i_model=self.i_model,
params=self.params,
log=self.out)
scaler_master.mpi_initialize(self.frame_files)
transmitted_info = dict(file_names=self.frame_files,
miller_set=self.miller_set,
model=self.i_model,
params=self.params)
if timing: print("SETUP END RANK=%d TIME=%f" % (rank, tt()))
else:
if timing: print("SETUP START RANK=%d TIME=%f" % (rank, tt()))
transmitted_info = None
if timing: print("SETUP END RANK=%d TIME=%f" % (rank, tt()))
if timing: print("BROADCAST START RANK=%d TIME=%f" % (rank, tt()))
transmitted_info = comm.bcast(transmitted_info, root=0)
if timing: print("BROADCAST END RANK=%d TIME=%f" % (rank, tt()))
# now actually do the work
if timing:
print("SCALER_WORKER_SETUP START RANK=%d TIME=%f" % (rank, tt()))
scaler_worker = self.scaler_class(transmitted_info["miller_set"],
transmitted_info["model"],
transmitted_info["params"],
log=sys.stdout)
if timing:
print("SCALER_WORKER_SETUP END RANK=%d TIME=%f" % (rank, tt()))
assert scaler_worker.params.backend == 'FS' # only option that makes sense
from xfel.merging.database.merging_database_fs import manager2 as manager
db_mgr = manager(scaler_worker.params)
file_names = [
transmitted_info["file_names"][i]
for i in range(len(transmitted_info["file_names"]))
if i % size == rank
]
if timing: print("SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt()))
scaler_worker._scale_all_serial(file_names, db_mgr)
if timing: print("SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt()))
scaler_worker.finished_db_mgr = db_mgr
# might want to clean up a bit before returning
del scaler_worker.log
del scaler_worker.params
del scaler_worker.miller_set
del scaler_worker.i_model
del scaler_worker.reverse_lookup
# gather reports and all add together
if timing: print("GATHER START RANK=%d TIME=%f" % (rank, tt()))
reports = comm.gather(scaler_worker, root=0)
if timing: print("GATHER END RANK=%d TIME=%f" % (rank, tt()))
if rank == 0:
print("Processing reports from %d ranks" % (len(reports)))
ireport = 0
for item in reports:
if timing:
print("SCALER_MASTER_ADD START RANK=%d TIME=%f" %
(rank, tt()))
scaler_master._add_all_frames(item)
if timing:
print("SCALER_MASTER_ADD END RANK=%d TIME=%f" %
(rank, tt()))
print("processing %d calls from report %d" %
(len(item.finished_db_mgr.sequencer), ireport))
ireport += 1
for call_instance in item.finished_db_mgr.sequencer:
if call_instance["call"] == "insert_frame":
if timing:
print(
"SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f"
% (rank, tt()))
frame_id_zero_base = scaler_master.master_db_mgr.insert_frame(
**call_instance["data"])
if timing:
print(
"SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f"
% (rank, tt()))
elif call_instance["call"] == "insert_observation":
if timing:
print(
"SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f"
% (rank, tt()))
call_instance["data"]['frame_id_0_base'] = [
frame_id_zero_base
] * len(call_instance["data"]['frame_id_0_base'])
scaler_master.master_db_mgr.insert_observation(
**call_instance["data"])
if timing:
print(
"SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f"
% (rank, tt()))
if timing:
print("SCALER_MASTER_FINALISE START RANK=%d TIME=%f" %
(rank, tt()))
scaler_master.master_db_mgr.join(
) # database written, finalize the manager
scaler_master.mpi_finalize()
if timing:
print("SCALER_MASTER_FINALISE END RANK=%d TIME=%f" %
(rank, tt()))
return self.finalize(scaler_master)
def run(self, comm, timing=False):
rank = comm.Get_rank()
size = comm.Get_size()
from time import time as tt
# set things up
if rank == 0:
self.initialize()
self.validate(comm)
self.read_models()
timing = self.params.mpi.logging
if timing: print "~SETUP START RANK=%d TIME=%f;" % (rank, tt())
scaler_master = self.scaler_class(miller_set=self.miller_set,
i_model=self.i_model,
params=self.params,
log=self.out)
scaler_master.mpi_initialize(self.frame_files)
transmitted_info = dict(file_names=self.frame_files,
miller_set=self.miller_set,
model=self.i_model,
params=self.params)
if timing: print "~SETUP END RANK=%d TIME=%f;" % (rank, tt())
else:
if timing: print "~SETUP START RANK=%d TIME=%f;" % (rank, tt())
transmitted_info = None
if timing: print "~SETUP END RANK=%d TIME=%f;" % (rank, tt())
if timing: print "~BROADCAST START RANK=%d TIME=%f;" % (rank, tt())
transmitted_info = comm.bcast(transmitted_info, root=0)
if timing: print "~BROADCAST END RANK=%d TIME=%f;" % (rank, tt())
# now actually do the work
if timing:
print "~SCALER_WORKER_SETUP START RANK=%d TIME=%f;" % (rank, tt())
scaler_worker = self.scaler_class(transmitted_info["miller_set"],
transmitted_info["model"],
transmitted_info["params"],
log=sys.stdout)
if timing:
print "~SCALER_WORKER_SETUP END RANK=%d TIME=%f;" % (rank, tt())
assert scaler_worker.params.backend == 'FS' # only option that makes sense
from xfel.merging.database.merging_database_fs import manager2 as manager
db_mgr = manager(scaler_worker.params)
# Use client-server distribution of work to the available MPI ranks.
# Each free rank requests a TAR ID and proceeds to process it.
if scaler_worker.params.mpi.cs == True:
tar_file_names = transmitted_info["file_names"]
if timing:
print "~SCALER_WORKERS START RANK=%d TIME=%f;" % (rank, tt())
if rank == 0:
for ix in range(len(tar_file_names)):
rankreq = comm.recv(source=MPI.ANY_SOURCE)
comm.send(ix, dest=rankreq)
for rankreq in range(size - 1):
rankreq = comm.recv(source=MPI.ANY_SOURCE)
comm.send('endrun', dest=rankreq)
scaler_worker.finished_db_mgr = db_mgr
else:
while True:
comm.send(rank, dest=0)
idx = comm.recv(source=0)
if idx == 'endrun':
scaler_worker.finished_db_mgr = db_mgr
break
if timing:
print "~SCALER_WORKER START=%d RANK=%d TIME=%f;" % (
idx, rank, tt())
scaler_worker._scale_all_serial([
tar_file_names[idx],
], db_mgr)
if timing:
print "~SCALER_WORKER END=%d RANK=%d TIME=%f;" % (
idx, rank, tt())
if timing:
print "~SCALER_WORKERS END RANK=%d TIME=%f;" % (rank, tt())
# Distribute chunks of TAR files to each MPI rank.
# The files are equidistributed across all the available ranks.
else:
file_names = [
transmitted_info["file_names"][i]
for i in range(len(transmitted_info["file_names"]))
if i % size == rank
]
if timing:
print "SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt())
scaler_worker._scale_all_serial(file_names, db_mgr)
if timing:
print "SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt())
scaler_worker.finished_db_mgr = db_mgr
# might want to clean up a bit before returning
del scaler_worker.log
del scaler_worker.params
del scaler_worker.miller_set
del scaler_worker.i_model
del scaler_worker.reverse_lookup
# gather reports and all add together
if timing: print "~GATHER START RANK=%d TIME=%f;" % (rank, tt())
reports = comm.gather(scaler_worker, root=0)
if timing: print "~GATHER END RANK=%d TIME=%f;" % (rank, tt())
if rank == 0:
print "Processing reports from %d ranks" % (len(reports))
ireport = 0
for item in reports:
if timing:
print "~SCALER_MASTER_ADD START RANK=%d TIME=%f;" % (rank,
tt())
scaler_master._add_all_frames(item)
if timing:
print "~SCALER_MASTER_ADD END RANK=%d TIME=%f;" % (rank,
tt())
print "processing %d calls from report %d" % (len(
item.finished_db_mgr.sequencer), ireport)
ireport += 1
for call_instance in item.finished_db_mgr.sequencer:
if call_instance["call"] == "insert_frame":
if timing:
print "~SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f;" % (
rank, tt())
frame_id_zero_base = scaler_master.master_db_mgr.insert_frame(
**call_instance["data"])
if timing:
print "~SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f;" % (
rank, tt())
elif call_instance["call"] == "insert_observation":
if timing:
print "~SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f;" % (
rank, tt())
call_instance["data"]['frame_id_0_base'] = [
frame_id_zero_base
] * len(call_instance["data"]['frame_id_0_base'])
scaler_master.master_db_mgr.insert_observation(
**call_instance["data"])
if timing:
print "~SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f;" % (
rank, tt())
if timing:
print "~SCALER_MASTER_FINALISE START RANK=%d TIME=%f;" % (rank,
tt())
scaler_master.master_db_mgr.join(
) # database written, finalize the manager
scaler_master.mpi_finalize()
if timing:
print "~SCALER_MASTER_FINALISE END RANK=%d TIME=%f;" % (rank,
tt())
return self.finalize(scaler_master)
def run(self, comm, timing=True):
from mpi4py import MPI
rank = comm.Get_rank()
size = comm.Get_size()
merge_op = MPI.Op.Create(self.mpi_merge_op, commute=True)
# set things up
if rank == 0:
if timing: print "SETUP START RANK=%d TIME=%f" % (rank, tt())
self.initialize()
self.validate()
self.read_models()
scaler_master = self.scaler_class(miller_set=self.miller_set,
i_model=self.i_model,
params=self.params,
log=self.out)
scaler_master.mpi_initialize(self.frame_files)
transmitted_info = dict(file_names=self.frame_files,
miller_set=self.miller_set,
model=self.i_model,
params=self.params)
if timing: print "SETUP END RANK=%d TIME=%f" % (rank, tt())
else:
if timing: print "SETUP START RANK=%d TIME=%f" % (rank, tt())
transmitted_info = None
if timing: print "SETUP END RANK=%d TIME=%f" % (rank, tt())
if timing: print "BROADCAST START RANK=%d TIME=%f" % (rank, tt())
transmitted_info = comm.bcast(transmitted_info, root=0)
if timing: print "BROADCAST END RANK=%d TIME=%f" % (rank, tt())
# now actually do the work
if timing:
print "SCALER_WORKER_SETUP START RANK=%d TIME=%f" % (rank, tt())
scaler_worker = self.scaler_class(transmitted_info["miller_set"],
transmitted_info["model"],
transmitted_info["params"],
log=sys.stdout)
if timing:
print "SCALER_WORKER_SETUP END RANK=%d TIME=%f" % (rank, tt())
assert scaler_worker.params.backend == 'FS' # only option that makes sense
from xfel.merging.database.merging_database_fs import manager2 as manager
db_mgr = manager(scaler_worker.params)
tar_file_names = transmitted_info["file_names"]
if timing: print "SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt())
if rank == 0:
for ix in range(len(tar_file_names)):
if timing:
print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % (
ix, rank, tt())
rankreq = comm.recv(source=MPI.ANY_SOURCE)
if timing:
print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % (
ix, rank, tt())
if timing:
print "SCALER_WORKER_SEND START=%d RANK=%d,%d TIME=%f" % (
ix, rank, rankreq, tt())
comm.send(ix, dest=rankreq)
if timing:
print "SCALER_WORKER_SEND END=%d RANK=%d,%d TIME=%f" % (
ix, rank, rankreq, tt())
for rankreq in range(size - 1):
if timing:
print "SCALER_WORKER_RECV_KILL START RANK=%d TIME=%f" % (
rank, tt())
rankreq = comm.recv(source=MPI.ANY_SOURCE)
if timing:
print "SCALER_WORKER_RECV_KILL END RANK=%d TIME=%f" % (
rank, tt())
if timing:
print "SCALER_WORKER_SEND_KILL START RANK=%d,%d TIME=%f" % (
rank, rankreq, tt())
comm.send('endrun', dest=rankreq)
if timing:
print "SCALER_WORKER_SEND_KILL END RANK=%d,%d TIME=%f" % (
rank, rankreq, tt())
scaler_worker.finished_db_mgr = db_mgr
else:
while True:
if timing:
print "SCALER_WORKER_RANKSEND START RANK=%d TIME=%f" % (
rank, tt())
comm.send(rank, dest=0)
if timing:
print "SCALER_WORKER_RANKSEND END RANK=%d TIME=%f" % (rank,
tt())
if timing:
print "SCALER_WORKER_IDXRECV START RANK=%d TIME=%f" % (
rank, tt())
idx = comm.recv(source=0)
if timing:
print "SCALER_WORKER_IDXRECV END RANK=%d TIME=%f" % (rank,
tt())
if idx == 'endrun':
scaler_worker.finished_db_mgr = db_mgr
break
if timing:
print "SCALER_WORKER START=%s RANK=%d TIME=%f" % (str(
tar_file_names[idx]), rank, tt())
scaler_worker._scale_all_serial([
tar_file_names[idx],
], db_mgr)
if timing:
print "SCALER_WORKER END=%s RANK=%d TIME=%f" % (str(
tar_file_names[idx]), rank, tt())
if timing: print "SCALER_WORKERS END RANK=%d TIME=%f" % (rank, tt())
# might want to clean up a bit before returning
del scaler_worker.log
#del scaler_worker.params
del scaler_worker.miller_set
del scaler_worker.i_model
del scaler_worker.reverse_lookup
scaler_worker.myRank = rank
if timing:
print "SCALER_WORKERS_REDUCE START RANK=%d TIME=%f" % (rank, tt())
scaler_workers = comm.reduce(scaler_worker, op=merge_op, root=0)
if timing:
print "SCALER_WORKERS_REDUCE END RANK=%d TIME=%f" % (rank, tt())
MPI.Finalize()
if rank == 0:
if timing:
print "SCALER_MASTER_ADD START RANK=%d TIME=%f" % (rank, tt())
scaler_master._add_all_frames(scaler_workers)
if timing:
print "SCALER_MASTER_ADD END RANK=%d TIME=%f" % (rank, tt())
for call_instance in scaler_workers.finished_db_mgr.sequencer:
if call_instance["call"] == "insert_frame":
if timing:
print "SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f" % (
rank, tt())
frame_id_zero_base = scaler_master.master_db_mgr.insert_frame(
**call_instance["data"])
if timing:
print "SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f" % (
rank, tt())
elif call_instance["call"] == "insert_observation":
if timing:
print "SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f" % (
rank, tt())
call_instance["data"]['frame_id_0_base'] = [
frame_id_zero_base
] * len(call_instance["data"]['frame_id_0_base'])
scaler_master.master_db_mgr.insert_observation(
**call_instance["data"])
if timing:
print "SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f" % (
rank, tt())
if timing:
print "SCALER_MASTER_FINALISE START RANK=%d TIME=%f" % (rank,
tt())
scaler_master.master_db_mgr.join(
) # database written, finalize the manager
scaler_master.mpi_finalize()
if timing:
print "SCALER_MASTER_FINALISE END RANK=%d TIME=%f" % (rank,
tt())
return self.finalize(scaler_master)
print "BROADCAST START RANK=%d TIME=%f" % (rank, tt())
transmitted_info = comm.bcast(transmitted_info, root=0)
print "BROADCAST END RANK=%d TIME=%f" % (rank, tt())
# now actually do the work
print "SCALER_WORKER_SETUP START RANK=%d TIME=%f" % (rank, tt())
scaler_worker = scaling_manager_mpi(transmitted_info["miller_set"],
transmitted_info["model"],
transmitted_info["params"],
log=sys.stdout)
print "SCALER_WORKER_SETUP END RANK=%d TIME=%f" % (rank, tt())
assert scaler_worker.params.backend == 'FS' # only option that makes sense
from xfel.merging.database.merging_database_fs import manager2 as manager
db_mgr = manager(scaler_worker.params)
tar_file_names = transmitted_info["file_names"][0]
print "SCALER_WORKERS START RANK=%d TIME=%f" % (rank, tt())
if rank == 0:
for ix in xrange(len(tar_file_names)):
print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % (ix, rank,
tt())
rankreq = comm.recv(source=MPI.ANY_SOURCE)
print "SCALER_WORKER_RECV START=%d RANK=%d TIME=%f" % (ix, rank,
tt())
print "SCALER_WORKER_SEND START=%d RANK=%d,%d TIME=%f" % (
ix, rank, rankreq, tt())
comm.send(ix, dest=rankreq)
print "SCALER_WORKER_SEND END=%d RANK=%d,%d TIME=%f" % (
ix, rank, rankreq, tt())
def run(self,comm,timing=False):
from mpi4py import MPI
rank = comm.Get_rank()
size = comm.Get_size()
from time import time as tt
merge_op = MPI.Op.Create(self.mpi_merge_op, commute=True)
# set things up
if rank == 0:
if timing: print "SETUP START RANK=%d TIME=%f"%(rank,tt())
self.initialize()
self.validate()
self.read_models()
scaler_master = self.scaler_class(
miller_set=self.miller_set,
i_model=self.i_model,
params=self.params,
log=self.out)
scaler_master.mpi_initialize(self.frame_files)
transmitted_info = dict(file_names=self.frame_files,
miller_set=self.miller_set,
model = self.i_model,
params = self.params )
if timing: print "SETUP END RANK=%d TIME=%f"%(rank,tt())
else:
if timing: print "SETUP START RANK=%d TIME=%f"%(rank,tt())
transmitted_info = None
if timing: print "SETUP END RANK=%d TIME=%f"%(rank,tt())
if timing: print "BROADCAST START RANK=%d TIME=%f"%(rank,tt())
transmitted_info = comm.bcast(transmitted_info, root = 0)
if timing: print "BROADCAST END RANK=%d TIME=%f"%(rank,tt())
# now actually do the work
if timing: print "SCALER_WORKER_SETUP START RANK=%d TIME=%f"%(rank,tt())
scaler_worker = self.scaler_class(transmitted_info["miller_set"],
transmitted_info["model"],
transmitted_info["params"],
log = sys.stdout)
if timing: print "SCALER_WORKER_SETUP END RANK=%d TIME=%f"%(rank,tt())
assert scaler_worker.params.backend == 'FS' # only option that makes sense
from xfel.merging.database.merging_database_fs import manager2 as manager
db_mgr = manager(scaler_worker.params)
file_names = [transmitted_info["file_names"][i] for i in xrange(len(transmitted_info["file_names"])) if i%size == rank]
if timing: print "SCALER_WORKERS START RANK=%d TIME=%f"%(rank, tt())
scaler_worker._scale_all_serial(file_names, db_mgr)
if timing: print "SCALER_WORKERS END RANK=%d TIME=%f"%(rank, tt())
scaler_worker.finished_db_mgr = db_mgr
# might want to clean up a bit before returning
del scaler_worker.log
#del scaler_worker.params
del scaler_worker.miller_set
del scaler_worker.i_model
del scaler_worker.reverse_lookup
if timing: print "SCALER_WORKERS_REDUCE START RANK=%d TIME=%f"%(rank, tt())
scaler_workers = comm.reduce(scaler_worker, op=merge_op, root=0)
if timing: print "SCALER_WORKERS_REDUCE END RANK=%d TIME=%f"%(rank, tt())
MPI.Finalize()
if rank == 0:
if timing: print "SCALER_MASTER_ADD START RANK=%d TIME=%f"%(rank, tt())
scaler_master._add_all_frames(scaler_workers)
if timing: print "SCALER_MASTER_ADD END RANK=%d TIME=%f"%(rank, tt())
for call_instance in scaler_workers.finished_db_mgr.sequencer:
if call_instance["call"] == "insert_frame":
if timing: print "SCALER_MASTER_INSERT_FRAME START RANK=%d TIME=%f"%(rank, tt())
frame_id_zero_base = scaler_master.master_db_mgr.insert_frame(**call_instance["data"])
if timing: print "SCALER_MASTER_INSERT_FRAME END RANK=%d TIME=%f"%(rank, tt())
elif call_instance["call"] == "insert_observation":
if timing: print "SCALER_MASTER_INSERT_OBS START RANK=%d TIME=%f"%(rank, tt())
call_instance["data"]['frame_id_0_base'] = [frame_id_zero_base] * len(call_instance["data"]['frame_id_0_base'])
scaler_master.master_db_mgr.insert_observation(**call_instance["data"])
if timing: print "SCALER_MASTER_INSERT_OBS END RANK=%d TIME=%f"%(rank, tt())
if timing: print "SCALER_MASTER_FINALISE START RANK=%d TIME=%f"%(rank, tt())
scaler_master.master_db_mgr.join() # database written, finalize the manager
scaler_master.mpi_finalize()
if timing: print "SCALER_MASTER_FINALISE END RANK=%d TIME=%f"%(rank, tt())
return self.finalize(scaler_master)
