def tst_all(quick=False, prefix="step5"):
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
iterator = SS.generate_recast_renormalized_images(20,
energy=7120.,
total_flux=1e12)
#
C = microcrystal(
Deff_A=4000, length_um=4.,
beam_diameter_um=1.0) # assume smaller than 10 um crystals
if quick: prefix_root = prefix + "_%06d"
else: prefix_root = prefix + "laue_%06d"
Nimages = 1 # 10000
from LS49 import legacy_random_orientations
random_orientations = legacy_random_orientations(Nimages)
for iteration in range(Nimages):
file_prefix = prefix_root % iteration
rand_ori = sqr(random_orientations[iteration])
run_sim2smv(prefix=file_prefix,
crystal=C,
spectra=iterator,
rotation=rand_ori,
quick=quick,
rank=0)
def tst_all(
serial_no
): #emulates the action of step5_pad.py in assigning a coarse orientation to each simulation event
Nimages = 100000
from LS49 import legacy_random_orientations
random_orientations = legacy_random_orientations(Nimages)
for iteration in range(Nimages):
rand_ori = sqr(random_orientations[iteration])
if serial_no == iteration:
return rand_ori
return None
def get_partiality_response(key,one_index,spectra_simulation,ROI): N_total = 100000 # number of items to simulate spectra = spectra_simulation crystal = microcrystal(Deff_A = 4000, length_um = 4., beam_diameter_um = 1.0) # assume smaller than 10 um crystals from LS49 import legacy_random_orientations random_orientations = legacy_random_orientations(N_total) iterator = spectra.generate_recast_renormalized_image(image=key,energy=7120.,total_flux=1e12) file_prefix = "key_slow_nonoise_%06d"%key rand_ori = sqr(random_orientations[key]) pixels = run_sim2smv(ROI,prefix = file_prefix,crystal = crystal,spectra=iterator,rotation=rand_ori,quick=False,rank=0) return pixels
N_stride = size # total number of worker tasks
print("hello from rank %d of %d" % (rank, size), "with omp_threads=",
omp_get_num_procs())
import datetime
start_elapse = time()
if rank == 0:
print("Rank 0 time", datetime.datetime.now())
from LS49.spectra.generate_spectra import spectra_simulation
from LS49.sim.step5_pad import microcrystal
print("hello2 from rank %d of %d" % (rank, size))
SS = spectra_simulation()
C = microcrystal(
Deff_A=4000, length_um=4.,
beam_diameter_um=1.0) # assume smaller than 10 um crystals
from LS49 import legacy_random_orientations
random_orientations = legacy_random_orientations(N_total)
transmitted_info = dict(spectra=SS,
crystal=C,
random_orientations=random_orientations)
else:
transmitted_info = None
transmitted_info = comm.bcast(transmitted_info, root=0)
comm.barrier()
parcels = list(range(rank, N_total, N_stride))
while len(parcels) > 0:
import random
idx = random.choice(parcels)
cache_time = time()
print("idx------start-------->", idx, "rank", rank, time())
tst_one(
image=idx,
def run_LY99_batch(test_without_mpi=False):
params, options = parse_input()
log_by_rank = bool(int(os.environ.get("LOG_BY_RANK", 0)))
rank_profile = bool(int(os.environ.get("RANK_PROFILE", 1)))
if log_by_rank:
import io, sys
if rank_profile:
import cProfile
pr = cProfile.Profile()
pr.enable()
if test_without_mpi:
from LS49.adse13_196.mock_mpi import mpiEmulator
MPI = mpiEmulator()
else:
from libtbx.mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
import omptbx
workaround_nt = int(os.environ.get("OMP_NUM_THREADS", 1))
omptbx.omp_set_num_threads(workaround_nt)
N_total = int(os.environ["N_SIM"]) # number of items to simulate
N_stride = size # total number of worker tasks
print("hello from rank %d of %d" % (rank, size), "with omp_threads=",
omp_get_num_procs())
import datetime
start_comp = time()
# now inside the Python imports, begin energy channel calculation
wavelength_A = 1.74 # general ballpark X-ray wavelength in Angstroms
wavlen = flex.double([12398.425 / (7070.5 + w) for w in range(100)])
direct_algo_res_limit = 1.7
local_data = data() # later put this through broadcast
GF = gen_fmodel(resolution=direct_algo_res_limit,
pdb_text=local_data.get("pdb_lines"),
algorithm="fft",
wavelength=wavelength_A)
GF.set_k_sol(0.435)
GF.make_P1_primitive()
# Generating sf for my wavelengths
sfall_channels = {}
for x in range(len(wavlen)):
if rank > len(wavlen): break
if x % size != rank: continue
GF.reset_wavelength(wavlen[x])
GF.reset_specific_at_wavelength(
label_has="FE1",
tables=local_data.get("Fe_oxidized_model"),
newvalue=wavlen[x])
GF.reset_specific_at_wavelength(
label_has="FE2",
tables=local_data.get("Fe_reduced_model"),
newvalue=wavlen[x])
sfall_channels[x] = GF.get_amplitudes()
reports = comm.gather(sfall_channels, root=0)
if rank == 0:
sfall_channels = {}
for report in reports:
sfall_channels.update(report)
comm.barrier()
print(
rank, time(),
"finished with the calculation of channels, now construct single broadcast"
)
if rank == 0:
print("Rank 0 time", datetime.datetime.now())
from LS49.spectra.generate_spectra import spectra_simulation
from LS49.adse13_196.revapi.LY99_pad import microcrystal
print("hello2 from rank %d of %d" % (rank, size))
SS = spectra_simulation()
C = microcrystal(
Deff_A=4000, length_um=4.,
beam_diameter_um=1.0) # assume smaller than 10 um crystals
from LS49 import legacy_random_orientations
random_orientations = legacy_random_orientations(N_total)
transmitted_info = dict(spectra=SS,
crystal=C,
sfall_info=sfall_channels,
random_orientations=random_orientations)
else:
transmitted_info = None
transmitted_info = comm.bcast(transmitted_info, root=0)
comm.barrier()
parcels = list(range(rank, N_total, N_stride))
print(rank, time(),
"finished with single broadcast, now set up the rank logger")
if log_by_rank:
expand_dir = os.path.expandvars(params.logger.outdir)
log_path = os.path.join(expand_dir, "rank_%d.log" % rank)
error_path = os.path.join(expand_dir, "rank_%d.err" % rank)
#print("Rank %d redirecting stdout/stderr to"%rank, log_path, error_path)
sys.stdout = io.TextIOWrapper(open(log_path, 'ab', 0),
write_through=True)
sys.stderr = io.TextIOWrapper(open(error_path, 'ab', 0),
write_through=True)
print(
rank, time(),
"finished with the rank logger, now construct the GPU cache container")
import random
gpu_instance = get_exascale("gpu_instance", params.context)
gpu_energy_channels = get_exascale("gpu_energy_channels", params.context)
gpu_run = gpu_instance(deviceId=rank %
int(os.environ.get("DEVICES_PER_NODE", 1)))
gpu_channels_singleton = gpu_energy_channels(
deviceId=gpu_run.get_deviceID())
# singleton will instantiate, regardless of gpu, device count, or exascale API
comm.barrier()
while len(parcels) > 0:
idx = random.choice(parcels)
cache_time = time()
print("idx------start-------->", idx, "rank", rank, time())
# if rank==0: os.system("nvidia-smi")
tst_one(
image=idx,
spectra=transmitted_info["spectra"],
crystal=transmitted_info["crystal"],
random_orientation=transmitted_info["random_orientations"][idx],
sfall_channels=transmitted_info["sfall_info"],
gpu_channels_singleton=gpu_channels_singleton,
rank=rank,
params=params)
parcels.remove(idx)
print("idx------finis-------->", idx, "rank", rank, time(), "elapsed",
time() - cache_time)
comm.barrier()
del gpu_channels_singleton
# avoid Kokkos allocation "device_Fhkl" being deallocated after Kokkos::finalize was called
print("Overall rank", rank, "at", datetime.datetime.now(),
"seconds elapsed after srun startup %.3f" % (time() - start_elapse))
print("Overall rank", rank, "at", datetime.datetime.now(),
"seconds elapsed after Python imports %.3f" % (time() - start_comp))
if rank_profile:
pr.disable()
pr.dump_stats("cpu_%d.prof" % rank)
