def tst_all(quick=False, prefix="step5"):
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
iterator = SS.generate_recast_renormalized_images(20,
energy=7120.,
total_flux=1e12)
#
C = microcrystal(
Deff_A=4000, length_um=4.,
beam_diameter_um=1.0) # assume smaller than 10 um crystals
if quick: prefix_root = prefix + "_%06d"
else: prefix_root = prefix + "laue_%06d"
Nimages = 1 # 10000
from LS49 import legacy_random_orientations
random_orientations = legacy_random_orientations(Nimages)
for iteration in range(Nimages):
file_prefix = prefix_root % iteration
rand_ori = sqr(random_orientations[iteration])
run_sim2smv(prefix=file_prefix,
crystal=C,
spectra=iterator,
rotation=rand_ori,
quick=quick,
rank=0)
def tst_all(quick=False, prefix="step6"):
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
iterator = SS.generate_recast_renormalized_images(20,
energy=7120.,
total_flux=1e12)
#
C = microcrystal(
Deff_A=4000, length_um=4.,
beam_diameter_um=1.0) # assume smaller than 10 um crystals
mt = flex.mersenne_twister(seed=0)
if quick: prefix_root = prefix + "_%06d"
else: prefix_root = prefix + "poly_%06d"
Nimages = 1 # 10000
for iteration in range(Nimages):
file_prefix = prefix_root % iteration
rand_ori = sqr(mt.random_double_r3_rotation_matrix())
run_sim2smv(prefix=file_prefix,
crystal=C,
spectra=iterator,
rotation=rand_ori,
quick=quick,
rank=0)
def tst_iterators():
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
import six
if six.PY3:
reference = cPickle.load(
open(os.path.join(ls49_big_data,"reference","tst_spectrum_iterator_data"),"rb"),encoding="bytes")
else:
reference = cPickle.load(open(os.path.join(ls49_big_data,"reference","tst_spectrum_iterator_data"),"rb"))
# use of single iterator with next()
iterator = SS.generate_recast_renormalized_images(20,energy=7120.,total_flux=1e12)
for x in range(20):
wavlen, flux, wavelength_A = next(iterator) # list of lambdas, list of fluxes, average wavelength
assert approx_equal(wavlen,reference[x][0]), "wavelength axis"
assert approx_equal(flux, reference[x][1]), "flux axis"
assert approx_equal(wavelength_A,reference[x][2],eps=1E-10), "mean wavelength"
# get a new iterator for every event
for x in range(10):
iterator = SS.generate_recast_renormalized_image(image=x,energy=7120.,total_flux=1e12)
wavlen, flux, wavelength_A = next(iterator) # list of lambdas, list of fluxes, average wavelength
assert approx_equal(wavlen,reference[x][0]), "iterator 2 wavelength axis"
assert approx_equal(flux, reference[x][1]), "iterator 2 flux axis"
assert approx_equal(wavelength_A, reference[x][2],eps=1E-10), "iterator 2 mean wavelength"
def create_reference_results():
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
iterator = SS.generate_recast_renormalized_images(20,energy=7120.,total_flux=1e12)
results=[]
#SS.plot_recast_images(20,7120.) # optionally plot these spectra
for x in range(20):
wavlen, flux, wavelength_A = next(iterator) # list of lambdas, list of fluxes, average wavelength
results.append((wavlen, flux, wavelength_A))
cPickle.dump(results,open(os.path.join(ls49_big_data,"reference","tst_spectrum_iterator_data"),"wb"),cPickle.HIGHEST_PROTOCOL)
def channel_wavelength_fmodel(create):
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
iterator = SS.generate_recast_renormalized_images(20,
energy=7120.,
total_flux=1e12)
wavlen, flux, wavelength_A = next(
iterator) # list of lambdas, list of fluxes, average wavelength
direct_algo_res_limit = 1.7
from LS49.sim.util_fmodel import gen_fmodel
GF = gen_fmodel(resolution=direct_algo_res_limit,
pdb_text=get_pdb_lines(),
algorithm="fft",
wavelength=wavelength_A)
GF.set_k_sol(0.435)
GF.make_P1_primitive()
for x in range(10, len(flux), 5):
print("+++++++++++++++++++++++++++++++++++++++ Wavelength", x)
GF.reset_wavelength(wavelength_A)
GF.reset_specific_at_wavelength(label_has="FE1",
tables=Fe_oxidized_model,
newvalue=wavelength_A)
GF.reset_specific_at_wavelength(label_has="FE2",
tables=Fe_reduced_model,
newvalue=wavelength_A)
sfall_channel = GF.get_amplitudes()
filename = "sf_reference_channel_%s" % ("%03d" % x)
if create: # write the reference for the first time
cPickle.dump(
sfall_channel,
open(os.path.join(ls49_big_data, "reference", filename), "wb"),
cPickle.HIGHEST_PROTOCOL)
else: # read the reference and assert sameness to sfall_channel
print(os.path.join(ls49_big_data, "reference", filename))
if six.PY3:
sfall_ref = cPickle.load(open(
os.path.join(ls49_big_data, "reference", filename), "rb"),
encoding="bytes")
fix_unpickled_attributes(sfall_ref)
else:
sfall_ref = cPickle.load(
open(os.path.join(ls49_big_data, "reference", filename),
"rb"))
T = sfall_channel
S = sfall_ref
assert S.space_group() == T.space_group()
assert S.unit_cell().parameters() == T.unit_cell().parameters()
assert S.indices() == T.indices()
assert S.data() == T.data()
def single_wavelength_fmodel(create):
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
iterator = SS.generate_recast_renormalized_images(20,
energy=7120.,
total_flux=1e12)
wavlen, flux, wavelength_A = next(
iterator) # list of lambdas, list of fluxes, average wavelength
direct_algo_res_limit = 1.7
from LS49.sim.util_fmodel import gen_fmodel
for flag in [True, False]:
GF = gen_fmodel(resolution=direct_algo_res_limit,
pdb_text=get_pdb_lines(),
algorithm="fft",
wavelength=wavelength_A)
GF.set_k_sol(0.435)
if flag: GF.make_P1_primitive()
sfall_main = GF.get_amplitudes()
sfall_main.show_summary(prefix="Amplitudes used ")
if create: # write the reference for the first time
cPickle.dump(
sfall_main,
open(
os.path.join(ls49_big_data, "reference",
"sf_reference_cb_to_P1_%s" % (str(flag))),
"wb"), cPickle.HIGHEST_PROTOCOL)
else: # read the reference and assert sameness to sfall_main
if six.PY3:
sfall_ref = cPickle.load(open(
os.path.join(ls49_big_data, "reference",
"sf_reference_cb_to_P1_%s" % (str(flag))),
"rb"),
encoding="bytes")
from LS49.tests.tst_sf_energies import fix_unpickled_attributes
fix_unpickled_attributes(sfall_ref)
else:
sfall_ref = cPickle.load(
open(
os.path.join(ls49_big_data, "reference",
"sf_reference_cb_to_P1_%s" % (str(flag))),
"rb"))
T = sfall_main
S = sfall_ref
assert S.space_group() == T.space_group()
assert S.unit_cell().parameters() == T.unit_cell().parameters()
assert S.indices() == T.indices()
assert S.data() == T.data()
print()
def test_Gi_factor(G):
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
x = flex.double()
y = flex.double()
for key in G.images_strong:
print (key,"of",len(G.images_strong),G.images_Gi[key])
#trying here to plot the Gi against the integrated spectrum for each event.
iterator = SS.generate_recast_renormalized_image(image=key,energy=7120.,total_flux=1e12)
wavlen, flux, wavelength_A = next(iterator) # list of lambdas, list of fluxes, average wavelength
total_flux = flex.sum(flux)
x.append(total_flux)
y.append(G.images_Gi[key])
from matplotlib import pyplot as plt
plt.plot(x,y,"r.")
plt.show()
def tst_all():
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
#SS.plot_recast_images(20,energy=7150.)
iterator = SS.generate_recast_renormalized_images(20,
energy=7150.,
total_flux=1e12)
#
fileout = "step3noiseimage_001.cbf"
#
C = microcrystal(Deff_A=4000, length_um=5.,
beam_diameter_um=3.) # assume smaller than 10 um crystals
mt = flex.mersenne_twister(seed=0)
rand_ori = sqr(mt.random_double_r3_rotation_matrix())
run_sim2smv(
fileout=fileout,
crystal=C,
spectra=iterator,
rotation=rand_ori,
)
import os
assert os.path.isfile(fileout)
omptbx.omp_set_num_threads(workaround_nt)
print("## hello from rank %d of %d"%(rank,size),"with omp_threads=",omp_get_num_procs())
## assign jobs
rank_in_pdb, size_in_pdb, ranks_in_pdb = jobAssign(size=size, num_pdb=simparams.num_pdbs, num_img=simparams.num_img[0])
import datetime
start_elapse = time.time()
if rank == 0:
print("Rank 0 time", datetime.datetime.now())
from LS49.spectra.generate_spectra import spectra_simulation
SS = spectra_simulation()
C = microcrystal(Deff_A = simparams.Deff_A, length_um = simparams.length_um, beam_diameter_um = simparams.beam_diameter_um)
# assume smaller than 10 um crystals
mt = flex.mersenne_twister(seed=0)
random_orientations = []
for iteration in range( sum(simparams.num_img) ):
random_orientations.append( mt.random_double_r3_rotation_matrix() )
# for ii in range(10): print("## TOP 10 orientations = ", random_orientations[ii])
print("## total orientations = ", len(random_orientations))
transmitted_info = dict(spectra = SS, crystal = C, random_orientations = random_orientations)
for idx_pdb in range( len(simparams.pdb_files) ):
save_folder = "./" + simparams.prefix + "_" + str(idx_pdb).zfill(3)
if not os.path.isdir(save_folder):
def run_LY99_batch(test_without_mpi=False):
params, options = parse_input()
log_by_rank = bool(int(os.environ.get("LOG_BY_RANK", 0)))
rank_profile = bool(int(os.environ.get("RANK_PROFILE", 1)))
if log_by_rank:
import io, sys
if rank_profile:
import cProfile
pr = cProfile.Profile()
pr.enable()
if test_without_mpi:
from LS49.adse13_196.mock_mpi import mpiEmulator
MPI = mpiEmulator()
else:
from libtbx.mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
import omptbx
workaround_nt = int(os.environ.get("OMP_NUM_THREADS", 1))
omptbx.omp_set_num_threads(workaround_nt)
N_total = int(os.environ["N_SIM"]) # number of items to simulate
N_stride = size # total number of worker tasks
print("hello from rank %d of %d" % (rank, size), "with omp_threads=",
omp_get_num_procs())
import datetime
start_comp = time()
# now inside the Python imports, begin energy channel calculation
wavelength_A = 1.74 # general ballpark X-ray wavelength in Angstroms
wavlen = flex.double([12398.425 / (7070.5 + w) for w in range(100)])
direct_algo_res_limit = 1.7
local_data = data() # later put this through broadcast
GF = gen_fmodel(resolution=direct_algo_res_limit,
pdb_text=local_data.get("pdb_lines"),
algorithm="fft",
wavelength=wavelength_A)
GF.set_k_sol(0.435)
GF.make_P1_primitive()
# Generating sf for my wavelengths
sfall_channels = {}
for x in range(len(wavlen)):
if rank > len(wavlen): break
if x % size != rank: continue
GF.reset_wavelength(wavlen[x])
GF.reset_specific_at_wavelength(
label_has="FE1",
tables=local_data.get("Fe_oxidized_model"),
newvalue=wavlen[x])
GF.reset_specific_at_wavelength(
label_has="FE2",
tables=local_data.get("Fe_reduced_model"),
newvalue=wavlen[x])
sfall_channels[x] = GF.get_amplitudes()
reports = comm.gather(sfall_channels, root=0)
if rank == 0:
sfall_channels = {}
for report in reports:
sfall_channels.update(report)
comm.barrier()
print(
rank, time(),
"finished with the calculation of channels, now construct single broadcast"
)
if rank == 0:
print("Rank 0 time", datetime.datetime.now())
from LS49.spectra.generate_spectra import spectra_simulation
from LS49.adse13_196.revapi.LY99_pad import microcrystal
print("hello2 from rank %d of %d" % (rank, size))
SS = spectra_simulation()
C = microcrystal(
Deff_A=4000, length_um=4.,
beam_diameter_um=1.0) # assume smaller than 10 um crystals
from LS49 import legacy_random_orientations
random_orientations = legacy_random_orientations(N_total)
transmitted_info = dict(spectra=SS,
crystal=C,
sfall_info=sfall_channels,
random_orientations=random_orientations)
else:
transmitted_info = None
transmitted_info = comm.bcast(transmitted_info, root=0)
comm.barrier()
parcels = list(range(rank, N_total, N_stride))
print(rank, time(),
"finished with single broadcast, now set up the rank logger")
if log_by_rank:
expand_dir = os.path.expandvars(params.logger.outdir)
log_path = os.path.join(expand_dir, "rank_%d.log" % rank)
error_path = os.path.join(expand_dir, "rank_%d.err" % rank)
#print("Rank %d redirecting stdout/stderr to"%rank, log_path, error_path)
sys.stdout = io.TextIOWrapper(open(log_path, 'ab', 0),
write_through=True)
sys.stderr = io.TextIOWrapper(open(error_path, 'ab', 0),
write_through=True)
print(
rank, time(),
"finished with the rank logger, now construct the GPU cache container")
import random
gpu_instance = get_exascale("gpu_instance", params.context)
gpu_energy_channels = get_exascale("gpu_energy_channels", params.context)
gpu_run = gpu_instance(deviceId=rank %
int(os.environ.get("DEVICES_PER_NODE", 1)))
gpu_channels_singleton = gpu_energy_channels(
deviceId=gpu_run.get_deviceID())
# singleton will instantiate, regardless of gpu, device count, or exascale API
comm.barrier()
while len(parcels) > 0:
idx = random.choice(parcels)
cache_time = time()
print("idx------start-------->", idx, "rank", rank, time())
# if rank==0: os.system("nvidia-smi")
tst_one(
image=idx,
spectra=transmitted_info["spectra"],
crystal=transmitted_info["crystal"],
random_orientation=transmitted_info["random_orientations"][idx],
sfall_channels=transmitted_info["sfall_info"],
gpu_channels_singleton=gpu_channels_singleton,
rank=rank,
params=params)
parcels.remove(idx)
print("idx------finis-------->", idx, "rank", rank, time(), "elapsed",
time() - cache_time)
comm.barrier()
del gpu_channels_singleton
# avoid Kokkos allocation "device_Fhkl" being deallocated after Kokkos::finalize was called
print("Overall rank", rank, "at", datetime.datetime.now(),
"seconds elapsed after srun startup %.3f" % (time() - start_elapse))
print("Overall rank", rank, "at", datetime.datetime.now(),
"seconds elapsed after Python imports %.3f" % (time() - start_comp))
if rank_profile:
pr.disable()
pr.dump_stats("cpu_%d.prof" % rank)
def run(self):
self.parse_input()
N_total = 100000 # self.params.N_total # number of items to simulate, nominally 100000
logical_rank = self.mpi_helper.rank
logical_size = self.mpi_helper.size
if self.mpi_helper.rank == 0 and self.mpi_helper.size == 1: # special case of testing it
try:
logical_rank = self.params.tester.rank
logical_size = self.params.tester.size
except Exception:
pass
N_stride = int(math.ceil(
N_total / logical_size)) # total number of tasks per rank
print("hello from rank %d of %d with stride %d" %
(logical_rank, logical_size, N_stride))
#from scitbx.lbfgs.tst_mpi_split_evaluator import mpi_split_evaluator_run
#from scitbx.lbfgs.tst_mpi_split_evaluator import run_mpi as simple_tester
#simple_tester()
if self.params.starting_model.algorithm == "to_file":
if self.mpi_helper.rank == 0:
HKL_lookup, static_fcalcs = get_static_fcalcs_with_HKL_lookup()
from LS49.work2_for_aca_lsq.remake_range_intensities_with_complex \
import get_intensity_structure
model_intensities = get_intensity_structure(
static_fcalcs,
FE1_model=Fe_oxidized_model,
FE2_model=Fe_reduced_model)
with (open(self.params.starting_model.filename, "wb")) as out:
pickle.dump(HKL_lookup, out, pickle.HIGHEST_PROTOCOL)
pickle.dump(static_fcalcs, out, pickle.HIGHEST_PROTOCOL)
pickle.dump(model_intensities, out,
pickle.HIGHEST_PROTOCOL)
return
else:
if self.mpi_helper.rank == 0:
with (open(self.params.starting_model.filename, "rb")) as inp:
print("the starting model (used for channel weighting) is",
self.params.starting_model.filename)
HKL_lookup = pickle.load(inp)
static_fcalcs = pickle.load(inp)
model_intensities = pickle.load(inp)
from LS49.spectra.generate_spectra import spectra_simulation
from LS49.sim.step5_pad import microcrystal
shuffA = list(range(N_total))
import random
random.shuffle(shuffA)
transmitted_info = dict(
HKL_lookup=HKL_lookup,
static_fcalcs=static_fcalcs,
model_intensities=model_intensities,
spectra_simulation=spectra_simulation(),
crystal=microcrystal(Deff_A=4000,
length_um=4.,
beam_diameter_um=1.0),
shuffA=shuffA)
else:
transmitted_info = None
print("before braodcast with ", self.mpi_helper.rank,
self.mpi_helper.size)
transmitted_info = self.mpi_helper.comm.bcast(transmitted_info, root=0)
self.mpi_helper.comm.barrier()
print("after barrier")
# -----------------------------------------------------------------------
if self.mpi_helper.rank == 0:
print("Finding initial G and abc factors")
per_rank_items = []
per_rank_keys = []
per_rank_G = []
min_spots = 3
N_input = 0
import os, omptbx # cori workaround, which does not get OMP_NUM_THREADS from environment
workaround_nt = int(os.environ.get("OMP_NUM_THREADS", 1))
omptbx.omp_set_num_threads(workaround_nt)
for item, key in get_items(logical_rank, N_total, N_stride,
transmitted_info["shuffA"],
self.params.cohort):
N_input += 1
if len(item) >= min_spots:
try:
FOI = fit_one_image_multispot(
key=key,
list_of_images=item,
HKL_lookup=transmitted_info["HKL_lookup"],
model_intensities=transmitted_info[
"model_intensities"],
spectra=transmitted_info["spectra_simulation"],
crystal=transmitted_info["crystal"])
except RuntimeError as e:
# no recovery from LBFGS error, skip event
continue
metric_P1, metric_C2 = FOI.DRM.get_current_angular_offsets(
FOI.x[-4:-1])
print(
"""LLG Image %06d on %d Bragg spots NLL channels F = %9.1f angular offsets in P1 and C2 (degrees): %8.5f %8.5f"""
%
(key, len(item), FOI.compute_functional_and_gradients()[0],
metric_P1, metric_C2), FOI.x[-4:-1])
# reporting out results to new abc_coverage pickles. Use the old one "item" as a template:
# item := [<LS49.work2_for_aca_lsq.abc_background.fit_roi_multichannel>,... one for each spot]
# each fit_roi has the following attributes and we modify them as follows:
# 'a', the abcG parameters of the original one-spot fit, unused
# 'asu_idx_C2_setting', unmodified
# 'image_no', same as key, unmodified
# 'n', number of parameters for one-spot fit, unused
# 'orig_idx_C2_setting', unmodified
# 'sb_data', original shoebox data [integers as floats], passed along unmodified
# 'simtbx_P1_miller', unmodified
# 'simtbx_intensity_7122', unmodified
# 'x', the abcG parameters of the original one-spot fit, unused
# modify these:
# 'bkgrd_a', the spot abc parameters output here, to be passed on to global data fit
for ispot in range(len(item)):
item[ispot].bkgrd_a = FOI.x[3 * ispot:3 * (ispot + 1)]
# 'channels', need to save the new data that was set during update_roi_model_pixels_with_current_rotation()
# but only for the Amat, not the derivatives.
item[ispot].channels = FOI.new_calc2_dict_last_round[
ispot]["channels"]
# 'roi', get new region of interest summation that was set during update_roi_model_pixels_with_current_rotation()
item[ispot].roi = FOI.roi_model_pixels[ispot]
# put the newly refined background model back into the item
per_rank_keys.append(key)
per_rank_G.append(FOI.a[-1])
print(
"pickling modified abc_coverage file for key %d in rank %d"
% (key, logical_rank), )
with open(abc_glob_pixel_ref % (key), "wb") as F:
pickle.dump(item, F, pickle.HIGHEST_PROTOCOL)
print("rank %d has %d refined images" %
(logical_rank, len(per_rank_keys)))
N_ranks = self.mpi_helper.comm.reduce(1, self.mpi_helper.MPI.SUM, 0)
N_refined_images = self.mpi_helper.comm.reduce(len(per_rank_keys),
self.mpi_helper.MPI.SUM,
0)
N_input_images = self.mpi_helper.comm.reduce(N_input,
self.mpi_helper.MPI.SUM,
0)
self.mpi_helper.comm.barrier()
if self.mpi_helper.rank == 0:
print("final report %d ranks, %d input images, %d refined models" %
(N_ranks, N_input_images, N_refined_images))
print("Finished finding initial G and abc factors")
self.mpi_helper.comm.barrier()