def job_runner(self, i_exp=0, spectra={}):
from simtbx.nanoBragg import utils
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
# Fixed hyperparameters
mosaic_spread_samples = 250
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt beam focal area
spot_scale = 500.
oversample = 1 # factor 1,2, or 3 probably enough
verbose = 0 # leave as 0, unless debug
shapetype = "gauss_argchk"
#<><><><><><><><>
os.environ[
"NXMX_LOCAL_DATA"] = "/global/cfs/cdirs/m3562/der/master_files/run_000795.JF07T32V01_master.h5"
expt = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)[0]
crystal = expt.crystal
detector = expt.detector
flat = True # enforce that the camera has 0 thickness
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
assert detector[0].get_thickness() == 0
alt_exper = ExperimentListFactory.from_json_file(
'/global/cfs/cdirs/m3562/der/braggnanimous/top8_newlam2/expers/rank0/stg1_top_0_0.expt',
check_format=False)[0]
AC = alt_crystal = alt_exper.crystal
beam = expt.beam
spec = expt.imageset.get_spectrum(0)
energies_raw = spec.get_energies_eV().as_numpy_array()
weights_raw = spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw,
weights_raw,
method=1,
total_flux=total_flux,
ev_width=ev_res)
device_Id = 0
if self.gpu_channels_singleton is not None:
device_Id = self.gpu_channels_singleton.get_deviceID()
mn_energy = (energies * weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
print(
"\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>"
)
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG = time()
print(self.gpu_channels_singleton.get_deviceID(), "device",
shapetype)
Famp_is_uninitialized = (
self.gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = self.amplitudes
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
self.gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert self.gpu_channels_singleton.get_nchannels() == 1
# Variable parameters
mosaic_spread = 0.00 # degrees
Ncells_abc = 130, 30, 10 # medians from best stage1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=alt_crystal,
DETECTOR=detector,
BEAM=beam,
Famp=self.gpu_channels_singleton,
energies=list(energies),
fluxes=list(weights),
background_wavelengths=[mn_wave],
background_wavelength_weights=[1],
background_total_flux=total_flux,
background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=mosaic_spread,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=False,
mask_file=mask_array)
TIME_EXA = time() - BEG
print(
"\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs"
% (TIME_BG, TIME_BRAGG, TIME_EXA))
print(
"<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n"
)
return JF16M_numpy_array
def tst_one_monkeypatch(i_exp, spectra, Fmerge, gpu_channels_singleton, rank,
params):
print("IN MONKEYPATCH")
from simtbx.nanoBragg import utils
from dxtbx.model.experiment_list import ExperimentListFactory
import numpy as np
print("Experiment %d" % i_exp, flush=True)
outfile = "boop_%d.hdf5" % i_exp
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
cuda = True # False # whether to use cuda
omp = False
ngpu_on_node = 1 # 8 # number of available GPUs
mosaic_spread = 0.07 # degrees
mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity
Ncells_abc = 30, 30, 10 # medians from best stage1
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt of beam focal area
spot_scale = 500. # 5.16324 # median from best stage1
plot_spec = False # plot the downsample spectra before simulating
oversample = 1 # oversample factor, 1,2, or 3 probable enough
panel_list = None # integer list of panels, usefule for debugging
rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU)
include_background = params.include_background # default is to add water background 100 mm thick
verbose = 0 # leave as 0, unles debug
flat = True # enfore that the camera has 0 thickness
#<><><><><><><><>
# XXX new code
El = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)
exper = El[0]
crystal = exper.crystal
detector = exper.detector
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
beam = exper.beam
# XXX new code
spec = exper.imageset.get_spectrum(0)
energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \
spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw,
weights_raw,
method=1,
total_flux=total_flux,
ev_width=ev_res)
if flat:
assert detector[0].get_thickness() == 0
if panel_list is None:
panel_list = list(range(len(detector)))
pids_for_rank = panel_list
device_Id = 0
if gpu_channels_singleton is not None:
device_Id = gpu_channels_singleton.get_deviceID()
print("Rank %d will use device %d" % (rank, device_Id))
show_params = False
time_panels = (rank == 0)
mn_energy = (energies * weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
print(
"\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG = time()
print(gpu_channels_singleton.get_deviceID(), "device", shapetype)
Famp_is_uninitialized = (gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = Fmerge.expand_to_p1()
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert gpu_channels_singleton.get_nchannels() == 1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal,
DETECTOR=detector,
BEAM=beam,
Famp=gpu_channels_singleton,
energies=list(energies),
fluxes=list(weights),
background_wavelengths=[mn_wave],
background_wavelength_weights=[1],
background_total_flux=total_flux,
background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=mosaic_spread,
mosaic_method="double_random",
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=show_params,
time_panels=time_panels,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
mask_file=params.mask_file,
context=params.context)
TIME_EXA = time() - BEG
print(
"\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs"
% (TIME_BG, TIME_BRAGG, TIME_EXA))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n")
if params.write_output:
if params.write_experimental_data:
data = exper.imageset.get_raw_data(0)
img_sh = JF16M_numpy_array.shape
assert img_sh == (256, 254, 254)
num_output_images = 1 + int(params.write_experimental_data)
print("Saving exascale output data of shape", img_sh)
beam_dict = beam.to_dict()
det_dict = detector.to_dict()
try:
beam_dict.pop("spectrum_energies")
beam_dict.pop("spectrum_weights")
except Exception:
pass
with utils.H5AttributeGeomWriter(
os.path.join(params.log.outdir, "exap_%d.hdf5" % i_exp),
image_shape=img_sh,
num_images=num_output_images,
detector=det_dict,
beam=beam_dict,
detector_and_beam_are_dicts=True) as writer:
writer.add_image(JF16M_numpy_array)
if params.write_experimental_data:
data = [data[pid].as_numpy_array() for pid in panel_list]
writer.add_image(data)
print("Saved output to file %s" % ("exap_%d.hdf5" % i_exp))
if not params.write_output:
# ability to read in the special file format
# note to end-user: The special file format can be installed permanently into a
# developmental version of dials/cctbx:
# dxtbx.install_format ./FormatHDF5AttributeGeometry.py --global # writes to build directory
# or alternatively to the user's account:
# dxtbx.install_format ./FormatHDF5AttributeGeometry.py --user # writes to ~/.dxtbx
from LS49.adse13_187.FormatHDF5AttributeGeometry import FormatHDF5AttributeGeometry as format_instance
from LS49 import ls49_big_data
filename = os.path.join(ls49_big_data, "adse13_228",
"exap_%d.hdf5" % i_exp)
instance = format_instance(filename)
reference = [D.as_numpy_array() for D in instance.get_raw_data()]
print("reference length for %s is %d" %
("exap_%d.hdf5" % i_exp, len(reference)))
# assertion on equality:
abs_diff = np.abs(JF16M_numpy_array - reference).max()
assert np.allclose(JF16M_numpy_array, reference), \
"max per-pixel difference: %f photons, experiment %d"%(abs_diff,i_exp)
def job_runner(self,i_exp=0,spectra={}):
from simtbx.nanoBragg import utils
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
# Fixed hyperparameters
mosaic_spread_samples = 500
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt beam focal area
spot_scale = 500.
oversample = 1 # factor 1,2, or 3 probably enough
include_background = False
verbose = 0 # leave as 0, unless debug
shapetype = "gauss_argchk"
#<><><><><><><><>
os.environ["NXMX_LOCAL_DATA"]="/global/cfs/cdirs/m3562/der/master_files/run_000795.JF07T32V01_master.h5"
expt = ExperimentListFactory.from_json_file(
experiment_file, check_format=True)[0]
crystal = expt.crystal
detector = expt.detector
flat = True # enforce that the camera has 0 thickness
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
assert detector[0].get_thickness() == 0
beam = expt.beam
spec = expt.imageset.get_spectrum(0)
energies_raw = spec.get_energies_eV().as_numpy_array()
weights_raw = spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw, weights_raw, method=1, total_flux=total_flux, ev_width=ev_res)
device_Id = 0
if self.gpu_channels_singleton is not None:
device_Id = self.gpu_channels_singleton.get_deviceID()
mn_energy = (energies*weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
print("\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG=time()
print (self.gpu_channels_singleton.get_deviceID(),"device",shapetype)
Famp_is_uninitialized = ( self.gpu_channels_singleton.get_nchannels() == 0 )
if Famp_is_uninitialized:
from iotbx.reflection_file_reader import any_reflection_file
from LS49 import ls49_big_data
merge_file = os.path.join(ls49_big_data,"adse13_228","cyto_init_merge.mtz")
self.merged_amplitudes = any_reflection_file(merge_file).as_miller_arrays()[0].as_amplitude_array()
F1 = self.merged_amplitudes.expand_to_p1()
F2 = self.amplitudes.expand_to_p1() # takes care of both transform to asu & expand
if False: # make sure that mtz file (F1) and strong spots (self.amplitudes) are roughly correlated
from matplotlib import pyplot as plt
from cctbx import miller
matches = miller.match_indices( F1.indices(), self.amplitudes.indices() )
sel0 = flex.size_t([p[0] for p in matches.pairs()])
sel1 = flex.size_t([p[1] for p in matches.pairs()])
data0 = F1.data().select(sel0)
data1 = self.amplitudes.data().select(sel1)
plt.plot(data0, data1, 'r.')
plt.show() # yes, the two are very roughly correlated
# end of test
#F_P1 = F1 # legacy, use a merged mtz file
#F_P1 = F2 # this one way absolutely wrong! way too many predictions, beyond the strong spots
F_P1 = F1
for x in range(1): # in this scenario, amplitudes are independent of lambda
self.gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert self.gpu_channels_singleton.get_nchannels() == 1
# Variable parameters
mosaic_spread = 0.07 # degrees
Ncells_abc = 30, 30, 10
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal, DETECTOR=detector, BEAM=beam,
Famp = self.gpu_channels_singleton,
energies=list(energies), fluxes=list(weights),
background_wavelengths=[mn_wave], background_wavelength_weights=[1],
background_total_flux=total_flux,background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample, Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples, mos_spread=mosaic_spread,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False, verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
mask_file=mask_array)
TIME_EXA = time()-BEG
print("\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs" % (TIME_BG, TIME_BRAGG, TIME_EXA))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n")
return JF16M_numpy_array
def tst_one(i_exp,spectra,Fmerge,gpu_channels_singleton,rank,params):
from simtbx.nanoBragg import utils
from dxtbx.model.experiment_list import ExperimentListFactory
import numpy as np
print("Experiment %d" % i_exp, flush=True)
sys.stdout.flush()
outfile = "boop_%d.hdf5" % i_exp
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
# Not used # refl_file = "/global/cfs/cdirs/m3562/der/run795/top_%d.refl" % i_exp
cuda = True # False # whether to use cuda
omp = False
ngpu_on_node = 1 # 8 # number of available GPUs
mosaic_spread = 0.07 # degrees
mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity
Ncells_abc = 30, 30, 10 # medians from best stage1
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt of beam focal area
spot_scale = 500. # 5.16324 # median from best stage1
plot_spec = False # plot the downsample spectra before simulating
oversample = 1 # oversample factor, 1,2, or 3 probable enough
panel_list = None # integer list of panels, usefule for debugging
rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU)
include_background = params.include_background # default is to add water background 100 mm thick
verbose = 0 # leave as 0, unles debug
flat = True # enfore that the camera has 0 thickness
#<><><><><><><><>
# XXX new code
El = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)
exper = El[0]
crystal = exper.crystal
detector = exper.detector
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
beam = exper.beam
# XXX new code
spec = exper.imageset.get_spectrum(0)
energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \
spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw, weights_raw, method=1, total_flux=total_flux,
ev_width=ev_res)
if flat:
assert detector[0].get_thickness() == 0
if panel_list is None:
panel_list = list(range(len(detector)))
pids_for_rank = panel_list
device_Id = 0
if gpu_channels_singleton is not None:
device_Id = gpu_channels_singleton.get_deviceID()
print("Rank %d will use device %d" % (rank, device_Id))
show_params = False
time_panels = (rank == 0)
mn_energy = (energies*weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
if params.use_exascale_api:
BEG=time()
print (gpu_channels_singleton.get_deviceID(),"device")
Famp_is_uninitialized = ( gpu_channels_singleton.get_nchannels() == 0 ) # uninitialized
if Famp_is_uninitialized:
F_P1 = Fmerge.expand_to_p1()
for x in range(1): # in this scenario, amplitudes are independent of lambda
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert gpu_channels_singleton.get_nchannels() == 1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal, DETECTOR=detector, BEAM=beam,
Famp = gpu_channels_singleton,
energies=list(energies), fluxes=list(weights),
background_wavelengths=[mn_wave], background_wavelength_weights=[1],
background_total_flux=total_flux,background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample, Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples, mos_spread=mosaic_spread,
mosaic_method=params.mosaic_method,
beamsize_mm=beamsize_mm,show_params=show_params,
time_panels=time_panels, verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
mask_file=params.mask_file)
TIME_EXA = time()-BEG
print ("Exascale time",TIME_EXA)
if params.write_experimental_data:
data = exper.imageset.get_raw_data(0)
tsave = time()
img_sh = JF16M_numpy_array.shape
assert img_sh == (256,254,254)
num_output_images = 1 + int(params.write_experimental_data)
print("Saving exascale output data of shape", img_sh)
beam_dict = beam.to_dict()
det_dict = detector.to_dict()
try:
beam_dict.pop("spectrum_energies")
beam_dict.pop("spectrum_weights")
except Exception: pass
# XXX no longer have two separate files
if params.write_output:
with utils.H5AttributeGeomWriter("exap_%d.hdf5"%i_exp,
image_shape=img_sh, num_images=num_output_images,
detector=det_dict, beam=beam_dict,
detector_and_beam_are_dicts=True) as writer:
writer.add_image(JF16M_numpy_array)
if params.write_experimental_data:
data = [data[pid].as_numpy_array() for pid in panel_list]
writer.add_image(data)
tsave = time() - tsave
print("Saved output to file %s. Saving took %.4f sec" % ("exap_%d.hdf5"%i_exp, tsave, ))
BEG2 = time()
#optional background
TIME_BG2 = time()
backgrounds = {pid: None for pid in panel_list}
if include_background:
backgrounds = {pid: utils.sim_background( # default is for water
detector, beam, wavelengths=[mn_wave], wavelength_weights=[1],
total_flux=total_flux, Fbg_vs_stol=water,
pidx=pid, beam_size_mm=beamsize_mm, sample_thick_mm=0.5)
for pid in pids_for_rank}
TIME_BG2 = time()-TIME_BG2
TIME_BRAGG2 = time()
pid_and_pdata = utils.flexBeam_sim_colors(
CRYSTAL=crystal, DETECTOR=detector, BEAM=beam,
energies=list(energies), fluxes=list(weights), Famp=Fmerge,
pids=pids_for_rank, cuda=cuda, device_Id=device_Id,
oversample=oversample, Ncells_abc=Ncells_abc, verbose=verbose,
time_panels=time_panels, show_params=show_params, spot_scale_override=spot_scale,
mos_dom=mosaic_spread_samples, mos_spread=mosaic_spread, beamsize_mm=beamsize_mm,
background_raw_pixels=backgrounds, include_noise=False, rois_perpanel=None)
TIME_BRAGG2 = time()-TIME_BRAGG2
pid_and_pdata = sorted(pid_and_pdata, key=lambda x: x[0])
_, pdata = zip(*pid_and_pdata)
TIME_VINTAGE = time()-BEG2
print("\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
if params.use_exascale_api:
print("\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs" % (TIME_BG, TIME_BRAGG, TIME_EXA))
print("\t\tVintage: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs" % (TIME_BG2, TIME_BRAGG2, TIME_VINTAGE))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n")
if params.test_pixel_congruency and params.use_exascale_api:
abs_diff = np.abs(np.array(pdata) - JF16M_numpy_array).max()
assert np.allclose(pdata, JF16M_numpy_array), "max per-pixel difference: %f photons"%abs_diff
print("pixel congruency: OK!")
# pdata is a list of 256 2D numpy arrays, now.
if len(panel_list) != len(detector):
print("Cant save partial detector image, exiting..")
exit()
#from dxtbx.model import Detector
#new_det = Detector()
#for pid in panel_list:
# new_det.add_panel(detector[pid])
#detector = new_det
if params.write_experimental_data:
data = exper.imageset.get_raw_data(0)
tsave = time()
pdata = np.array(pdata) # now pdata is a numpy array of shape 256,254,254
img_sh = pdata.shape
num_output_images = 3 + int(params.write_experimental_data)
print("BOOPZ: Rank=%d ; i_exp=%d, RAM usage=%f" % (rank, i_exp,get_memory_usage()/1e6 ))
beam_dict = beam.to_dict()
det_dict = detector.to_dict()
try:
beam_dict.pop("spectrum_energies")
beam_dict.pop("spectrum_weights")
except Exception: pass
if params.write_output:
print("Saving output data of shape", img_sh)
with utils.H5AttributeGeomWriter(outfile, image_shape=img_sh, num_images=num_output_images,
detector=det_dict, beam=beam_dict,
detector_and_beam_are_dicts=True) as writer:
writer.add_image(JF16M_numpy_array/pdata)
writer.add_image(JF16M_numpy_array)
writer.add_image(pdata)
if params.write_experimental_data:
data = [data[pid].as_numpy_array() for pid in panel_list]
writer.add_image(data)
tsave = time() - tsave
print("Saved output to file %s. Saving took %.4f sec" % (outfile, tsave, ))
def tst_one_monkeypatch(i_exp, spectra, Fmerge, gpu_channels_singleton, rank,
params):
print("IN MONKEYPATCH")
from simtbx.nanoBragg import utils
from dxtbx.model.experiment_list import ExperimentListFactory
import numpy as np
print("Experiment %d" % i_exp, flush=True)
outfile = "boop_%d.hdf5" % i_exp
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
cuda = True # False # whether to use cuda
omp = False
ngpu_on_node = 1 # 8 # number of available GPUs
mosaic_spread = 0.07 # degrees
mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity
Ncells_abc = 30, 30, 10 # medians from best stage1
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt of beam focal area
spot_scale = 500. # 5.16324 # median from best stage1
plot_spec = False # plot the downsample spectra before simulating
oversample = 1 # oversample factor, 1,2, or 3 probable enough
panel_list = None # integer list of panels, usefule for debugging
rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU)
include_background = params.include_background # default is to add water background 100 mm thick
verbose = 0 # leave as 0, unles debug
flat = True # enfore that the camera has 0 thickness
#<><><><><><><><>
# XXX new code
El = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)
exper = El[0]
crystal = exper.crystal
detector = exper.detector
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
beam = exper.beam
# XXX new code
spec = exper.imageset.get_spectrum(0)
energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \
spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw,
weights_raw,
method=1,
total_flux=total_flux,
ev_width=ev_res)
if flat:
assert detector[0].get_thickness() == 0
if panel_list is None:
panel_list = list(range(len(detector)))
pids_for_rank = panel_list
device_Id = 0
if gpu_channels_singleton is not None:
device_Id = gpu_channels_singleton.get_deviceID()
print("Rank %d will use device %d" % (rank, device_Id))
show_params = False
time_panels = (rank == 0)
mn_energy = (energies * weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
jf16m_numpy_array = {}
from_gpu_amplitudes_cuda = {}
print(
"\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk", "gauss"]:
BEG = time()
print(gpu_channels_singleton.get_deviceID(), "device", shapetype)
Famp_is_uninitialized = (gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = Fmerge.expand_to_p1()
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert gpu_channels_singleton.get_nchannels() == 1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal,
DETECTOR=detector,
BEAM=beam,
Famp=gpu_channels_singleton,
energies=list(energies),
fluxes=list(weights),
background_wavelengths=[mn_wave],
background_wavelength_weights=[1],
background_total_flux=total_flux,
background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=mosaic_spread,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=show_params,
time_panels=time_panels,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
context=params.context)
TIME_EXA = time() - BEG
jf16m_numpy_array[shapetype] = JF16M_numpy_array
from_gpu_amplitudes_cuda[shapetype] = TIME_BRAGG
print(
"\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs"
% (TIME_BG, TIME_BRAGG, TIME_EXA))
ratio = from_gpu_amplitudes_cuda["gauss"] / from_gpu_amplitudes_cuda[
"gauss_argchk"]
print(
"<><><><><><><><><ratio<%.2f><><><><><><><><><><><><><><><><><><><>\n"
% (ratio))
# assertion on elapsed time:
assert ratio > 1.0, "ratio is %.3f, experiment %d" % (ratio, i_exp)
# assertion on equality:
abs_diff = np.abs(jf16m_numpy_array["gauss"] - \
jf16m_numpy_array["gauss_argchk"]).max()
assert np.allclose(jf16m_numpy_array["gauss"], \
jf16m_numpy_array["gauss_argchk"]), \
"max per-pixel difference: %f photons, experiment %d"%(abs_diff,i_exp)