def make_detector(stype,
fast_axis,
slow_axis,
origin,
pixel_size,
image_size,
trusted_range=(0.0, 0.0),
px_mm=None,
name="Panel",
thickness=0.0,
material='',
mu=0.0,
gain=None,
identifier=""):
"""Ensure all types are correct before creating c++ detector class."""
if px_mm is None:
px_mm = SimplePxMmStrategy()
d = Detector()
p = d.add_panel()
p.set_type(str(stype))
p.set_name(str(name))
p.set_local_frame(tuple(map(float, fast_axis)),
tuple(map(float, slow_axis)),
tuple(map(float, origin)))
p.set_pixel_size(tuple(map(float, pixel_size)))
p.set_image_size(tuple(map(int, image_size)))
p.set_trusted_range(tuple(map(float, trusted_range)))
p.set_thickness(thickness)
p.set_material(material)
p.set_px_mm_strategy(px_mm)
p.set_identifier(identifier)
if gain is not None:
p.set_gain(gain)
return d
def imgCIF(cif_file, sensor):
'''Initialize a detector model from an imgCIF file.'''
cbf_handle = pycbf.cbf_handle_struct()
cbf_handle.read_file(cif_file, pycbf.MSG_DIGEST)
cbf_detector = cbf_handle.construct_detector(0)
pixel = (cbf_detector.get_inferred_pixel_size(1),
cbf_detector.get_inferred_pixel_size(2))
# FIXME can probably simplify the code which follows below by
# making proper use of cctbx vector calls - should not be as
# complex as it appears to be...
origin = tuple(cbf_detector.get_pixel_coordinates(0, 0))
fast = cbf_detector.get_pixel_coordinates(0, 1)
slow = cbf_detector.get_pixel_coordinates(1, 0)
dfast = [fast[j] - origin[j] for j in range(3)]
dslow = [slow[j] - origin[j] for j in range(3)]
lfast = math.sqrt(sum([dfast[j] * dfast[j] for j in range(3)]))
lslow = math.sqrt(sum([dslow[j] * dslow[j] for j in range(3)]))
fast = tuple([dfast[j] / lfast for j in range(3)])
slow = tuple([dslow[j] / lslow for j in range(3)])
size = tuple(reversed(cbf_handle.get_image_size(0)))
try:
underload = find_undefined_value(cbf_handle)
overload = cbf_handle.get_overload(0) * dxtbx_overload_scale
trusted_range = (underload, overload)
except: # intentional
trusted_range = (0.0, 0.0)
cbf_detector.__swig_destroy__(cbf_detector)
del (cbf_detector)
# Get the sensor type
dtype = detector_factory.sensor(sensor)
# If the sensor type is PAD then create the detector with a
# parallax corrected pixel to millimeter function
#if dtype == detector_helper_sensors.SENSOR_PAD:
#px_mm = ParallaxCorrectedPxMmStrategy(0.252500934883)
#else:
px_mm = SimplePxMmStrategy()
return detector_factory.make_detector(dtype, fast, slow, origin, pixel,
size, trusted_range, px_mm)
def job_runner(self, i_exp=0, spectra={}):
from simtbx.nanoBragg import utils
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
# Fixed hyperparameters
mosaic_spread_samples = 250
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt beam focal area
spot_scale = 500.
oversample = 1 # factor 1,2, or 3 probably enough
verbose = 0 # leave as 0, unless debug
shapetype = "gauss_argchk"
#<><><><><><><><>
os.environ[
"NXMX_LOCAL_DATA"] = "/global/cfs/cdirs/m3562/der/master_files/run_000795.JF07T32V01_master.h5"
expt = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)[0]
crystal = expt.crystal
detector = expt.detector
flat = True # enforce that the camera has 0 thickness
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
assert detector[0].get_thickness() == 0
alt_exper = ExperimentListFactory.from_json_file(
'/global/cfs/cdirs/m3562/der/braggnanimous/top8_newlam2/expers/rank0/stg1_top_0_0.expt',
check_format=False)[0]
AC = alt_crystal = alt_exper.crystal
beam = expt.beam
spec = expt.imageset.get_spectrum(0)
energies_raw = spec.get_energies_eV().as_numpy_array()
weights_raw = spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw,
weights_raw,
method=1,
total_flux=total_flux,
ev_width=ev_res)
device_Id = 0
if self.gpu_channels_singleton is not None:
device_Id = self.gpu_channels_singleton.get_deviceID()
mn_energy = (energies * weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
print(
"\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>"
)
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG = time()
print(self.gpu_channels_singleton.get_deviceID(), "device",
shapetype)
Famp_is_uninitialized = (
self.gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = self.amplitudes
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
self.gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert self.gpu_channels_singleton.get_nchannels() == 1
# Variable parameters
mosaic_spread = 0.00 # degrees
Ncells_abc = 130, 30, 10 # medians from best stage1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=alt_crystal,
DETECTOR=detector,
BEAM=beam,
Famp=self.gpu_channels_singleton,
energies=list(energies),
fluxes=list(weights),
background_wavelengths=[mn_wave],
background_wavelength_weights=[1],
background_total_flux=total_flux,
background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=mosaic_spread,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=False,
mask_file=mask_array)
TIME_EXA = time() - BEG
print(
"\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs"
% (TIME_BG, TIME_BRAGG, TIME_EXA))
print(
"<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n"
)
return JF16M_numpy_array
def tst_one_monkeypatch(i_exp, spectra, Fmerge, gpu_channels_singleton, rank,
params):
print("IN MONKEYPATCH")
from simtbx.nanoBragg import utils
from dxtbx.model.experiment_list import ExperimentListFactory
import numpy as np
print("Experiment %d" % i_exp, flush=True)
outfile = "boop_%d.hdf5" % i_exp
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
cuda = True # False # whether to use cuda
omp = False
ngpu_on_node = 1 # 8 # number of available GPUs
mosaic_spread = 0.07 # degrees
mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity
Ncells_abc = 30, 30, 10 # medians from best stage1
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt of beam focal area
spot_scale = 500. # 5.16324 # median from best stage1
plot_spec = False # plot the downsample spectra before simulating
oversample = 1 # oversample factor, 1,2, or 3 probable enough
panel_list = None # integer list of panels, usefule for debugging
rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU)
include_background = params.include_background # default is to add water background 100 mm thick
verbose = 0 # leave as 0, unles debug
flat = True # enfore that the camera has 0 thickness
#<><><><><><><><>
# XXX new code
El = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)
exper = El[0]
crystal = exper.crystal
detector = exper.detector
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
beam = exper.beam
# XXX new code
spec = exper.imageset.get_spectrum(0)
energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \
spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw,
weights_raw,
method=1,
total_flux=total_flux,
ev_width=ev_res)
if flat:
assert detector[0].get_thickness() == 0
if panel_list is None:
panel_list = list(range(len(detector)))
pids_for_rank = panel_list
device_Id = 0
if gpu_channels_singleton is not None:
device_Id = gpu_channels_singleton.get_deviceID()
print("Rank %d will use device %d" % (rank, device_Id))
show_params = False
time_panels = (rank == 0)
mn_energy = (energies * weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
print(
"\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG = time()
print(gpu_channels_singleton.get_deviceID(), "device", shapetype)
Famp_is_uninitialized = (gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = Fmerge.expand_to_p1()
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert gpu_channels_singleton.get_nchannels() == 1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal,
DETECTOR=detector,
BEAM=beam,
Famp=gpu_channels_singleton,
energies=list(energies),
fluxes=list(weights),
background_wavelengths=[mn_wave],
background_wavelength_weights=[1],
background_total_flux=total_flux,
background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=mosaic_spread,
mosaic_method="double_random",
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=show_params,
time_panels=time_panels,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
mask_file=params.mask_file,
context=params.context)
TIME_EXA = time() - BEG
print(
"\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs"
% (TIME_BG, TIME_BRAGG, TIME_EXA))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n")
if params.write_output:
if params.write_experimental_data:
data = exper.imageset.get_raw_data(0)
img_sh = JF16M_numpy_array.shape
assert img_sh == (256, 254, 254)
num_output_images = 1 + int(params.write_experimental_data)
print("Saving exascale output data of shape", img_sh)
beam_dict = beam.to_dict()
det_dict = detector.to_dict()
try:
beam_dict.pop("spectrum_energies")
beam_dict.pop("spectrum_weights")
except Exception:
pass
with utils.H5AttributeGeomWriter(
os.path.join(params.log.outdir, "exap_%d.hdf5" % i_exp),
image_shape=img_sh,
num_images=num_output_images,
detector=det_dict,
beam=beam_dict,
detector_and_beam_are_dicts=True) as writer:
writer.add_image(JF16M_numpy_array)
if params.write_experimental_data:
data = [data[pid].as_numpy_array() for pid in panel_list]
writer.add_image(data)
print("Saved output to file %s" % ("exap_%d.hdf5" % i_exp))
if not params.write_output:
# ability to read in the special file format
# note to end-user: The special file format can be installed permanently into a
# developmental version of dials/cctbx:
# dxtbx.install_format ./FormatHDF5AttributeGeometry.py --global # writes to build directory
# or alternatively to the user's account:
# dxtbx.install_format ./FormatHDF5AttributeGeometry.py --user # writes to ~/.dxtbx
from LS49.adse13_187.FormatHDF5AttributeGeometry import FormatHDF5AttributeGeometry as format_instance
from LS49 import ls49_big_data
filename = os.path.join(ls49_big_data, "adse13_228",
"exap_%d.hdf5" % i_exp)
instance = format_instance(filename)
reference = [D.as_numpy_array() for D in instance.get_raw_data()]
print("reference length for %s is %d" %
("exap_%d.hdf5" % i_exp, len(reference)))
# assertion on equality:
abs_diff = np.abs(JF16M_numpy_array - reference).max()
assert np.allclose(JF16M_numpy_array, reference), \
"max per-pixel difference: %f photons, experiment %d"%(abs_diff,i_exp)
def tst_one(i_exp,spectra,Fmerge,gpu_channels_singleton,rank,params):
from simtbx.nanoBragg import utils
from dxtbx.model.experiment_list import ExperimentListFactory
import numpy as np
print("Experiment %d" % i_exp, flush=True)
sys.stdout.flush()
outfile = "boop_%d.hdf5" % i_exp
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
# Not used # refl_file = "/global/cfs/cdirs/m3562/der/run795/top_%d.refl" % i_exp
cuda = True # False # whether to use cuda
omp = False
ngpu_on_node = 1 # 8 # number of available GPUs
mosaic_spread = 0.07 # degrees
mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity
Ncells_abc = 30, 30, 10 # medians from best stage1
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt of beam focal area
spot_scale = 500. # 5.16324 # median from best stage1
plot_spec = False # plot the downsample spectra before simulating
oversample = 1 # oversample factor, 1,2, or 3 probable enough
panel_list = None # integer list of panels, usefule for debugging
rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU)
include_background = params.include_background # default is to add water background 100 mm thick
verbose = 0 # leave as 0, unles debug
flat = True # enfore that the camera has 0 thickness
#<><><><><><><><>
# XXX new code
El = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)
exper = El[0]
crystal = exper.crystal
detector = exper.detector
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
beam = exper.beam
# XXX new code
spec = exper.imageset.get_spectrum(0)
energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \
spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw, weights_raw, method=1, total_flux=total_flux,
ev_width=ev_res)
if flat:
assert detector[0].get_thickness() == 0
if panel_list is None:
panel_list = list(range(len(detector)))
pids_for_rank = panel_list
device_Id = 0
if gpu_channels_singleton is not None:
device_Id = gpu_channels_singleton.get_deviceID()
print("Rank %d will use device %d" % (rank, device_Id))
show_params = False
time_panels = (rank == 0)
mn_energy = (energies*weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
if params.use_exascale_api:
BEG=time()
print (gpu_channels_singleton.get_deviceID(),"device")
Famp_is_uninitialized = ( gpu_channels_singleton.get_nchannels() == 0 ) # uninitialized
if Famp_is_uninitialized:
F_P1 = Fmerge.expand_to_p1()
for x in range(1): # in this scenario, amplitudes are independent of lambda
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert gpu_channels_singleton.get_nchannels() == 1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal, DETECTOR=detector, BEAM=beam,
Famp = gpu_channels_singleton,
energies=list(energies), fluxes=list(weights),
background_wavelengths=[mn_wave], background_wavelength_weights=[1],
background_total_flux=total_flux,background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample, Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples, mos_spread=mosaic_spread,
mosaic_method=params.mosaic_method,
beamsize_mm=beamsize_mm,show_params=show_params,
time_panels=time_panels, verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
mask_file=params.mask_file)
TIME_EXA = time()-BEG
print ("Exascale time",TIME_EXA)
if params.write_experimental_data:
data = exper.imageset.get_raw_data(0)
tsave = time()
img_sh = JF16M_numpy_array.shape
assert img_sh == (256,254,254)
num_output_images = 1 + int(params.write_experimental_data)
print("Saving exascale output data of shape", img_sh)
beam_dict = beam.to_dict()
det_dict = detector.to_dict()
try:
beam_dict.pop("spectrum_energies")
beam_dict.pop("spectrum_weights")
except Exception: pass
# XXX no longer have two separate files
if params.write_output:
with utils.H5AttributeGeomWriter("exap_%d.hdf5"%i_exp,
image_shape=img_sh, num_images=num_output_images,
detector=det_dict, beam=beam_dict,
detector_and_beam_are_dicts=True) as writer:
writer.add_image(JF16M_numpy_array)
if params.write_experimental_data:
data = [data[pid].as_numpy_array() for pid in panel_list]
writer.add_image(data)
tsave = time() - tsave
print("Saved output to file %s. Saving took %.4f sec" % ("exap_%d.hdf5"%i_exp, tsave, ))
BEG2 = time()
#optional background
TIME_BG2 = time()
backgrounds = {pid: None for pid in panel_list}
if include_background:
backgrounds = {pid: utils.sim_background( # default is for water
detector, beam, wavelengths=[mn_wave], wavelength_weights=[1],
total_flux=total_flux, Fbg_vs_stol=water,
pidx=pid, beam_size_mm=beamsize_mm, sample_thick_mm=0.5)
for pid in pids_for_rank}
TIME_BG2 = time()-TIME_BG2
TIME_BRAGG2 = time()
pid_and_pdata = utils.flexBeam_sim_colors(
CRYSTAL=crystal, DETECTOR=detector, BEAM=beam,
energies=list(energies), fluxes=list(weights), Famp=Fmerge,
pids=pids_for_rank, cuda=cuda, device_Id=device_Id,
oversample=oversample, Ncells_abc=Ncells_abc, verbose=verbose,
time_panels=time_panels, show_params=show_params, spot_scale_override=spot_scale,
mos_dom=mosaic_spread_samples, mos_spread=mosaic_spread, beamsize_mm=beamsize_mm,
background_raw_pixels=backgrounds, include_noise=False, rois_perpanel=None)
TIME_BRAGG2 = time()-TIME_BRAGG2
pid_and_pdata = sorted(pid_and_pdata, key=lambda x: x[0])
_, pdata = zip(*pid_and_pdata)
TIME_VINTAGE = time()-BEG2
print("\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
if params.use_exascale_api:
print("\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs" % (TIME_BG, TIME_BRAGG, TIME_EXA))
print("\t\tVintage: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs" % (TIME_BG2, TIME_BRAGG2, TIME_VINTAGE))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n")
if params.test_pixel_congruency and params.use_exascale_api:
abs_diff = np.abs(np.array(pdata) - JF16M_numpy_array).max()
assert np.allclose(pdata, JF16M_numpy_array), "max per-pixel difference: %f photons"%abs_diff
print("pixel congruency: OK!")
# pdata is a list of 256 2D numpy arrays, now.
if len(panel_list) != len(detector):
print("Cant save partial detector image, exiting..")
exit()
#from dxtbx.model import Detector
#new_det = Detector()
#for pid in panel_list:
# new_det.add_panel(detector[pid])
#detector = new_det
if params.write_experimental_data:
data = exper.imageset.get_raw_data(0)
tsave = time()
pdata = np.array(pdata) # now pdata is a numpy array of shape 256,254,254
img_sh = pdata.shape
num_output_images = 3 + int(params.write_experimental_data)
print("BOOPZ: Rank=%d ; i_exp=%d, RAM usage=%f" % (rank, i_exp,get_memory_usage()/1e6 ))
beam_dict = beam.to_dict()
det_dict = detector.to_dict()
try:
beam_dict.pop("spectrum_energies")
beam_dict.pop("spectrum_weights")
except Exception: pass
if params.write_output:
print("Saving output data of shape", img_sh)
with utils.H5AttributeGeomWriter(outfile, image_shape=img_sh, num_images=num_output_images,
detector=det_dict, beam=beam_dict,
detector_and_beam_are_dicts=True) as writer:
writer.add_image(JF16M_numpy_array/pdata)
writer.add_image(JF16M_numpy_array)
writer.add_image(pdata)
if params.write_experimental_data:
data = [data[pid].as_numpy_array() for pid in panel_list]
writer.add_image(data)
tsave = time() - tsave
print("Saved output to file %s. Saving took %.4f sec" % (outfile, tsave, ))
def overwrite_from_phil(params, detector, beam=None):
'''
Overwrite from phil parameters
'''
from cctbx.eltbx import attenuation_coefficient
# Override any panel parameters
for panel_params in params.detector.panel:
panel = detector[panel_params.id]
if panel_params.name is not None:
panel.set_name(panel_params.name)
if panel_params.type is not None:
panel.set_type(panel_params.type)
if panel_params.gain is not None:
panel.set_gain(panel_params.gain)
if panel_params.pixel_size is not None:
panel.set_pixel_size(panel_params.pixel_size)
if panel_params.image_size is not None:
panel.set_image_size(panel_params.image_size)
if panel_params.trusted_range is not None:
panel.set_trusted_range(panel_params.trusted_range)
if panel_params.thickness is not None:
panel.set_thickness(panel_params.thickness)
if panel_params.material is not None:
panel.set_material(panel_params.material)
if panel_params.parallax_correction is None:
if isinstance(panel.get_px_mm_strategy(),
ParallaxCorrectedPxMmStrategy):
panel_params.parallax_correction = True
else:
panel_params.parallax_correction = False
if panel_params.parallax_correction is True:
if beam is None:
raise RuntimeError("No beam for parallax correction")
table = attenuation_coefficient.get_table(panel.get_material())
mu = table.mu_at_angstrom(beam.get_wavelength()) / 10.0
t0 = panel.get_thickness()
panel.set_px_mm_strategy(ParallaxCorrectedPxMmStrategy(mu, t0))
else:
panel.set_px_mm_strategy(SimplePxMmStrategy())
axes = [
panel_params.fast_axis, panel_params.slow_axis,
panel_params.origin
]
if axes.count(None) != 3:
if panel_params.fast_axis is None:
panel_params.fast_axis = panel.get_local_fast_axis()
if panel_params.slow_axis is None:
panel_params.slow_axis = panel.get_local_slow_axis()
if panel_params.origin is None:
panel_params.origin = panel.get_local_origin()
panel.set_local_frame(panel_params.fast_axis,
panel_params.slow_axis,
panel_params.origin)
# Create the hierarchy
if params.detector.hierarchy is not None:
root = detector.hierarchy()
if params.detector.hierarchy.name is not None:
root.set_name(params.detector.hierarchy.name)
if (params.detector.hierarchy.fast_axis is not None
or params.detector.hierarchy.slow_axis is not None
or params.detector.hierarchy.origin is not None):
if params.detector.hierarchy.fast_axis is None:
params.detector.hierarchy.fast_axis = root.get_fast_axis()
if params.detector.hierarchy.slow_axis is None:
params.detector.hierarchy.slow_axis = root.get_slow_axis()
if params.detector.hierarchy.origin is None:
params.detector.hierarchy.origin = root.get_origin()
root.set_frame(params.detector.hierarchy.fast_axis,
params.detector.hierarchy.slow_axis,
params.detector.hierarchy.origin)
def get_group(node, index):
if len(index) == 0:
return node
return get_group(node[index[0]], index[1:])
for group_params in params.detector.hierarchy.group:
group = get_group(root, group_params.id)
if group_params.name is not None:
group.set_name(group_params.name)
if (group_params.fast_axis is not None
or group_params.slow_axis is not None
or group_params.origin is not None):
if group_params.fast_axis is None:
group_params.fast_axis = group.get_local_fast_axis()
if group_params.slow_axis is None:
group_params.slow_axis = group.get_local_slow_axis()
if group_params.origin is None:
group_params.origin = group.get_local_origin()
group.set_local_frame(group_params.fast_axis,
group_params.slow_axis,
group_params.origin)
if len(group_params.panel) != 0:
raise RuntimeError("Can't reassign panels in groups")
# Return the detector
return detector
def job_runner(self,i_exp=0,spectra={}):
from simtbx.nanoBragg import utils
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
# Fixed hyperparameters
mosaic_spread_samples = 500
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt beam focal area
spot_scale = 500.
oversample = 1 # factor 1,2, or 3 probably enough
include_background = False
verbose = 0 # leave as 0, unless debug
shapetype = "gauss_argchk"
#<><><><><><><><>
os.environ["NXMX_LOCAL_DATA"]="/global/cfs/cdirs/m3562/der/master_files/run_000795.JF07T32V01_master.h5"
expt = ExperimentListFactory.from_json_file(
experiment_file, check_format=True)[0]
crystal = expt.crystal
detector = expt.detector
flat = True # enforce that the camera has 0 thickness
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
assert detector[0].get_thickness() == 0
beam = expt.beam
spec = expt.imageset.get_spectrum(0)
energies_raw = spec.get_energies_eV().as_numpy_array()
weights_raw = spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw, weights_raw, method=1, total_flux=total_flux, ev_width=ev_res)
device_Id = 0
if self.gpu_channels_singleton is not None:
device_Id = self.gpu_channels_singleton.get_deviceID()
mn_energy = (energies*weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
print("\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG=time()
print (self.gpu_channels_singleton.get_deviceID(),"device",shapetype)
Famp_is_uninitialized = ( self.gpu_channels_singleton.get_nchannels() == 0 )
if Famp_is_uninitialized:
from iotbx.reflection_file_reader import any_reflection_file
from LS49 import ls49_big_data
merge_file = os.path.join(ls49_big_data,"adse13_228","cyto_init_merge.mtz")
self.merged_amplitudes = any_reflection_file(merge_file).as_miller_arrays()[0].as_amplitude_array()
F1 = self.merged_amplitudes.expand_to_p1()
F2 = self.amplitudes.expand_to_p1() # takes care of both transform to asu & expand
if False: # make sure that mtz file (F1) and strong spots (self.amplitudes) are roughly correlated
from matplotlib import pyplot as plt
from cctbx import miller
matches = miller.match_indices( F1.indices(), self.amplitudes.indices() )
sel0 = flex.size_t([p[0] for p in matches.pairs()])
sel1 = flex.size_t([p[1] for p in matches.pairs()])
data0 = F1.data().select(sel0)
data1 = self.amplitudes.data().select(sel1)
plt.plot(data0, data1, 'r.')
plt.show() # yes, the two are very roughly correlated
# end of test
#F_P1 = F1 # legacy, use a merged mtz file
#F_P1 = F2 # this one way absolutely wrong! way too many predictions, beyond the strong spots
F_P1 = F1
for x in range(1): # in this scenario, amplitudes are independent of lambda
self.gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert self.gpu_channels_singleton.get_nchannels() == 1
# Variable parameters
mosaic_spread = 0.07 # degrees
Ncells_abc = 30, 30, 10
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal, DETECTOR=detector, BEAM=beam,
Famp = self.gpu_channels_singleton,
energies=list(energies), fluxes=list(weights),
background_wavelengths=[mn_wave], background_wavelength_weights=[1],
background_total_flux=total_flux,background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample, Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples, mos_spread=mosaic_spread,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False, verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
mask_file=mask_array)
TIME_EXA = time()-BEG
print("\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs" % (TIME_BG, TIME_BRAGG, TIME_EXA))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n")
return JF16M_numpy_array
def tst_one_monkeypatch(i_exp, spectra, Fmerge, gpu_channels_singleton, rank,
params):
print("IN MONKEYPATCH")
from simtbx.nanoBragg import utils
from dxtbx.model.experiment_list import ExperimentListFactory
import numpy as np
print("Experiment %d" % i_exp, flush=True)
outfile = "boop_%d.hdf5" % i_exp
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
cuda = True # False # whether to use cuda
omp = False
ngpu_on_node = 1 # 8 # number of available GPUs
mosaic_spread = 0.07 # degrees
mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity
Ncells_abc = 30, 30, 10 # medians from best stage1
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt of beam focal area
spot_scale = 500. # 5.16324 # median from best stage1
plot_spec = False # plot the downsample spectra before simulating
oversample = 1 # oversample factor, 1,2, or 3 probable enough
panel_list = None # integer list of panels, usefule for debugging
rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU)
include_background = params.include_background # default is to add water background 100 mm thick
verbose = 0 # leave as 0, unles debug
flat = True # enfore that the camera has 0 thickness
#<><><><><><><><>
# XXX new code
El = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)
exper = El[0]
crystal = exper.crystal
detector = exper.detector
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
beam = exper.beam
# XXX new code
spec = exper.imageset.get_spectrum(0)
energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \
spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw,
weights_raw,
method=1,
total_flux=total_flux,
ev_width=ev_res)
if flat:
assert detector[0].get_thickness() == 0
if panel_list is None:
panel_list = list(range(len(detector)))
pids_for_rank = panel_list
device_Id = 0
if gpu_channels_singleton is not None:
device_Id = gpu_channels_singleton.get_deviceID()
print("Rank %d will use device %d" % (rank, device_Id))
show_params = False
time_panels = (rank == 0)
mn_energy = (energies * weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
jf16m_numpy_array = {}
from_gpu_amplitudes_cuda = {}
print(
"\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk", "gauss"]:
BEG = time()
print(gpu_channels_singleton.get_deviceID(), "device", shapetype)
Famp_is_uninitialized = (gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = Fmerge.expand_to_p1()
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert gpu_channels_singleton.get_nchannels() == 1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal,
DETECTOR=detector,
BEAM=beam,
Famp=gpu_channels_singleton,
energies=list(energies),
fluxes=list(weights),
background_wavelengths=[mn_wave],
background_wavelength_weights=[1],
background_total_flux=total_flux,
background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=mosaic_spread,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=show_params,
time_panels=time_panels,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
context=params.context)
TIME_EXA = time() - BEG
jf16m_numpy_array[shapetype] = JF16M_numpy_array
from_gpu_amplitudes_cuda[shapetype] = TIME_BRAGG
print(
"\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs"
% (TIME_BG, TIME_BRAGG, TIME_EXA))
ratio = from_gpu_amplitudes_cuda["gauss"] / from_gpu_amplitudes_cuda[
"gauss_argchk"]
print(
"<><><><><><><><><ratio<%.2f><><><><><><><><><><><><><><><><><><><>\n"
% (ratio))
# assertion on elapsed time:
assert ratio > 1.0, "ratio is %.3f, experiment %d" % (ratio, i_exp)
# assertion on equality:
abs_diff = np.abs(jf16m_numpy_array["gauss"] - \
jf16m_numpy_array["gauss_argchk"]).max()
assert np.allclose(jf16m_numpy_array["gauss"], \
jf16m_numpy_array["gauss_argchk"]), \
"max per-pixel difference: %f photons, experiment %d"%(abs_diff,i_exp)
def job_runner(self,expt,alt_expt,params,mask_array=None,i_exp=0,spectra={},mos_aniso=None):
# Fixed hyperparameters
mosaic_spread_samples = 250
beamsize_mm = 0.000886226925452758 # sqrt beam focal area
spot_scale = 500.
oversample = 1 # factor 1,2, or 3 probably enough
verbose = 0 # leave as 0, unless debug
shapetype = "gauss_argchk"
if mask_array is not None:
assert type(mask_array) is flex.bool # type check intending to convert active-pixel-bools to whitelist-ints
active_pixels = flex.int()
for i, x in enumerate(mask_array):
if x: active_pixels.append(i)
mask_array = active_pixels
detector = expt.detector
flat = True # enforce that the camera has 0 thickness
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
assert detector[0].get_thickness() == 0
alt_crystal = alt_expt.crystal
beam = expt.beam
spec = expt.imageset.get_spectrum(0)
energies_raw = spec.get_energies_eV().as_numpy_array()
weights_raw = spec.get_weights().as_numpy_array()
from LS49.adse13_187.adse13_221.explore_spectrum import method3
energies, weights, _ = method3(energies_raw, weights_raw,); weights = 5000000.*weights
energies = list(energies); weights = list(weights)
device_Id = 0
if self.gpu_channels_singleton is not None:
device_Id = self.gpu_channels_singleton.get_deviceID()
print("\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG=time()
print (self.gpu_channels_singleton.get_deviceID(),"device",shapetype)
Famp_is_uninitialized = ( self.gpu_channels_singleton.get_nchannels() == 0 )
if Famp_is_uninitialized:
F_P1 = self.amplitudes
for x in range(1): # in this scenario, amplitudes are independent of lambda
self.gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert self.gpu_channels_singleton.get_nchannels() == 1
# Variable parameters
mosaic_spread = params.mosaic_spread.value
Ncells_abc = params.Nabc.value
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=alt_crystal, DETECTOR=detector, BEAM=beam,
Famp = self.gpu_channels_singleton,
energies=energies, fluxes=weights,
cuda=True,
oversample=oversample, Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples, mos_spread=mosaic_spread,
mos_aniso = mos_aniso,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False, verbose=verbose,
spot_scale_override=spot_scale,
include_background=False,
mask_file=mask_array)
TIME_EXA = time()-BEG
print("\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs" % (TIME_BG, TIME_BRAGG, TIME_EXA))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n")
return JF16M_numpy_array
def chain_runner(self,
expt,
alt_expt,
params,
mask_array=None,
n_cycles=100,
s_cycles=0,
Zscore_callback=None,
rmsd_callback=None):
self.model_plot_enable = params.plot
if mask_array is not None:
assert type(
mask_array
) is flex.bool # type check intending to convert active-pixel-bools to whitelist-ints
active_pixels = flex.int()
for i, x in enumerate(mask_array):
if x: active_pixels.append(i)
mask_array = active_pixels
# Fixed hyperparameters
mosaic_spread_samples = 250
beamsize_mm = 0.000886226925452758 # sqrt beam focal area
spot_scale = 500.
oversample = 1 # factor 1,2, or 3 probably enough
verbose = 0 # leave as 0, unless debug
shapetype = "gauss_argchk"
detector = expt.detector
flat = True # enforce that the camera has 0 thickness
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
assert detector[0].get_thickness() == 0
alt_crystal = copy.deepcopy(
alt_expt.crystal) # avoid perturbing the original dials cell
beam = expt.beam
spec = expt.imageset.get_spectrum(0)
energies_raw = spec.get_energies_eV().as_numpy_array()
weights_raw = spec.get_weights().as_numpy_array()
from LS49.adse13_187.adse13_221.explore_spectrum import method3
energies, weights, _ = method3(
energies_raw,
weights_raw,
)
weights = 5000000. * weights
energies = list(energies)
weights = list(weights)
device_Id = 0 # XXX revisit for multiprocess service
assert self.gpu_channels_singleton is not None
device_Id = self.gpu_channels_singleton.get_deviceID()
print(device_Id, "device", shapetype)
Famp_is_uninitialized = (
self.gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = self.amplitudes
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
self.gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert self.gpu_channels_singleton.get_nchannels() == 1
# Variable parameters
mosaic_spread = params.mosaic_spread.value
Ncells_abc = params.Nabc.value
from LS49.adse13_187.adse13_221.parameters import variable_mosaicity
from LS49.adse13_187.adse13_221.parameters import covariant_cell, covariant_rot, covariant_ncells
self.parameters = {}
self.parameters["cell"] = covariant_cell.from_covariance(
alt_crystal, params.cell)
self.parameters["etaa"] = variable_mosaicity(
mosaic_spread, label="η a", params=params.mosaic_spread)
self.parameters["etab"] = variable_mosaicity(
mosaic_spread, label="η b", params=params.mosaic_spread)
self.parameters["etac"] = variable_mosaicity(
mosaic_spread, label="η c", params=params.mosaic_spread)
self.parameters2 = {}
if params.rot.refine:
self.parameters2["rot"] = covariant_rot(alt_crystal, params.rot)
if params.Nabc.refine:
self.parameters2["ncells"] = covariant_ncells(params.Nabc)
self.ref_params = {}
self.ref_params.update(self.parameters)
self.ref_params.update(self.parameters2)
# XXX TO DO list (Nick/Dan discuss)
# 1) change the variable mosaicity model to use updated aniso Derek model (Nick)
# 2) add rotx/roty/rotz. Covariant excursion values from Sauter 2014 paper: rotz 0.02° rotx 0.03° roty 0.03°
# 3) refine ncells a b c
self.rmsd_chain = flex.double()
self.sigz_chain = flex.double()
self.llg_chain = flex.double()
self.cycle_list = [key for key in self.ref_params]
self.accept = flex.int()
self.beginning_iteration = 0
if s_cycles > 0:
from LS49.adse13_187.adse13_221.simplex_method import simplex_detail
# initialize prior to simplex
whitelist_only, TIME_BG, TIME_BRAGG, self.exascale_mos_blocks = multipanel_sim(
CRYSTAL=alt_crystal,
DETECTOR=detector,
BEAM=beam,
Famp=self.gpu_channels_singleton,
energies=energies,
fluxes=weights,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=self.parameters["etaa"].proposal,
mos_aniso=(self.parameters["etaa"].proposal,
self.parameters["etab"].proposal,
self.parameters["etac"].proposal),
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=False,
mask_file=mask_array,
skip_numpy=True,
relevant_whitelist_order=self.relevant_whitelist_order)
Rmsd, sigZ, LLG = Zscore_callback(kernel_model=whitelist_only,
plot=False)
self.accept.append(1)
self.rmsd_chain.append(Rmsd)
self.sigz_chain.append(sigZ)
self.llg_chain.append(LLG)
PP = dict(detector=detector,
beam=beam,
energies=energies,
weights=weights,
oversample=oversample,
mosaic_spread_samples=mosaic_spread_samples,
beamsize_mm=beamsize_mm,
shapetype=shapetype,
verbose=verbose,
spot_scale=spot_scale,
mask_array=mask_array,
Z=Zscore_callback)
MIN = simplex_detail(alt_crystal,
Ncells_abc,
host_runner=self,
PP=PP,
n_cycles=n_cycles,
s_cycles=s_cycles)
self.beginning_iteration = MIN.iteration + 1
for macro_iteration in range(self.beginning_iteration, n_cycles):
BEG = time()
turn = self.cycle_list[macro_iteration % len(self.cycle_list)]
if turn == "cell":
alt_crystal = self.parameters[
"cell"].get_current_crystal_model(alt_crystal)
elif turn == "rot":
alt_crystal = self.parameters2[
"rot"].get_current_crystal_model(alt_crystal)
elif turn == "ncells":
Ncells_abc = self.parameters2["ncells"].get_current_model()
whitelist_only, TIME_BG, TIME_BRAGG, self.exascale_mos_blocks = multipanel_sim(
CRYSTAL=alt_crystal,
DETECTOR=detector,
BEAM=beam,
Famp=self.gpu_channels_singleton,
energies=energies,
fluxes=weights,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=self.parameters["etaa"].proposal,
mos_aniso=(self.parameters["etaa"].proposal,
self.parameters["etab"].proposal,
self.parameters["etac"].proposal),
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=False,
mask_file=mask_array,
skip_numpy=True,
relevant_whitelist_order=self.relevant_whitelist_order)
Rmsd, sigZ, LLG = Zscore_callback(kernel_model=whitelist_only,
plot=False)
if macro_iteration == self.beginning_iteration:
for key in self.ref_params:
self.ref_params[key].accept()
self.accept.append(1)
self.rmsd_chain.append(Rmsd)
self.sigz_chain.append(sigZ)
self.llg_chain.append(LLG)
else:
print("Old NLL ", self.llg_chain[-1], "NEW LLG", LLG, "diff",
self.llg_chain[-1] - LLG)
this_cycle_key = self.cycle_list[(macro_iteration) %
len(self.cycle_list)]
acceptance_prob = min(
1.,
math.exp((self.llg_chain[-1] - LLG) /
len(whitelist_only)) # normalize by no. of pixels
* self.ref_params[this_cycle_key].
transition_probability_ratio # q(X|Y)/q(Y|X), Y=proposal, X=last value
)
if random.random() < acceptance_prob:
for key in self.ref_params:
if key == turn: self.ref_params[key].accept()
else: self.ref_params[key].reject()
self.accept.append(1)
self.rmsd_chain.append(Rmsd)
self.sigz_chain.append(sigZ)
self.llg_chain.append(LLG)
else:
for key in self.ref_params:
self.ref_params[key].reject()
self.accept.append(0)
self.rmsd_chain.append(self.rmsd_chain[-1])
self.sigz_chain.append(self.sigz_chain[-1])
self.llg_chain.append(self.llg_chain[-1])
P = Profiler("%40s" % "key maintenance")
for key in self.ref_params:
if key == self.cycle_list[(macro_iteration + 1) %
len(self.cycle_list)]:
self.ref_params[key].generate_next_proposal()
P = Profiler("%40s" % "plot all")
self.plot_all(macro_iteration + 1, of=n_cycles)
del P
TIME_EXA = time() - BEG
print("\t\tExascale: time for Bragg sim: %.4fs; total: %.4fs\n" %
(TIME_BRAGG, TIME_EXA))
print("MCMC <RMSD> %.2f" %
(flex.mean(self.rmsd_chain[len(self.rmsd_chain) // 2:])))
print("MCMC <sigz> %.2f" %
(flex.mean(self.sigz_chain[len(self.sigz_chain) // 2:])))
print("MCMC <-LLG> %.2f" %
(flex.mean(self.llg_chain[len(self.llg_chain) // 2:])))
for key in self.ref_params:
self.ref_params[key].show()
if self.model_plot_enable:
plt.close(self.fig)
plt.close(self.fig2)
plt.close(self.fig3)
plt.ioff()
print(flush=True)