def target(selfOO, vector):
selfOO.iteration += 1
ptr = 0
for key in selfOO.crnm.ref_params:
n_values = len(selfOO.crnm.ref_params[key].display_labels)
these_values = vector[ptr:ptr + n_values]
selfOO.crnm.ref_params[key].set_proposal_from_simplex(these_values)
ptr += n_values
BEG = time()
if "cell" in selfOO.crnm.ref_params:
selfOO.alt_crystal = selfOO.crnm.ref_params[
"cell"].get_current_crystal_model(selfOO.alt_crystal)
if "rot" in selfOO.crnm.ref_params:
selfOO.alt_crystal = selfOO.crnm.ref_params[
"rot"].get_current_crystal_model(selfOO.alt_crystal)
if "ncells" in selfOO.crnm.ref_params:
selfOO.Ncells_abc = selfOO.crnm.ref_params[
"ncells"].get_current_model()
whitelist_only, TIME_BG, TIME_BRAGG, selfOO.crnm.exascale_mos_blocks = multipanel_sim(
CRYSTAL=selfOO.alt_crystal,
DETECTOR=selfOO.PP["detector"],
BEAM=selfOO.PP["beam"],
Famp=selfOO.crnm.gpu_channels_singleton,
energies=selfOO.PP["energies"],
fluxes=selfOO.PP["weights"],
cuda=True,
oversample=selfOO.PP["oversample"],
Ncells_abc=selfOO.Ncells_abc,
mos_dom=selfOO.PP["mosaic_spread_samples"],
mos_spread=selfOO.crnm.parameters["etaa"].proposal,
mos_aniso=(selfOO.crnm.parameters["etaa"].proposal,
selfOO.crnm.parameters["etab"].proposal,
selfOO.crnm.parameters["etac"].proposal),
beamsize_mm=selfOO.PP["beamsize_mm"],
profile=selfOO.PP["shapetype"],
show_params=False,
time_panels=False,
verbose=selfOO.PP["verbose"],
spot_scale_override=selfOO.PP["spot_scale"],
include_background=False,
mask_file=selfOO.PP["mask_array"],
skip_numpy=True,
relevant_whitelist_order=selfOO.crnm.relevant_whitelist_order)
Rmsd, sigZ, LLG = selfOO.PP["Z"](kernel_model=whitelist_only,
plot=False)
#print ("Old NLL ",selfOO.crnm.llg_chain[-1], "NEW LLG",LLG, "diff",selfOO.crnm.llg_chain[-1] - LLG)
for key in selfOO.crnm.ref_params:
selfOO.crnm.ref_params[key].accept()
selfOO.crnm.accept.append(1)
selfOO.crnm.rmsd_chain.append(Rmsd)
selfOO.crnm.sigz_chain.append(sigZ)
selfOO.crnm.llg_chain.append(LLG)
selfOO.crnm.plot_all(selfOO.iteration + 1, of=selfOO.n_cycles)
TIME_EXA = time() - BEG
#print("\t\tExascale: time for Bragg sim: %.4fs; total: %.4fs" % (TIME_BRAGG, TIME_EXA))
return LLG
def job_runner(self, i_exp=0, spectra={}):
from simtbx.nanoBragg import utils
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
# Fixed hyperparameters
mosaic_spread_samples = 250
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt beam focal area
spot_scale = 500.
oversample = 1 # factor 1,2, or 3 probably enough
verbose = 0 # leave as 0, unless debug
shapetype = "gauss_argchk"
#<><><><><><><><>
os.environ[
"NXMX_LOCAL_DATA"] = "/global/cfs/cdirs/m3562/der/master_files/run_000795.JF07T32V01_master.h5"
expt = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)[0]
crystal = expt.crystal
detector = expt.detector
flat = True # enforce that the camera has 0 thickness
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
assert detector[0].get_thickness() == 0
alt_exper = ExperimentListFactory.from_json_file(
'/global/cfs/cdirs/m3562/der/braggnanimous/top8_newlam2/expers/rank0/stg1_top_0_0.expt',
check_format=False)[0]
AC = alt_crystal = alt_exper.crystal
beam = expt.beam
spec = expt.imageset.get_spectrum(0)
energies_raw = spec.get_energies_eV().as_numpy_array()
weights_raw = spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw,
weights_raw,
method=1,
total_flux=total_flux,
ev_width=ev_res)
device_Id = 0
if self.gpu_channels_singleton is not None:
device_Id = self.gpu_channels_singleton.get_deviceID()
mn_energy = (energies * weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
print(
"\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>"
)
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG = time()
print(self.gpu_channels_singleton.get_deviceID(), "device",
shapetype)
Famp_is_uninitialized = (
self.gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = self.amplitudes
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
self.gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert self.gpu_channels_singleton.get_nchannels() == 1
# Variable parameters
mosaic_spread = 0.00 # degrees
Ncells_abc = 130, 30, 10 # medians from best stage1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=alt_crystal,
DETECTOR=detector,
BEAM=beam,
Famp=self.gpu_channels_singleton,
energies=list(energies),
fluxes=list(weights),
background_wavelengths=[mn_wave],
background_wavelength_weights=[1],
background_total_flux=total_flux,
background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=mosaic_spread,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=False,
mask_file=mask_array)
TIME_EXA = time() - BEG
print(
"\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs"
% (TIME_BG, TIME_BRAGG, TIME_EXA))
print(
"<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n"
)
return JF16M_numpy_array
def tst_one_monkeypatch(i_exp, spectra, Fmerge, gpu_channels_singleton, rank,
params):
print("IN MONKEYPATCH")
from simtbx.nanoBragg import utils
from dxtbx.model.experiment_list import ExperimentListFactory
import numpy as np
print("Experiment %d" % i_exp, flush=True)
outfile = "boop_%d.hdf5" % i_exp
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
cuda = True # False # whether to use cuda
omp = False
ngpu_on_node = 1 # 8 # number of available GPUs
mosaic_spread = 0.07 # degrees
mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity
Ncells_abc = 30, 30, 10 # medians from best stage1
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt of beam focal area
spot_scale = 500. # 5.16324 # median from best stage1
plot_spec = False # plot the downsample spectra before simulating
oversample = 1 # oversample factor, 1,2, or 3 probable enough
panel_list = None # integer list of panels, usefule for debugging
rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU)
include_background = params.include_background # default is to add water background 100 mm thick
verbose = 0 # leave as 0, unles debug
flat = True # enfore that the camera has 0 thickness
#<><><><><><><><>
# XXX new code
El = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)
exper = El[0]
crystal = exper.crystal
detector = exper.detector
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
beam = exper.beam
# XXX new code
spec = exper.imageset.get_spectrum(0)
energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \
spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw,
weights_raw,
method=1,
total_flux=total_flux,
ev_width=ev_res)
if flat:
assert detector[0].get_thickness() == 0
if panel_list is None:
panel_list = list(range(len(detector)))
pids_for_rank = panel_list
device_Id = 0
if gpu_channels_singleton is not None:
device_Id = gpu_channels_singleton.get_deviceID()
print("Rank %d will use device %d" % (rank, device_Id))
show_params = False
time_panels = (rank == 0)
mn_energy = (energies * weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
print(
"\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG = time()
print(gpu_channels_singleton.get_deviceID(), "device", shapetype)
Famp_is_uninitialized = (gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = Fmerge.expand_to_p1()
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert gpu_channels_singleton.get_nchannels() == 1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal,
DETECTOR=detector,
BEAM=beam,
Famp=gpu_channels_singleton,
energies=list(energies),
fluxes=list(weights),
background_wavelengths=[mn_wave],
background_wavelength_weights=[1],
background_total_flux=total_flux,
background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=mosaic_spread,
mosaic_method="double_random",
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=show_params,
time_panels=time_panels,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
mask_file=params.mask_file,
context=params.context)
TIME_EXA = time() - BEG
print(
"\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs"
% (TIME_BG, TIME_BRAGG, TIME_EXA))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n")
if params.write_output:
if params.write_experimental_data:
data = exper.imageset.get_raw_data(0)
img_sh = JF16M_numpy_array.shape
assert img_sh == (256, 254, 254)
num_output_images = 1 + int(params.write_experimental_data)
print("Saving exascale output data of shape", img_sh)
beam_dict = beam.to_dict()
det_dict = detector.to_dict()
try:
beam_dict.pop("spectrum_energies")
beam_dict.pop("spectrum_weights")
except Exception:
pass
with utils.H5AttributeGeomWriter(
os.path.join(params.log.outdir, "exap_%d.hdf5" % i_exp),
image_shape=img_sh,
num_images=num_output_images,
detector=det_dict,
beam=beam_dict,
detector_and_beam_are_dicts=True) as writer:
writer.add_image(JF16M_numpy_array)
if params.write_experimental_data:
data = [data[pid].as_numpy_array() for pid in panel_list]
writer.add_image(data)
print("Saved output to file %s" % ("exap_%d.hdf5" % i_exp))
if not params.write_output:
# ability to read in the special file format
# note to end-user: The special file format can be installed permanently into a
# developmental version of dials/cctbx:
# dxtbx.install_format ./FormatHDF5AttributeGeometry.py --global # writes to build directory
# or alternatively to the user's account:
# dxtbx.install_format ./FormatHDF5AttributeGeometry.py --user # writes to ~/.dxtbx
from LS49.adse13_187.FormatHDF5AttributeGeometry import FormatHDF5AttributeGeometry as format_instance
from LS49 import ls49_big_data
filename = os.path.join(ls49_big_data, "adse13_228",
"exap_%d.hdf5" % i_exp)
instance = format_instance(filename)
reference = [D.as_numpy_array() for D in instance.get_raw_data()]
print("reference length for %s is %d" %
("exap_%d.hdf5" % i_exp, len(reference)))
# assertion on equality:
abs_diff = np.abs(JF16M_numpy_array - reference).max()
assert np.allclose(JF16M_numpy_array, reference), \
"max per-pixel difference: %f photons, experiment %d"%(abs_diff,i_exp)
def model_spots_from_pandas(pandas_frame,
rois_per_panel=None,
mtz_file=None,
mtz_col=None,
oversample_override=None,
Ncells_abc_override=None,
pink_stride_override=None,
spectrum_override=None,
cuda=False,
device_Id=0,
time_panels=False,
d_max=999,
d_min=1.5,
defaultF=1e3,
omp=False,
norm_by_spectrum=False,
symbol_override=None,
quiet=False,
reset_Bmatrix=False,
nopolar=False,
force_no_detector_thickness=False,
printout_pix=None,
norm_by_nsource=False,
use_exascale_api=False,
use_db=False):
if use_exascale_api:
assert gpu_energy_channels is not None, "cant use exascale api if not in a GPU build"
assert multipanel_sim is not None, "cant use exascale api if LS49: https://github.com/nksauter/LS49.git is not configured\n install in the modules folder"
df = pandas_frame
if not quiet: LOGGER.info("Loading experiment models")
expt_name = df.opt_exp_name.values[0]
El = ExperimentListFactory.from_json_file(expt_name, check_format=False)
expt = El[0]
columns = list(df)
if "detz_shift_mm" in columns: # NOTE, this could also be inside expt_name directly
expt.detector = utils.shift_panelZ(expt.detector,
df.detz_shift_mm.values[0])
if force_no_detector_thickness:
expt.detector = utils.strip_thickness_from_detector(expt.detector)
if reset_Bmatrix:
ucell_params = df[["a", "b", "c", "al", "be", "ga"]].values[0]
ucell_man = utils.manager_from_params(ucell_params)
expt.crystal.set_B(ucell_man.B_recipspace)
assert len(df) == 1
Ncells_abc = df.ncells.values[0]
if Ncells_abc_override is not None:
Ncells_abc = Ncells_abc_override
spot_scale = df.spot_scales.values[0]
beamsize_mm = df.beamsize_mm.values[0]
total_flux = df.total_flux.values[0]
oversample = df.oversample.values[0]
if oversample_override is not None:
oversample = oversample_override
# get the optimized spectra
if spectrum_override is None:
if "spectrum_filename" in list(
df) and df.spectrum_filename.values[0] is not None:
spectrum_file = df.spectrum_filename.values[0]
pink_stride = df.spectrum_stride.values[0]
if norm_by_spectrum:
nspec = utils.load_spectra_file(spectrum_file)[0].shape[0]
spot_scale = spot_scale / nspec
if pink_stride_override is not None:
pink_stride = pink_stride_override
fluxes, energies = utils.load_spectra_file(spectrum_file,
total_flux=total_flux,
pinkstride=pink_stride)
else:
fluxes = np.array([total_flux])
energies = np.array(
[utils.ENERGY_CONV / expt.beam.get_wavelength()])
if not quiet: LOGGER.info("Running MONO sim")
else:
wavelens, fluxes = map(np.array, zip(*spectrum_override))
energies = utils.ENERGY_CONV / wavelens
lam0 = df.lam0.values[0]
lam1 = df.lam1.values[0]
if lam0 == -1:
lam0 = 0
if lam1 == -1:
lam1 = 1
wavelens = utils.ENERGY_CONV / energies
wavelens = lam0 + lam1 * wavelens
energies = utils.ENERGY_CONV / wavelens
if mtz_file is not None:
assert mtz_col is not None
Famp = utils.open_mtz(mtz_file, mtz_col)
else:
Famp = utils.make_miller_array_from_crystal(expt.crystal,
dmin=d_min,
dmax=d_max,
defaultF=defaultF,
symbol=symbol_override)
diffuse_params = None
if "use_diffuse_models" in columns and df.use_diffuse_models.values[0]:
if not use_db:
raise RuntimeError(
"Cant simulate diffuse models unless use_db=True (diffBragg modeler)"
)
diffuse_params = {
"gamma": tuple(df.diffuse_gamma.values[0]),
"sigma": tuple(df.diffuse_sigma.values[0]),
"gamma_miller_units": False
}
if "gamma_miller_units" in list(df):
diffuse_params[
"gamma_miller_units"] = df.gamma_miller_units.values[0]
if use_exascale_api:
#===================
gpu_channels_singleton = gpu_energy_channels(deviceId=0)
print(gpu_channels_singleton.get_deviceID(), "device")
from simtbx.nanoBragg import nanoBragg_crystal
C = nanoBragg_crystal.NBcrystal(init_defaults=False)
C.miller_array = Famp
F_P1 = C.miller_array
F_P1 = Famp.expand_to_p1()
gpu_channels_singleton.structure_factors_to_GPU_direct(
0, F_P1.indices(), F_P1.data())
Famp = gpu_channels_singleton
#===========
results, _, _ = multipanel_sim(CRYSTAL=expt.crystal,
DETECTOR=expt.detector,
BEAM=expt.beam,
Famp=Famp,
energies=energies,
fluxes=fluxes,
Ncells_abc=Ncells_abc,
beamsize_mm=beamsize_mm,
oversample=oversample,
spot_scale_override=spot_scale,
default_F=0,
interpolate=0,
include_background=False,
profile="gauss",
cuda=True,
show_params=False)
return results, expt
elif use_db:
results = diffBragg_forward(CRYSTAL=expt.crystal,
DETECTOR=expt.detector,
BEAM=expt.beam,
Famp=Famp,
fluxes=fluxes,
energies=energies,
beamsize_mm=beamsize_mm,
Ncells_abc=Ncells_abc,
spot_scale_override=spot_scale,
device_Id=device_Id,
oversample=oversample,
show_params=not quiet,
nopolar=nopolar,
printout_pix=printout_pix,
diffuse_params=diffuse_params,
cuda=cuda)
return results, expt
else:
pids = None
if rois_per_panel is not None:
pids = list(rois_per_panel.keys()),
results = flexBeam_sim_colors(CRYSTAL=expt.crystal,
DETECTOR=expt.detector,
BEAM=expt.beam,
Famp=Famp,
fluxes=fluxes,
energies=energies,
beamsize_mm=beamsize_mm,
Ncells_abc=Ncells_abc,
spot_scale_override=spot_scale,
cuda=cuda,
device_Id=device_Id,
oversample=oversample,
time_panels=time_panels and not quiet,
pids=pids,
rois_perpanel=rois_per_panel,
omp=omp,
show_params=not quiet,
nopolar=nopolar,
printout_pix=printout_pix)
if norm_by_nsource:
return np.array([image / len(energies)
for _, image in results]), expt
else:
return np.array([image for _, image in results]), expt
def job_runner(self,i_exp=0,spectra={}):
from simtbx.nanoBragg import utils
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
# Fixed hyperparameters
mosaic_spread_samples = 500
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt beam focal area
spot_scale = 500.
oversample = 1 # factor 1,2, or 3 probably enough
include_background = False
verbose = 0 # leave as 0, unless debug
shapetype = "gauss_argchk"
#<><><><><><><><>
os.environ["NXMX_LOCAL_DATA"]="/global/cfs/cdirs/m3562/der/master_files/run_000795.JF07T32V01_master.h5"
expt = ExperimentListFactory.from_json_file(
experiment_file, check_format=True)[0]
crystal = expt.crystal
detector = expt.detector
flat = True # enforce that the camera has 0 thickness
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
assert detector[0].get_thickness() == 0
beam = expt.beam
spec = expt.imageset.get_spectrum(0)
energies_raw = spec.get_energies_eV().as_numpy_array()
weights_raw = spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw, weights_raw, method=1, total_flux=total_flux, ev_width=ev_res)
device_Id = 0
if self.gpu_channels_singleton is not None:
device_Id = self.gpu_channels_singleton.get_deviceID()
mn_energy = (energies*weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
print("\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG=time()
print (self.gpu_channels_singleton.get_deviceID(),"device",shapetype)
Famp_is_uninitialized = ( self.gpu_channels_singleton.get_nchannels() == 0 )
if Famp_is_uninitialized:
from iotbx.reflection_file_reader import any_reflection_file
from LS49 import ls49_big_data
merge_file = os.path.join(ls49_big_data,"adse13_228","cyto_init_merge.mtz")
self.merged_amplitudes = any_reflection_file(merge_file).as_miller_arrays()[0].as_amplitude_array()
F1 = self.merged_amplitudes.expand_to_p1()
F2 = self.amplitudes.expand_to_p1() # takes care of both transform to asu & expand
if False: # make sure that mtz file (F1) and strong spots (self.amplitudes) are roughly correlated
from matplotlib import pyplot as plt
from cctbx import miller
matches = miller.match_indices( F1.indices(), self.amplitudes.indices() )
sel0 = flex.size_t([p[0] for p in matches.pairs()])
sel1 = flex.size_t([p[1] for p in matches.pairs()])
data0 = F1.data().select(sel0)
data1 = self.amplitudes.data().select(sel1)
plt.plot(data0, data1, 'r.')
plt.show() # yes, the two are very roughly correlated
# end of test
#F_P1 = F1 # legacy, use a merged mtz file
#F_P1 = F2 # this one way absolutely wrong! way too many predictions, beyond the strong spots
F_P1 = F1
for x in range(1): # in this scenario, amplitudes are independent of lambda
self.gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert self.gpu_channels_singleton.get_nchannels() == 1
# Variable parameters
mosaic_spread = 0.07 # degrees
Ncells_abc = 30, 30, 10
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal, DETECTOR=detector, BEAM=beam,
Famp = self.gpu_channels_singleton,
energies=list(energies), fluxes=list(weights),
background_wavelengths=[mn_wave], background_wavelength_weights=[1],
background_total_flux=total_flux,background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample, Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples, mos_spread=mosaic_spread,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False, verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
mask_file=mask_array)
TIME_EXA = time()-BEG
print("\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs" % (TIME_BG, TIME_BRAGG, TIME_EXA))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n")
return JF16M_numpy_array
def tst_one_monkeypatch(i_exp, spectra, Fmerge, gpu_channels_singleton, rank,
params):
print("IN MONKEYPATCH")
from simtbx.nanoBragg import utils
from dxtbx.model.experiment_list import ExperimentListFactory
import numpy as np
print("Experiment %d" % i_exp, flush=True)
outfile = "boop_%d.hdf5" % i_exp
from LS49.adse13_187.case_data import retrieve_from_repo
experiment_file = retrieve_from_repo(i_exp)
cuda = True # False # whether to use cuda
omp = False
ngpu_on_node = 1 # 8 # number of available GPUs
mosaic_spread = 0.07 # degrees
mosaic_spread_samples = params.mosaic_spread_samples # number of mosaic blocks sampling mosaicity
Ncells_abc = 30, 30, 10 # medians from best stage1
ev_res = 1.5 # resolution of the downsample spectrum
total_flux = 1e12 # total flux across channels
beamsize_mm = 0.000886226925452758 # sqrt of beam focal area
spot_scale = 500. # 5.16324 # median from best stage1
plot_spec = False # plot the downsample spectra before simulating
oversample = 1 # oversample factor, 1,2, or 3 probable enough
panel_list = None # integer list of panels, usefule for debugging
rois_only = False # only set True if you are running openMP, or CPU-only (i.e. not for GPU)
include_background = params.include_background # default is to add water background 100 mm thick
verbose = 0 # leave as 0, unles debug
flat = True # enfore that the camera has 0 thickness
#<><><><><><><><>
# XXX new code
El = ExperimentListFactory.from_json_file(experiment_file,
check_format=True)
exper = El[0]
crystal = exper.crystal
detector = exper.detector
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
beam = exper.beam
# XXX new code
spec = exper.imageset.get_spectrum(0)
energies_raw, weights_raw = spec.get_energies_eV().as_numpy_array(), \
spec.get_weights().as_numpy_array()
energies, weights = utils.downsample_spectrum(energies_raw,
weights_raw,
method=1,
total_flux=total_flux,
ev_width=ev_res)
if flat:
assert detector[0].get_thickness() == 0
if panel_list is None:
panel_list = list(range(len(detector)))
pids_for_rank = panel_list
device_Id = 0
if gpu_channels_singleton is not None:
device_Id = gpu_channels_singleton.get_deviceID()
print("Rank %d will use device %d" % (rank, device_Id))
show_params = False
time_panels = (rank == 0)
mn_energy = (energies * weights).sum() / weights.sum()
mn_wave = utils.ENERGY_CONV / mn_energy
jf16m_numpy_array = {}
from_gpu_amplitudes_cuda = {}
print(
"\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk", "gauss"]:
BEG = time()
print(gpu_channels_singleton.get_deviceID(), "device", shapetype)
Famp_is_uninitialized = (gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = Fmerge.expand_to_p1()
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert gpu_channels_singleton.get_nchannels() == 1
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=crystal,
DETECTOR=detector,
BEAM=beam,
Famp=gpu_channels_singleton,
energies=list(energies),
fluxes=list(weights),
background_wavelengths=[mn_wave],
background_wavelength_weights=[1],
background_total_flux=total_flux,
background_sample_thick_mm=0.5,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=mosaic_spread,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=show_params,
time_panels=time_panels,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=include_background,
context=params.context)
TIME_EXA = time() - BEG
jf16m_numpy_array[shapetype] = JF16M_numpy_array
from_gpu_amplitudes_cuda[shapetype] = TIME_BRAGG
print(
"\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs"
% (TIME_BG, TIME_BRAGG, TIME_EXA))
ratio = from_gpu_amplitudes_cuda["gauss"] / from_gpu_amplitudes_cuda[
"gauss_argchk"]
print(
"<><><><><><><><><ratio<%.2f><><><><><><><><><><><><><><><><><><><>\n"
% (ratio))
# assertion on elapsed time:
assert ratio > 1.0, "ratio is %.3f, experiment %d" % (ratio, i_exp)
# assertion on equality:
abs_diff = np.abs(jf16m_numpy_array["gauss"] - \
jf16m_numpy_array["gauss_argchk"]).max()
assert np.allclose(jf16m_numpy_array["gauss"], \
jf16m_numpy_array["gauss_argchk"]), \
"max per-pixel difference: %f photons, experiment %d"%(abs_diff,i_exp)
def job_runner(self,expt,alt_expt,params,mask_array=None,i_exp=0,spectra={},mos_aniso=None):
# Fixed hyperparameters
mosaic_spread_samples = 250
beamsize_mm = 0.000886226925452758 # sqrt beam focal area
spot_scale = 500.
oversample = 1 # factor 1,2, or 3 probably enough
verbose = 0 # leave as 0, unless debug
shapetype = "gauss_argchk"
if mask_array is not None:
assert type(mask_array) is flex.bool # type check intending to convert active-pixel-bools to whitelist-ints
active_pixels = flex.int()
for i, x in enumerate(mask_array):
if x: active_pixels.append(i)
mask_array = active_pixels
detector = expt.detector
flat = True # enforce that the camera has 0 thickness
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
assert detector[0].get_thickness() == 0
alt_crystal = alt_expt.crystal
beam = expt.beam
spec = expt.imageset.get_spectrum(0)
energies_raw = spec.get_energies_eV().as_numpy_array()
weights_raw = spec.get_weights().as_numpy_array()
from LS49.adse13_187.adse13_221.explore_spectrum import method3
energies, weights, _ = method3(energies_raw, weights_raw,); weights = 5000000.*weights
energies = list(energies); weights = list(weights)
device_Id = 0
if self.gpu_channels_singleton is not None:
device_Id = self.gpu_channels_singleton.get_deviceID()
print("\n<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>")
print("\tBreakdown:")
for shapetype in ["gauss_argchk"]:
BEG=time()
print (self.gpu_channels_singleton.get_deviceID(),"device",shapetype)
Famp_is_uninitialized = ( self.gpu_channels_singleton.get_nchannels() == 0 )
if Famp_is_uninitialized:
F_P1 = self.amplitudes
for x in range(1): # in this scenario, amplitudes are independent of lambda
self.gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert self.gpu_channels_singleton.get_nchannels() == 1
# Variable parameters
mosaic_spread = params.mosaic_spread.value
Ncells_abc = params.Nabc.value
JF16M_numpy_array, TIME_BG, TIME_BRAGG, _ = multipanel_sim(
CRYSTAL=alt_crystal, DETECTOR=detector, BEAM=beam,
Famp = self.gpu_channels_singleton,
energies=energies, fluxes=weights,
cuda=True,
oversample=oversample, Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples, mos_spread=mosaic_spread,
mos_aniso = mos_aniso,
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False, verbose=verbose,
spot_scale_override=spot_scale,
include_background=False,
mask_file=mask_array)
TIME_EXA = time()-BEG
print("\t\tExascale: time for bkgrd sim: %.4fs; Bragg sim: %.4fs; total: %.4fs" % (TIME_BG, TIME_BRAGG, TIME_EXA))
print("<><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>\n")
return JF16M_numpy_array
def chain_runner(self,
expt,
alt_expt,
params,
mask_array=None,
n_cycles=100,
s_cycles=0,
Zscore_callback=None,
rmsd_callback=None):
self.model_plot_enable = params.plot
if mask_array is not None:
assert type(
mask_array
) is flex.bool # type check intending to convert active-pixel-bools to whitelist-ints
active_pixels = flex.int()
for i, x in enumerate(mask_array):
if x: active_pixels.append(i)
mask_array = active_pixels
# Fixed hyperparameters
mosaic_spread_samples = 250
beamsize_mm = 0.000886226925452758 # sqrt beam focal area
spot_scale = 500.
oversample = 1 # factor 1,2, or 3 probably enough
verbose = 0 # leave as 0, unless debug
shapetype = "gauss_argchk"
detector = expt.detector
flat = True # enforce that the camera has 0 thickness
if flat:
from dxtbx_model_ext import SimplePxMmStrategy
for panel in detector:
panel.set_px_mm_strategy(SimplePxMmStrategy())
panel.set_mu(0)
panel.set_thickness(0)
assert detector[0].get_thickness() == 0
alt_crystal = copy.deepcopy(
alt_expt.crystal) # avoid perturbing the original dials cell
beam = expt.beam
spec = expt.imageset.get_spectrum(0)
energies_raw = spec.get_energies_eV().as_numpy_array()
weights_raw = spec.get_weights().as_numpy_array()
from LS49.adse13_187.adse13_221.explore_spectrum import method3
energies, weights, _ = method3(
energies_raw,
weights_raw,
)
weights = 5000000. * weights
energies = list(energies)
weights = list(weights)
device_Id = 0 # XXX revisit for multiprocess service
assert self.gpu_channels_singleton is not None
device_Id = self.gpu_channels_singleton.get_deviceID()
print(device_Id, "device", shapetype)
Famp_is_uninitialized = (
self.gpu_channels_singleton.get_nchannels() == 0)
if Famp_is_uninitialized:
F_P1 = self.amplitudes
for x in range(
1
): # in this scenario, amplitudes are independent of lambda
self.gpu_channels_singleton.structure_factors_to_GPU_direct(
x, F_P1.indices(), F_P1.data())
assert self.gpu_channels_singleton.get_nchannels() == 1
# Variable parameters
mosaic_spread = params.mosaic_spread.value
Ncells_abc = params.Nabc.value
from LS49.adse13_187.adse13_221.parameters import variable_mosaicity
from LS49.adse13_187.adse13_221.parameters import covariant_cell, covariant_rot, covariant_ncells
self.parameters = {}
self.parameters["cell"] = covariant_cell.from_covariance(
alt_crystal, params.cell)
self.parameters["etaa"] = variable_mosaicity(
mosaic_spread, label="η a", params=params.mosaic_spread)
self.parameters["etab"] = variable_mosaicity(
mosaic_spread, label="η b", params=params.mosaic_spread)
self.parameters["etac"] = variable_mosaicity(
mosaic_spread, label="η c", params=params.mosaic_spread)
self.parameters2 = {}
if params.rot.refine:
self.parameters2["rot"] = covariant_rot(alt_crystal, params.rot)
if params.Nabc.refine:
self.parameters2["ncells"] = covariant_ncells(params.Nabc)
self.ref_params = {}
self.ref_params.update(self.parameters)
self.ref_params.update(self.parameters2)
# XXX TO DO list (Nick/Dan discuss)
# 1) change the variable mosaicity model to use updated aniso Derek model (Nick)
# 2) add rotx/roty/rotz. Covariant excursion values from Sauter 2014 paper: rotz 0.02° rotx 0.03° roty 0.03°
# 3) refine ncells a b c
self.rmsd_chain = flex.double()
self.sigz_chain = flex.double()
self.llg_chain = flex.double()
self.cycle_list = [key for key in self.ref_params]
self.accept = flex.int()
self.beginning_iteration = 0
if s_cycles > 0:
from LS49.adse13_187.adse13_221.simplex_method import simplex_detail
# initialize prior to simplex
whitelist_only, TIME_BG, TIME_BRAGG, self.exascale_mos_blocks = multipanel_sim(
CRYSTAL=alt_crystal,
DETECTOR=detector,
BEAM=beam,
Famp=self.gpu_channels_singleton,
energies=energies,
fluxes=weights,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=self.parameters["etaa"].proposal,
mos_aniso=(self.parameters["etaa"].proposal,
self.parameters["etab"].proposal,
self.parameters["etac"].proposal),
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=False,
mask_file=mask_array,
skip_numpy=True,
relevant_whitelist_order=self.relevant_whitelist_order)
Rmsd, sigZ, LLG = Zscore_callback(kernel_model=whitelist_only,
plot=False)
self.accept.append(1)
self.rmsd_chain.append(Rmsd)
self.sigz_chain.append(sigZ)
self.llg_chain.append(LLG)
PP = dict(detector=detector,
beam=beam,
energies=energies,
weights=weights,
oversample=oversample,
mosaic_spread_samples=mosaic_spread_samples,
beamsize_mm=beamsize_mm,
shapetype=shapetype,
verbose=verbose,
spot_scale=spot_scale,
mask_array=mask_array,
Z=Zscore_callback)
MIN = simplex_detail(alt_crystal,
Ncells_abc,
host_runner=self,
PP=PP,
n_cycles=n_cycles,
s_cycles=s_cycles)
self.beginning_iteration = MIN.iteration + 1
for macro_iteration in range(self.beginning_iteration, n_cycles):
BEG = time()
turn = self.cycle_list[macro_iteration % len(self.cycle_list)]
if turn == "cell":
alt_crystal = self.parameters[
"cell"].get_current_crystal_model(alt_crystal)
elif turn == "rot":
alt_crystal = self.parameters2[
"rot"].get_current_crystal_model(alt_crystal)
elif turn == "ncells":
Ncells_abc = self.parameters2["ncells"].get_current_model()
whitelist_only, TIME_BG, TIME_BRAGG, self.exascale_mos_blocks = multipanel_sim(
CRYSTAL=alt_crystal,
DETECTOR=detector,
BEAM=beam,
Famp=self.gpu_channels_singleton,
energies=energies,
fluxes=weights,
cuda=True,
oversample=oversample,
Ncells_abc=Ncells_abc,
mos_dom=mosaic_spread_samples,
mos_spread=self.parameters["etaa"].proposal,
mos_aniso=(self.parameters["etaa"].proposal,
self.parameters["etab"].proposal,
self.parameters["etac"].proposal),
beamsize_mm=beamsize_mm,
profile=shapetype,
show_params=False,
time_panels=False,
verbose=verbose,
spot_scale_override=spot_scale,
include_background=False,
mask_file=mask_array,
skip_numpy=True,
relevant_whitelist_order=self.relevant_whitelist_order)
Rmsd, sigZ, LLG = Zscore_callback(kernel_model=whitelist_only,
plot=False)
if macro_iteration == self.beginning_iteration:
for key in self.ref_params:
self.ref_params[key].accept()
self.accept.append(1)
self.rmsd_chain.append(Rmsd)
self.sigz_chain.append(sigZ)
self.llg_chain.append(LLG)
else:
print("Old NLL ", self.llg_chain[-1], "NEW LLG", LLG, "diff",
self.llg_chain[-1] - LLG)
this_cycle_key = self.cycle_list[(macro_iteration) %
len(self.cycle_list)]
acceptance_prob = min(
1.,
math.exp((self.llg_chain[-1] - LLG) /
len(whitelist_only)) # normalize by no. of pixels
* self.ref_params[this_cycle_key].
transition_probability_ratio # q(X|Y)/q(Y|X), Y=proposal, X=last value
)
if random.random() < acceptance_prob:
for key in self.ref_params:
if key == turn: self.ref_params[key].accept()
else: self.ref_params[key].reject()
self.accept.append(1)
self.rmsd_chain.append(Rmsd)
self.sigz_chain.append(sigZ)
self.llg_chain.append(LLG)
else:
for key in self.ref_params:
self.ref_params[key].reject()
self.accept.append(0)
self.rmsd_chain.append(self.rmsd_chain[-1])
self.sigz_chain.append(self.sigz_chain[-1])
self.llg_chain.append(self.llg_chain[-1])
P = Profiler("%40s" % "key maintenance")
for key in self.ref_params:
if key == self.cycle_list[(macro_iteration + 1) %
len(self.cycle_list)]:
self.ref_params[key].generate_next_proposal()
P = Profiler("%40s" % "plot all")
self.plot_all(macro_iteration + 1, of=n_cycles)
del P
TIME_EXA = time() - BEG
print("\t\tExascale: time for Bragg sim: %.4fs; total: %.4fs\n" %
(TIME_BRAGG, TIME_EXA))
print("MCMC <RMSD> %.2f" %
(flex.mean(self.rmsd_chain[len(self.rmsd_chain) // 2:])))
print("MCMC <sigz> %.2f" %
(flex.mean(self.sigz_chain[len(self.sigz_chain) // 2:])))
print("MCMC <-LLG> %.2f" %
(flex.mean(self.llg_chain[len(self.llg_chain) // 2:])))
for key in self.ref_params:
self.ref_params[key].show()
if self.model_plot_enable:
plt.close(self.fig)
plt.close(self.fig2)
plt.close(self.fig3)
plt.ioff()
print(flush=True)