def run_uniform(eta_angle, sample_size=20000, verbose=True):
UMAT = flex.mat3_double()
d_UMAT_d_eta = flex.mat3_double()
# the axis is sampled randomly on a sphere.
# Alternately it could have been calculated on a regular hemispheric grid
# but regular grid is not needed; the only goal is to have the ability to compute gradient
mersenne_twister = flex.mersenne_twister(
seed=0) # set seed, get reproducible results
# for each axis, sample both the + and - angle
assert sample_size % 2 == 0
# the angle is sampled uniformly from its distribution
mosaic_rotation0 = np.array(range(sample_size // 2))
mosaic_rotation1 = special.erfinv(mosaic_rotation0 / (sample_size // 2))
d_theta_d_eta = math.sqrt(2.0) * flex.double(mosaic_rotation1)
mosaic_rotation = (math.pi / 180.) * eta_angle * d_theta_d_eta
for m, d in zip(mosaic_rotation, d_theta_d_eta):
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False))
UMAT.append(site.axis_and_angle_as_r3_rotation_matrix(-m, deg=False))
d_umat = site.axis_and_angle_as_r3_derivative_wrt_angle(m, deg=False)
d_UMAT_d_eta.append((math.pi / 180.) * d * d_umat)
d_umat = site.axis_and_angle_as_r3_derivative_wrt_angle(-m, deg=False)
d_UMAT_d_eta.append((math.pi / 180.) * -d * d_umat)
#sanity check on the gaussian distribution
if verbose:
nm_angles = check_distributions.get_angular_rotation(UMAT)
nm_rms_angle = math.sqrt(flex.mean(nm_angles * nm_angles))
print("Normal rms angle is ", nm_rms_angle)
return UMAT, d_UMAT_d_eta
def Umats(mos_spread_deg,
n_mos_doms,
isotropic=True,
seed=777,
norm_dist_seed=777):
import scitbx
from scitbx.matrix import col
import math
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=seed)
scitbx.random.set_random_seed(norm_dist_seed)
rand_norm = scitbx.random.normal_distribution(mean=0,
sigma=(mos_spread_deg *
math.pi / 180.0))
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(n_mos_doms)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
if mos_spread_deg > 0:
UMAT_nm.append(
site.axis_and_angle_as_r3_rotation_matrix(m, deg=False))
else:
UMAT_nm.append(
site.axis_and_angle_as_r3_rotation_matrix(0, deg=False))
if isotropic and mos_spread_deg > 0:
UMAT_nm.append(
site.axis_and_angle_as_r3_rotation_matrix((-m), deg=False))
return UMAT_nm
def gradients(self):
"""A generator function to return the gradients dR/dp for all the restraints
referring to a particular crystal's cell parameters. The return value is
a list of sparse matrices, one for each of the 6 cell parameters being
restrained. Each sparse matrix has as many columns as the crystal unit
cell parameterisation has parameters, and as many rows as there are crystals
being restrained. Gradients of zero are detected and not set in the sparse
matrices to save memory."""
for i, xlucp in enumerate(self._xlucp):
B = xlucp.get_state()
dB_dp = flex.mat3_double(xlucp.get_ds_dp())
# Use C++ function for speed
ccg = CalculateCellGradients(B, dB_dp)
dRdp = []
if self._sel[0]:
dRdp.append(self._construct_grad_block(ccg.da_dp(), i))
if self._sel[1]:
dRdp.append(self._construct_grad_block(ccg.db_dp(), i))
if self._sel[2]:
dRdp.append(self._construct_grad_block(ccg.dc_dp(), i))
if self._sel[3]:
dRdp.append(self._construct_grad_block(ccg.daa_dp(), i))
if self._sel[4]:
dRdp.append(self._construct_grad_block(ccg.dbb_dp(), i))
if self._sel[5]:
dRdp.append(self._construct_grad_block(ccg.dcc_dp(), i))
yield dRdp
def gradients(self):
"""A generator function to return the gradients dR/dp for all the restraints
referring to a particular crystal's cell parameters. The return value is
a list of sparse matrices, one for each of the 6 cell parameters being
restrained. Each sparse matrix has as many columns as the crystal unit
cell parameterisation has parameters, and as many rows as there are crystals
being restrained. Gradients of zero are detected and not set in the sparse
matrices to save memory."""
for i, xlucp in enumerate(self._xlucp):
B = xlucp.get_state()
dB_dp = flex.mat3_double(xlucp.get_ds_dp())
# Use C++ function for speed
ccg = CalculateCellGradients(B, dB_dp)
dRdp = []
if self._sel[0]:
dRdp.append(self._construct_grad_block(ccg.da_dp(), i))
if self._sel[1]:
dRdp.append(self._construct_grad_block(ccg.db_dp(), i))
if self._sel[2]:
dRdp.append(self._construct_grad_block(ccg.dc_dp(), i))
if self._sel[3]:
dRdp.append(self._construct_grad_block(ccg.daa_dp(), i))
if self._sel[4]:
dRdp.append(self._construct_grad_block(ccg.dbb_dp(), i))
if self._sel[5]:
dRdp.append(self._construct_grad_block(ccg.dcc_dp(), i))
yield dRdp
def channel_wavelength_fmodel(create):
N_mosaic_domains = 25
mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(mean=0, sigma=mosaic_spread_deg * math.pi/180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(N_mosaic_domains)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) )
if create: # write the reference for the first time
cPickle.dump(UMAT_nm,
open(os.path.join(ls49_big_data,"reference",filename),"wb"),cPickle.HIGHEST_PROTOCOL)
else: # read the reference and assert sameness to production run
import six
if six.PY3:
UMAT_ref = cPickle.load(open(os.path.join(ls49_big_data,"reference",filename),"rb"),encoding="bytes")
else:
UMAT_ref = cPickle.load(open(os.path.join(ls49_big_data,"reference",filename),"rb"))
for x in range(len(UMAT_nm)):
print(x," ".join(
["%18.15f"%UMAT_ref[x][z] for z in range(9)]
))
assert UMAT_nm[x] == UMAT_ref[x]
expected_output = """
def _init_nanoBragg_umats(self, model_mosaic_spread):
# umat implementation for the exascale api
if model_mosaic_spread and self.umat_maker is None:
# initialize the parameterized distribution
assert self.crystal.n_mos_domains % 2 == 0
self.umat_maker = AnisoUmats(num_random_samples=2*self.crystal.n_mos_domains)
if self.crystal.anisotropic_mos_spread_deg is None:
# isotropic case
self.D.mosaic_spread_deg = self.crystal.mos_spread_deg
self.D.mosaic_domains = self.crystal.n_mos_domains
if self.Umats_method == 0: # double_random legacy method, exafel tests
Umats = SimData.Umats(self.crystal.mos_spread_deg,
self.crystal.n_mos_domains,
seed=self.mosaic_seeds[0],
norm_dist_seed=self.mosaic_seeds[1])
elif self.Umats_method == 2: # double_uniform isotropic, NOTE: if Umats_method was set as 5, it was modified to be 2
Umats, _,_ = self.umat_maker.generate_Umats(
self.crystal.mos_spread_deg, compute_derivs=False, crystal=None, how=2)
else: raise ValueError("Invalid method for nanoBragg isotropic case")
#SimData.plot_isotropic_umats(Umats, self.crystal.mos_spread_deg)
else:
# anisotropic case with diagonal elements
Umats, _,_ = self.umat_maker.generate_Umats(
self.crystal.anisotropic_mos_spread_deg, compute_derivs=False, crystal=self.crystal.dxtbx_crystal, how=1)
self.exascale_mos_blocks = flex.mat3_double(Umats)
self.D.set_mosaic_blocks(self.exascale_mos_blocks)
def run_sim2smv(fileout):
SIM = nanoBragg(detpixels_slowfast=(1000, 1000),
pixel_size_mm=0.1,
Ncells_abc=(5, 5, 5),
verbose=0)
SIM.mosaic_spread_deg = MOSAIC_SPREAD # apparently this is half width
SIM.mosaic_domains = SAMPLE_SIZE
SIM.distance_mm = 100 # this triggers the generation of mosaic distribution
UMAT_th = SIM.get_mosaic_blocks() # extract top-hat distributed U-mats
#sanity checks on the top hat distribution
th_angles = check_distributions.get_angular_rotation(UMAT_th)
max_angle = flex.max(th_angles)
assert max_angle <= MOSAIC_SPREAD + 0.0000001 # need to allow a small epsilon
assert max_angle > 0.99 * MOSAIC_SPREAD # insist that max angle is near the limit we gave it
rms_angle = math.sqrt(flex.mean(th_angles * th_angles))
print(rms_angle)
assert rms_angle < MOSAIC_SPREAD
import scitbx # compute an array of normally-distributed U-mats into the simulator
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(
seed=0) # set seed, get reproducible results
scitbx.random.set_random_seed(4321) # set seed, get reproducibe results
rand_norm = scitbx.random.normal_distribution(mean=0,
sigma=rms_angle * math.pi /
180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(SAMPLE_SIZE)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False))
# set the normally-distributed U-mats into the simulator:
SIM.set_mosaic_blocks(UMAT_nm)
# get them back
new_UMAT_nm = SIM.get_mosaic_blocks()
#double check that simulator does not alter the UMATs
for iumat in range(0, SAMPLE_SIZE, 100):
assert UMAT_nm[iumat] == new_UMAT_nm[iumat]
#sanity check on the gaussian distribution
nm_angles = check_distributions.get_angular_rotation(new_UMAT_nm)
nm_rms_angle = math.sqrt(flex.mean(nm_angles * nm_angles))
print(nm_rms_angle)
assert approx_equal(rms_angle,nm_rms_angle,eps=1e-03), \
"The top hat and gaussian models should have similar standard deviations"
return UMAT_th, new_UMAT_nm
def test_flex_looping_vecs():
# We want to try and avoid recursive nesting, but don't want to
# return the original object if attempting to miscast vec<->numeric
# however.... this means lots of conditions to check, so ignore now
fo = flex.vec3_double(5)
npo = flumpy.to_numpy(fo)
assert flumpy.vec_from_numpy(npo) is fo
# Don't be dumb when casting
flex_nonvec = flex.double((9, 3))
assert not flumpy.vec_from_numpy(
flumpy.to_numpy(flex_nonvec)) is flex_nonvec
# mat3
fo = flex.mat3_double(5)
npo = flumpy.to_numpy(fo)
assert flumpy.mat3_from_numpy(npo) is fo
def _unit_cell_surplus_reflections(self, p):
F_dbdp = flex.mat3_double(p.get_ds_dp())
min_nref = self._options.min_nref_per_parameter
reflections = self.reflection_manager.get_obs()
# if no free parameters, do as _surplus_reflections
if len(F_dbdp) == 0:
exp_ids = p.get_experiment_ids()
isel = flex.size_t()
for exp_id in exp_ids:
isel.extend((reflections["id"] == exp_id).iselection())
return len(isel)
return (uc_surpl(
reflections["id"],
reflections["miller_index"],
p.get_experiment_ids(),
F_dbdp,
).result - min_nref)
def run_sim2smv(fileout):
SIM = nanoBragg(detpixels_slowfast=(1000,1000),pixel_size_mm=0.1,Ncells_abc=(5,5,5),verbose=0)
SIM.mosaic_domains = 10000
SIM.mosaic_spread_deg = 2.0
SIM.distance_mm=100 # this triggers the generation of mosaic distribution
UMAT_th = SIM.get_mosaic_blocks()
import scitbx
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
rand_norm = scitbx.random.normal_distribution(mean=0, sigma=math.pi/180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(10000)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) )
return UMAT_th, UMAT_nm
def _calculate_uc_gradients(self, sel=[True] * 6):
"""Calculate gradients of the unit cell parameters with respect to
each of the parameters of the crystal unit cell model parameterisation"""
B = self._xlucp.get_state()
dB_dp = flex.mat3_double(self._xlucp.get_ds_dp())
# Use C++ function for speed
ccg = CalculateCellGradients(B, dB_dp)
nparam = len(dB_dp)
da = list(ccg.da_dp()) if sel[0] else [0.0] * nparam
db = list(ccg.db_dp()) if sel[1] else [0.0] * nparam
dc = list(ccg.dc_dp()) if sel[2] else [0.0] * nparam
daa = list(ccg.daa_dp()) if sel[3] else [0.0] * nparam
dbb = list(ccg.dbb_dp()) if sel[4] else [0.0] * nparam
dcc = list(ccg.dcc_dp()) if sel[5] else [0.0] * nparam
return (da, db, dc, daa, dbb, dcc)
def _calculate_uc_gradients(self, sel=[True]*6):
'''Calculate gradients of the unit cell parameters with respect to
each of the parameters of the crystal unit cell model parameterisation'''
B = self._xlucp.get_state()
dB_dp = flex.mat3_double(self._xlucp.get_ds_dp())
# Use C++ function for speed
ccg = CalculateCellGradients(B, dB_dp)
nparam = len(dB_dp)
da = list(ccg.da_dp()) if sel[0] else [0.0] * nparam
db = list(ccg.db_dp()) if sel[1] else [0.0] * nparam
dc = list(ccg.dc_dp()) if sel[2] else [0.0] * nparam
daa = list(ccg.daa_dp()) if sel[3] else [0.0] * nparam
dbb = list(ccg.dbb_dp()) if sel[4] else [0.0] * nparam
dcc = list(ccg.dcc_dp()) if sel[5] else [0.0] * nparam
return (da, db, dc, daa, dbb, dcc)
def run_sim2smv(fileout):
SIM = nanoBragg(detpixels_slowfast=(1000, 1000),
pixel_size_mm=0.1,
Ncells_abc=(5, 5, 5),
verbose=0)
SIM.mosaic_domains = 10000
SIM.mosaic_spread_deg = MOSAIC_SPREAD
SIM.distance_mm = 100 # this triggers the generation of mosaic distribution
UMAT_th = SIM.get_mosaic_blocks() # extract top-hat distributed U-mats
#sanity checks on the top hat distribution
th_angles = check_distributions.get_angular_rotation(UMAT_th)
max_angle = flex.max(th_angles)
assert max_angle <= MOSAIC_SPREAD + 0.0000001 # need to allow a small epsilon
assert max_angle > 0.99 * MOSAIC_SPREAD # insist that max angle is near the limit we gave it
rms_angle = math.sqrt(flex.mean(th_angles * th_angles))
print rms_angle
import scitbx # computer an array of normally-distributed U-mats into the simulator
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
rand_norm = scitbx.random.normal_distribution(mean=0,
sigma=rms_angle * math.pi /
180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(10000)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False))
# set the normally-distributed U-mats into the simulator:
SIM.set_mosaic_blocks(UMAT_nm)
# get them back
new_UMAT_nm = SIM.get_mosaic_blocks()
#sanity check on the gaussian distribution
nm_angles = check_distributions.get_angular_rotation(new_UMAT_nm)
nm_rms_angle = math.sqrt(flex.mean(nm_angles * nm_angles))
print nm_rms_angle
return UMAT_th, new_UMAT_nm
def ref_gen_varying(experiments):
"""Generate some reflections using the scan varying predictor"""
beam = experiments[0].beam
crystal = experiments[0].crystal
detector = experiments[0].detector
scan = experiments[0].scan
# We need a UB matrix at the beginning of every image, and at the end of the
# last image. These are all the same - we want to compare the scan-varying
# predictor with the scan-static one for a flat scan.
ar_range = scan.get_array_range()
UBlist = [crystal.get_A() for t in range(ar_range[0], ar_range[1] + 1)]
dmin = detector.get_max_resolution(beam.get_s0())
from dials.algorithms.spot_prediction import ScanVaryingReflectionPredictor
sv_predictor = ScanVaryingReflectionPredictor(experiments[0], dmin=dmin)
refs = sv_predictor.for_ub(flex.mat3_double(UBlist))
return refs
def ref_gen_varying(experiments):
"""Generate some reflections using the scan varying predictor"""
beam = experiments[0].beam
crystal = experiments[0].crystal
goniometer = experiments[0].goniometer
detector = experiments[0].detector
scan = experiments[0].scan
# We need a UB matrix at the beginning of every image, and at the end of the
# last image. These are all the same - we want to compare the scan-varying
# predictor with the scan-static one for a flat scan.
ar_range = scan.get_array_range()
UBlist = [crystal.get_A() for t in range(ar_range[0], ar_range[1]+1)]
dmin = detector.get_max_resolution(beam.get_s0())
from dials.algorithms.spot_prediction import ScanVaryingReflectionPredictor
sv_predictor = ScanVaryingReflectionPredictor(experiments[0],
dmin=dmin)
refs = sv_predictor.for_ub(flex.mat3_double(UBlist))
return refs
def __init__(self, mosaic_domains=25, mosaic_spread_deg=0.05):
local_data = data()
direct_algo_res_limit = 1.7
wavelength_A = 1.3 # Angstroms, dummy value
GF = gen_fmodel(resolution=direct_algo_res_limit,
pdb_text=local_data.get("pdb_lines"),
algorithm="fft",
wavelength=wavelength_A)
GF.set_k_sol(0.435)
GF.make_P1_primitive()
self.sfall_main = GF.get_amplitudes()
SIM = nanoBragg(
detpixels_slowfast=(3000, 3000),
pixel_size_mm=0.11,
Ncells_abc=(10, 10, 10),
# workaround for problem with wavelength array, specify it separately in constructor.
wavelength_A=wavelength_A,
verbose=0)
SIM.mosaic_spread_deg = mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = mosaic_domains # mosaic_domains setter must come after mosaic_spread_deg setter
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(
mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(SIM.mosaic_domains)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append(
site.axis_and_angle_as_r3_rotation_matrix(m, deg=False))
SIM.set_mosaic_blocks(UMAT_nm)
self.SIM = SIM
def run_sim2smv(prefix, crystal, spectra, rotation, rank, quick=False):
local_data = data()
smv_fileout = prefix + ".img"
if quick is not True:
if not write_safe(smv_fileout):
print("File %s already exists, skipping in rank %d" %
(smv_fileout, rank))
return
direct_algo_res_limit = 1.7
wavlen, flux, wavelength_A = next(
spectra) # list of lambdas, list of fluxes, average wavelength
if quick:
wavlen = flex.double([wavelength_A])
flux = flex.double([flex.sum(flux)])
print("Quick sim, lambda=%f, flux=%f" % (wavelength_A, flux[0]))
GF = gen_fmodel(resolution=direct_algo_res_limit,
pdb_text=local_data.get("pdb_lines"),
algorithm="fft",
wavelength=wavelength_A)
GF.set_k_sol(0.435)
GF.make_P1_primitive()
sfall_main = GF.get_amplitudes()
# use crystal structure to initialize Fhkl array
sfall_main.show_summary(prefix="Amplitudes used ")
N = crystal.number_of_cells(sfall_main.unit_cell())
#SIM = nanoBragg(detpixels_slowfast=(2000,2000),pixel_size_mm=0.11,Ncells_abc=(5,5,5),verbose=0)
SIM = nanoBragg(
detpixels_slowfast=(3000, 3000),
pixel_size_mm=0.11,
Ncells_abc=(N, N, N),
# workaround for problem with wavelength array, specify it separately in constructor.
wavelength_A=wavelength_A,
verbose=0)
SIM.adc_offset_adu = 0 # Do not offset by 40
SIM.adc_offset_adu = 10 # Do not offset by 40
import sys
if len(sys.argv) > 2:
SIM.seed = -int(sys.argv[2])
print("GOTHERE seed=", SIM.seed)
if len(sys.argv) > 1:
if sys.argv[1] == "random": SIM.randomize_orientation()
SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = n_mosaic_domains # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image
# 3000000 images would be 100000 hours on a 60-core machine (dials), or 11.4 years
# using 2 nodes, 5.7 years. Do this at SLAC? NERSC? combination of all?
# SLAC downtimes: Tues Dec 5 (24 hrs), Mon Dec 11 (72 hrs), Mon Dec 18 light use, 24 days
# mosaic_domains setter must come after mosaic_spread_deg setter
SIM.distance_mm = 141.7
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(mean=0,
sigma=SIM.mosaic_spread_deg *
math.pi / 180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(SIM.mosaic_domains)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False))
SIM.set_mosaic_blocks(UMAT_nm)
#SIM.detector_thick_mm = 0.5 # = 0 for Rayonix
#SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD
#SIM.detector_attenuation_length_mm = default is silicon
# get same noise each time this test is run
SIM.seed = 1
SIM.oversample = 1
SIM.wavelength_A = wavelength_A
SIM.polarization = 1
# this will become F000, marking the beam center
SIM.default_F = 0
#SIM.missets_deg= (10,20,30)
print("mosaic_seed=", SIM.mosaic_seed)
print("seed=", SIM.seed)
print("calib_seed=", SIM.calib_seed)
print("missets_deg =", SIM.missets_deg)
SIM.Fhkl = sfall_main
print("Determinant", rotation.determinant())
Amatrix_rot = (
rotation *
sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose()
print("RAND_ORI", prefix, end=' ')
for i in Amatrix_rot:
print(i, end=' ')
print()
SIM.Amatrix_RUB = Amatrix_rot
#workaround for failing init_cell, use custom written Amatrix setter
print("unit_cell_Adeg=", SIM.unit_cell_Adeg)
print("unit_cell_tuple=", SIM.unit_cell_tuple)
Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM
from cctbx import crystal_orientation
Ori = crystal_orientation.crystal_orientation(
Amat, crystal_orientation.basis_type.reciprocal)
print("Python unit cell from SIM state", Ori.unit_cell())
# fastest option, least realistic
#SIM.xtal_shape=shapetype.Tophat # RLP = hard sphere
#SIM.xtal_shape=shapetype.Square # gives fringes
SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian
#SIM.xtal_shape=shapetype.Round # Crystal is a hard sphere
# only really useful for long runs
SIM.progress_meter = False
# prints out value of one pixel only. will not render full image!
#SIM.printout_pixel_fastslow=(500,500)
#SIM.printout=True
SIM.show_params()
# flux is always in photons/s
SIM.flux = 1e12
SIM.exposure_s = 1.0 # so total fluence is e12
# assumes round beam
SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense
temp = SIM.Ncells_abc
print("Ncells_abc=", SIM.Ncells_abc)
SIM.Ncells_abc = temp
print("Ncells_abc=", SIM.Ncells_abc)
print("xtal_size_mm=", SIM.xtal_size_mm)
print("unit_cell_Adeg=", SIM.unit_cell_Adeg)
print("unit_cell_tuple=", SIM.unit_cell_tuple)
print("missets_deg=", SIM.missets_deg)
print("Amatrix=", SIM.Amatrix)
print("beam_center_mm=", SIM.beam_center_mm)
print("XDS_ORGXY=", SIM.XDS_ORGXY)
print("detector_pivot=", SIM.detector_pivot)
print("xtal_shape=", SIM.xtal_shape)
print("beamcenter_convention=", SIM.beamcenter_convention)
print("fdet_vector=", SIM.fdet_vector)
print("sdet_vector=", SIM.sdet_vector)
print("odet_vector=", SIM.odet_vector)
print("beam_vector=", SIM.beam_vector)
print("polar_vector=", SIM.polar_vector)
print("spindle_axis=", SIM.spindle_axis)
print("twotheta_axis=", SIM.twotheta_axis)
print("distance_meters=", SIM.distance_meters)
print("distance_mm=", SIM.distance_mm)
print("close_distance_mm=", SIM.close_distance_mm)
print("detector_twotheta_deg=", SIM.detector_twotheta_deg)
print("detsize_fastslow_mm=", SIM.detsize_fastslow_mm)
print("detpixels_fastslow=", SIM.detpixels_fastslow)
print("detector_rot_deg=", SIM.detector_rot_deg)
print("curved_detector=", SIM.curved_detector)
print("pixel_size_mm=", SIM.pixel_size_mm)
print("point_pixel=", SIM.point_pixel)
print("polarization=", SIM.polarization)
print("nopolar=", SIM.nopolar)
print("oversample=", SIM.oversample)
print("region_of_interest=", SIM.region_of_interest)
print("wavelength_A=", SIM.wavelength_A)
print("energy_eV=", SIM.energy_eV)
print("fluence=", SIM.fluence)
print("flux=", SIM.flux)
print("exposure_s=", SIM.exposure_s)
print("beamsize_mm=", SIM.beamsize_mm)
print("dispersion_pct=", SIM.dispersion_pct)
print("dispsteps=", SIM.dispsteps)
print("divergence_hv_mrad=", SIM.divergence_hv_mrad)
print("divsteps_hv=", SIM.divsteps_hv)
print("divstep_hv_mrad=", SIM.divstep_hv_mrad)
print("round_div=", SIM.round_div)
print("phi_deg=", SIM.phi_deg)
print("osc_deg=", SIM.osc_deg)
print("phisteps=", SIM.phisteps)
print("phistep_deg=", SIM.phistep_deg)
print("detector_thick_mm=", SIM.detector_thick_mm)
print("detector_thicksteps=", SIM.detector_thicksteps)
print("detector_thickstep_mm=", SIM.detector_thickstep_mm)
print("***mosaic_spread_deg=", SIM.mosaic_spread_deg)
print("***mosaic_domains=", SIM.mosaic_domains)
print("indices=", SIM.indices)
print("amplitudes=", SIM.amplitudes)
print("Fhkl_tuple=", SIM.Fhkl_tuple)
print("default_F=", SIM.default_F)
print("interpolate=", SIM.interpolate)
print("integral_form=", SIM.integral_form)
from libtbx.development.timers import Profiler
P = Profiler("nanoBragg")
# now actually burn up some CPU
#SIM.add_nanoBragg_spots()
del P
# simulated crystal is only 125 unit cells (25 nm wide)
# amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume)
print(crystal.domains_per_crystal)
SIM.raw_pixels *= crystal.domains_per_crystal
# must calculate the correct scale!
# Use single wavelength for all energy channels for the purpose of Fcalc
wavelength_hi_remote = wavlen[-1]
GF.reset_wavelength(wavelength_hi_remote)
GF.reset_specific_at_wavelength(label_has="FE1",
tables=local_data.get("Fe_oxidized_model"),
newvalue=wavelength_hi_remote)
GF.reset_specific_at_wavelength(label_has="FE2",
tables=local_data.get("Fe_reduced_model"),
newvalue=wavelength_hi_remote)
sfall_channel = GF.get_amplitudes()
# sources
channel_source_XYZ = flex.vec3_double(len(flux), SIM.xray_source_XYZ[0])
CP = channel_pixels(
wavlen, flux,
channel_source_XYZ) # class interface for multi-wavelength
print("+++++++++++++++++++++++++++++++++++++++ Multiwavelength call")
CH = CP(N=N,
UMAT_nm=UMAT_nm,
Amatrix_rot=Amatrix_rot,
sfall_channel=sfall_channel)
SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal
CH.free_all()
if quick: SIM.to_smv_format(fileout=prefix + "_intimage_001.img")
# rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor
bg = flex.vec2_double([(0, 2.57), (0.0365, 2.58), (0.07, 2.8), (0.12, 5),
(0.162, 8), (0.2, 6.75),
(0.18, 7.32), (0.216, 6.75), (0.236, 6.5),
(0.28, 4.5), (0.3, 4.3), (0.345, 4.36),
(0.436, 3.77), (0.5, 3.17)])
SIM.Fbg_vs_stol = bg
SIM.amorphous_sample_thick_mm = 0.1
SIM.amorphous_density_gcm3 = 1
SIM.amorphous_molecular_weight_Da = 18
SIM.flux = 1e12
SIM.beamsize_mm = 0.003 # square (not user specified)
SIM.exposure_s = 1.0 # multiplies flux x exposure
SIM.add_background()
if quick: SIM.to_smv_format(fileout=prefix + "_intimage_002.img")
# rough approximation to air
bg = flex.vec2_double([(0, 14.1), (0.045, 13.5), (0.174, 8.35),
(0.35, 4.78), (0.5, 4.22)])
SIM.Fbg_vs_stol = bg
#SIM.amorphous_sample_thick_mm = 35 # between beamstop and collimator
SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator
SIM.amorphous_density_gcm3 = 1.2e-3
SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2
print("amorphous_sample_size_mm=", SIM.amorphous_sample_size_mm)
print("amorphous_sample_thick_mm=", SIM.amorphous_sample_thick_mm)
print("amorphous_density_gcm3=", SIM.amorphous_density_gcm3)
print("amorphous_molecular_weight_Da=", SIM.amorphous_molecular_weight_Da)
SIM.add_background()
#apply beamstop mask here
# set this to 0 or -1 to trigger automatic radius. could be very slow with bright images
# settings for CCD
SIM.detector_psf_kernel_radius_pixels = 5
#SIM.detector_psf_fwhm_mm=0.08;
#SIM.detector_psf_type=shapetype.Fiber # rayonix=Fiber, CSPAD=None (or small Gaussian)
SIM.detector_psf_type = shapetype.Unknown # for CSPAD
SIM.detector_psf_fwhm_mm = 0
#SIM.apply_psf()
print("One pixel-->", SIM.raw_pixels[500000])
# at this point we scale the raw pixels so that the output array is on an scale from 0 to 50000.
# that is the default behavior (intfile_scale<=0), otherwise it applies intfile_scale as a multiplier on an abs scale.
if quick: SIM.to_smv_format(fileout=prefix + "_intimage_003.img")
print("quantum_gain=",
SIM.quantum_gain) #defaults to 1. converts photons to ADU
print("adc_offset_adu=", SIM.adc_offset_adu)
print("detector_calibration_noise_pct=",
SIM.detector_calibration_noise_pct)
print("flicker_noise_pct=", SIM.flicker_noise_pct)
print("readout_noise_adu=", SIM.readout_noise_adu
) # gaussian random number to add to every pixel (0 for PAD)
# apply Poissonion correction, then scale to ADU, then adc_offset.
# should be 10 for most Rayonix, Pilatus should be 0, CSPAD should be 0.
print("detector_psf_type=", SIM.detector_psf_type)
print("detector_psf_fwhm_mm=", SIM.detector_psf_fwhm_mm)
print("detector_psf_kernel_radius_pixels=",
SIM.detector_psf_kernel_radius_pixels)
SIM.add_noise() #converts phtons to ADU.
print("raw_pixels=", SIM.raw_pixels)
extra = "PREFIX=%s;\nRANK=%d;\n" % (prefix, rank)
SIM.to_smv_format_py(fileout=smv_fileout,
intfile_scale=1,
rotmat=True,
extra=extra,
gz=True)
# try to write as CBF
if False:
import dxtbx
from dxtbx.format.FormatCBFMiniPilatus import FormatCBFMiniPilatus
img = dxtbx.load(prefix + ".img")
print(img)
FormatCBFMiniPilatus.as_file(detector=img.get_detector(),
beam=img.get_beam(),
gonio=img.get_goniometer(),
scan=img.get_scan(),
data=img.get_raw_data(),
path=prefix + ".cbf")
SIM.free_all()
def run_sim2smv(Nshot_max, odir, prefix, rank, n_jobs, save_bragg=False,
save_smv=True, save_h5 =False, return_pixels=False):
from six.moves import range, StringIO
from six.moves import cPickle as pickle
from cxid9114.sim import sim_utils
import os
import h5py
import math
import sys
import numpy as np
from IPython import embed
from cxid9114.bigsim.bigsim_geom import DET,BEAM
import scitbx
from scitbx.array_family import flex
from scitbx.matrix import sqr,col
from simtbx.nanoBragg import shapetype
from simtbx.nanoBragg import nanoBragg
import libtbx.load_env # possibly implicit
from libtbx.development.timers import Profiler
from cctbx import crystal,crystal_orientation
from LS49.sim.step4_pad import microcrystal
from cxid9114 import utils
from cxid9114.parameters import ENERGY_CONV, ENERGY_HIGH, ENERGY_LOW
from cxid9114.bigsim import sim_spectra
odir_j = os.path.join( odir, "job%d" % rank)
if not os.path.exists(odir_j):
os.makedirs(odir_j)
add_noise = False
add_background = False
overwrite = True #$False
sample_thick_mm = 0.005 # 50 micron GDVN nozzle makes a ~5ish micron jet
air_thick_mm =0 # mostly vacuum, maybe helium layer of 1 micron
flux_ave=2e11
add_spots_algorithm="cuda"
big_data = "." # directory location for reference files
detpixels_slowfast = (1800,1800)
pixsize_mm=0.11
distance_mm = 125
offset_adu=30
mos_spread_deg=0.015
mos_doms=1000
beam_size_mm=0.001
exposure_s=1
use_microcrystal=True #False
Ncells_abc=(120,120,120)
Deff_A = 2200
length_um = 2.2
timelog = False
background = utils.open_flex("background")
crystal = microcrystal(Deff_A = Deff_A, length_um = length_um,
beam_diameter_um = beam_size_mm*1000, verbose=False)
spec_file = h5py.File("simMe_data_run62.h5", "r")
spec_data = spec_file["hist_spec"]
Umat_data = spec_file["Umats"]
en_chans = spec_file["energy_bins"][()]
ilow = np.abs(en_chans - ENERGY_LOW).argmin()
ihigh = np.abs(en_chans - ENERGY_HIGH).argmin()
wave_chans = ENERGY_CONV/en_chans
sfall_main = sim_spectra.load_spectra("test_sfall.h5")
Nshot = spec_data.shape[0]
idx_range = np.array_split(np.arange(Nshot), n_jobs)
Nshot_per_job = len(idx_range[rank])
if Nshot_max > 0 :
Nshot_per_job = min( Nshot_max, Nshot_per_job)
print ("Job %d: Simulating %d shots" % (rank, Nshot_per_job))
istart = idx_range[rank][0]
istop = istart + Nshot_per_job
smi_stride = 10
for idx in range( istart, istop):
print ("<><><><><><><><><><><><><><>")
print ("Job %d; Image %d (%d - %d)" % (rank, idx+1, istart, istop))
print ("<><><><><><><><><><><><><><>")
smv_fileout = os.path.join( odir_j, prefix % idx + ".img")
h5_fileout = smv_fileout + ".h5"
if os.path.exists(smv_fileout) and not overwrite and save_smv:
print("Shot %s exists: moving on" % smv_fileout)
continue
if os.path.exists(h5_fileout) and not overwrite and save_h5:
print("Shot %s exists: moving on" % h5_fileout)
continue
if (rank==0 and idx % smi_stride==0):
print("GPU status")
os.system("nvidia-smi")
print("\n\n")
print("CPU memory usage")
mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'"""
os.system(mem_usg)
spec = spec_data[2]
rotation = sqr(Umat_data[2])
wavelength_A = np.mean(wave_chans)
spectra = iter([(wave_chans, spec, wavelength_A)])
direct_algo_res_limit = 1.7
wavlen, flux, wavelength_A = next(spectra) # list of lambdas, list of fluxes, average wavelength
assert wavelength_A > 0
assert (len(wavlen)==len(flux)==len(sfall_main))
N = crystal.number_of_cells(sfall_main[0].unit_cell())
if use_mcrocrystal:
Ncells_abc = (N,N,N)
flux *= 0
flux[ilow] = 1e12
flux[ihigh]=1e12
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(mean=0, sigma=mos_spread_deg * math.pi/180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(mos_doms)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) )
Amatrix_rot = (rotation * sqr(sfall_main[0].unit_cell().orthogonalization_matrix())).transpose()
#SIM = nanoBragg(detpixels_slowfast=detpixels_slowfast,
# pixel_size_mm=pixsize_mm,Ncells_abc=Ncells_abc,
# wavelength_A=wavelength_A,verbose=verbose)
SIM = nanoBragg(detector=DET, beam=BEAM, panel_id=0,verbose=verbose)
SIM.adc_offset_adu = offset_adu # Do not offset by 40
SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = mos_doms
SIM.distance_mm=distance_mm
SIM.set_mosaic_blocks(UMAT_nm)
SIM.seed = 1
SIM.oversample=1
SIM.wavelength_A = wavelength_A
SIM.polarization=1
SIM.default_F=0
SIM.Fhkl=sfall_main[0].amplitudes()
SIM.progress_meter=False
SIM.flux=flux_ave
SIM.exposure_s=exposure_s
SIM.beamsize_mm=beam_size_mm
SIM.Ncells_abc=Ncells_abc
SIM.xtal_shape=shapetype.Gauss
idxpath = "try3_idx2/job0/dump_0_data.pkl"
Amat = sim_utils.Amatrix_dials2nanoBragg(utils.open_flex(idxpath)["crystalAB"])
#SIM.Umatrix = rotation.elems
#SIM.unit_cell_Adeg = sfall_main[0].unit_cell()
#SIM2 = nanoBragg(detector=DET, beam=BEAM, panel_id=0,verbose=verbose)
#SIM2.adc_offset_adu = offset_adu # Do not offset by 40
#SIM2.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev
#SIM2.mosaic_domains = mos_doms
#SIM2.distance_mm=distance_mm
#SIM2.set_mosaic_blocks(UMAT_nm)
#SIM2.seed = 1
#SIM2.oversample=1
#SIM2.wavelength_A = wavelength_A
#SIM2.polarization=1
#SIM2.default_F=0
#SIM2.Fhkl=sfall_main[0].amplitudes()
#SIM2.progress_meter=False
#SIM2.flux=flux_ave
#SIM2.exposure_s=exposure_s
#SIM2.beamsize_mm=beam_size_mm
#SIM2.Ncells_abc=Ncells_abc
#SIM2.xtal_shape=shapetype.Gauss
#SIM2.Umatrix = rotation.elems
#SIM2.unit_cell_Adeg = sfall_main[0].unit_cell()
#SIM.Amatrix_RUB = Amatrix_rot
#SIM.Amatrix = Amatrix_rot.inverse()
raw_pixel_sum = flex.double(len(SIM.raw_pixels))
Nflux = len(flux)
print("Beginning the loop")
for x in range(Nflux):
if x % 10==0:
print("+++++++++++++++++++++++++++++++++++++++ Wavelength %d / %d" % (x+1, Nflux) , end="\r")
if flux[x] ==0:
continue
#SIM.Amatrix = Amatrix_rot.inverse()
print (SIM.Ncells_abc)
SIM.wavelength_A=wavlen[x]
SIM.flux=flux[x]
SIM.Fhkl=sfall_main[x].amplitudes()
SIM.Amatrix = Amat#Amatrix_rot.inverse()
#sim_utils.compare_sims(SIM, SIM2)
#SIM.Ncells_abc=Ncells_abc
#SIM.adc_offset_adu = offset_adu
#SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev
#SIM.mosaic_domains = mos_doms #
#SIM.distance_mm=distance_mm
#SIM.set_mosaic_blocks(UMAT_nm)
#SIM.seed = 1
#SIM.polarization=1
#SIM.default_F=0
#SIM.xtal_shape=shapetype.Gauss
#SIM.progress_meter=False
#SIM.exposure_s = exposure_s
#SIM.beamsize_mm=beam_size_mm
SIM.timelog=timelog
SIM.device_Id=rank
SIM.raw_pixels *= 0 # just in case!
SIM.add_nanoBragg_spots_cuda()
if use_microcrystal:
raw_pixel_sum += SIM.raw_pixels * crystal.domains_per_crystal
print()
SIM.raw_pixels = raw_pixel_sum
if add_background:
SIM.raw_pixels = SIM.raw_pixels + background
SIM.detector_psf_kernel_radius_pixels=5;
#SIM.detector_psf_fwhm_mm=0.08;
#SIM.detector_psf_type=shapetype.Fiber # rayonix=Fiber, CSPAD=None (or small Gaussian)
SIM.detector_psf_type=shapetype.Unknown # for CSPAD
SIM.detector_psf_fwhm_mm=0
SIM.quantum_gain = 28.
#SIM.apply_psf()
if add_noise:
SIM.add_noise() #converts phtons to ADU.
extra = "PREFIX=%s;\nRANK=%d;\n"%(prefix,rank)
out = SIM.raw_pixels.as_numpy_array()
if save_smv:
SIM.to_smv_format_py(fileout=smv_fileout,intfile_scale=1,rotmat=True,extra=extra,gz=True)
elif save_h5:
f = h5py.File(h5_fileout, "w")
f.create_dataset("bigsim_d9114",
data=SIM.raw_pixels.as_numpy_array().astype(np.uint16).reshape(detpixels_slowfast),
compression="lzf")
f.close()
if npout is not None:
np.save(npout, SIM.raw_pixels.as_numpy_array())
SIM.free_all()
def perform_simulations(self, spectrum, crystal, tophat_spectrum=True):
#borrow code from util_partiality.
#def run_sim2smv(ROI,prefix,crystal,spectra,rotation,rank,tophat_spectrum=True,quick=False):
direct_algo_res_limit = 1.7
self.wavlen, self.flux, self.wavelength_A = next(
spectrum) # list of lambdas, list of fluxes, average wavelength
if tophat_spectrum:
sum_flux = flex.sum(self.flux)
ave_flux = sum_flux / 60. # 60 energy channels
for ix in range(len(self.wavlen)):
energy = 12398.425 / self.wavlen[ix]
if energy >= 7090 and energy <= 7150:
self.flux[ix] = ave_flux
else:
self.flux[ix] = 0.
# use crystal structure to initialize Fhkl array
self.sfall_main.show_summary(prefix="Amplitudes used ")
N = crystal.number_of_cells(self.sfall_main.unit_cell())
self.crystal = crystal # delete later
self.N = N # delete later
SIM = nanoBragg(
detpixels_slowfast=(3000, 3000),
pixel_size_mm=0.11,
Ncells_abc=(N, N, N),
# workaround for problem with wavelength array, specify it separately in constructor.
wavelength_A=self.wavelength_A,
verbose=0)
self.SIM = SIM
SIM.adc_offset_adu = 10 # Do not offset by 40
SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = 50 # mosaic_domains setter must come after mosaic_spread_deg setter
#manuscript says 200, June 15 abc_cov was done with 50
SIM.distance_mm = 141.7
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(
mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(SIM.mosaic_domains)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append(
site.axis_and_angle_as_r3_rotation_matrix(m, deg=False))
SIM.set_mosaic_blocks(UMAT_nm)
self.UMAT_nm = UMAT_nm # delete later
# get same noise each time this test is run
SIM.seed = 1
SIM.oversample = 1
SIM.wavelength_A = self.wavelength_A
SIM.polarization = 1
# this will become F000, marking the beam center
SIM.default_F = 0
#SIM.Fhkl=self.sfall_main # no it turns out we don't use these calculations for abc_coverage
# use local file with (open(something,"wb")) as F:
with (open("confirm_sfall_P1_7122_amplitudes.pickle", "rb")) as F:
sfall_P1_7122_amplitudes = pickle.load(F)
SIM.Fhkl = sfall_P1_7122_amplitudes
SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian
SIM.progress_meter = False
SIM.show_params()
# flux is always in photons/s
SIM.flux = 1e12
SIM.exposure_s = 1.0 # so total fluence is e12
# assumes round beam
SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense
temp = SIM.Ncells_abc
print("Ncells_abc=", SIM.Ncells_abc)
SIM.Ncells_abc = temp
def __init__(self, model_parameterisations, sigma):
"""model_parameterisations is a list of CrystalUnitCellParameterisations
sigma is a sequence of 6 elements giving the 'sigma' for each of the
unit cell parameters, from which weights for the residuals will be
calculated. Values of zero in sigma will remove the restraint for the
cell parameter at that position"""
self._xlucp = model_parameterisations
self._nxls = len(model_parameterisations)
# common factors used in gradient calculations
self._meangradfac = 1./self._nxls
self._gradfac = (1. - self._meangradfac)
self._weights = []
# initially want to calculate all gradients
self._sel = [True] * 6
# identify any cell dimensions constrained to be equal. If any are and a
# restraint has been requested for that cell dimension, remove the restraint
# for all crystals and warn in the log
msg = ('Unit cell similarity restraints were requested for both the '
'{0} and {1} dimensions, however for the crystal in experiment '
'{2} these are constrained to be equal. Only the strongest '
'of these restraints will be retained for all crystals in '
'the restrained group.')
for ixl, xlucp in enumerate(self._xlucp):
B = xlucp.get_state()
dB_dp = flex.mat3_double(xlucp.get_ds_dp())
ccg = CalculateCellGradients(B, dB_dp)
grads = [ccg.da_dp(), ccg.db_dp(), ccg.dc_dp()]
a, b, c, aa, bb, cc = xlucp.get_model().get_unit_cell().parameters()
if abs(a - b) < 1e-10:
grad_diff = [abs(e1 - e2) for (e1, e2) in zip(grads[0], grads[1])]
if max(grad_diff) < 1e-10:
# a and b are equal for this crystal, therefore keep only the
# strongest requested restraint
if sigma[0] > 0.0 and sigma[1] > 0.0:
warning(msg.format('a', 'b', xlucp.get_experiment_ids()[0]))
strong, weak = sorted([sigma[0], sigma[1]])
sigma[0] = strong
sigma[1] = 0.0
if abs(a - c) < 1e-10:
grad_diff = [abs(e1 - e2) for (e1, e2) in zip(grads[0], grads[2])]
if max(grad_diff) < 1e-10:
# a and c are equal for this crystal, therefore keep only the
# strongest requested restraint
if sigma[0] > 0.0 and sigma[2] > 0.0:
warning(msg.format('a', 'c', xlucp.get_experiment_ids()[0]))
strong, weak = sorted([sigma[0], sigma[2]])
sigma[0] = strong
sigma[2] = 0.0
if abs(b - c) < 1e-10:
grad_diff = [abs(e1 - e2) for (e1, e2) in zip(grads[1], grads[2])]
if max(grad_diff) < 1e-10:
# b and c are equal for this crystal, therefore keep only the
# strongest requested restraint
if sigma[1] > 0.0 and sigma[2] > 0.0:
strong, weak = sorted([sigma[1], sigma[2]])
sigma[1] = strong
sigma[2] = 0.0
# A gradient of zero indicates that cell parameter is constrained and thus
# to be ignored in restraints
#_sigma = []
msg = ('Unit cell similarity restraints were requested for the {0} '
'parameter, however for the crystal in experiment {1}, {0} is '
'constrained. This restraint will be removed for all crystals in '
'the restrained group.')
for i, (grad, pname) in enumerate(zip(grads, ['a', 'b', 'c', 'alpha', 'beta', 'gamma'])):
tst = [abs(g) <= 1.e-10 for g in grad]
if all(tst):
# this parameter is constrained, so remove any requested restraints
# at this position
if sigma[i] > 0.0:
warning(msg.format(xlucp.get_experiment_ids()[0], pname[i]))
sigma[i] = 0.0
# set the selection for gradient calculations to the unconstrained parameters
self._sel = [s > 0.0 for s in sigma]
self.nrestraints_per_cell = self._sel.count(True)
# repeat the weights for each unit cell being restrained
weights = [1./s**2 for s in sigma if s > 0.0]
weights = [flex.double(self._nxls, w) for w in weights]
self._weights = weights[0]
for w in weights[1:]:
self._weights.extend(w)
return
def run_sim2smv(prefix,
crystal,
spectra,
rotation,
rank,
gpu_channels_singleton,
params,
quick=False,
save_bragg=False,
sfall_channels=None):
smv_fileout = prefix + ".img"
burst_buffer_expand_dir = os.path.expandvars(params.logger.outdir)
burst_buffer_fileout = os.path.join(burst_buffer_expand_dir, smv_fileout)
reference_fileout = os.path.join(".", smv_fileout)
if not quick:
if not write_safe(reference_fileout):
print("File %s already exists, skipping in rank %d" %
(reference_fileout, rank))
return
direct_algo_res_limit = 1.7
wavlen, flux, wavelength_A = next(
spectra) # list of lambdas, list of fluxes, average wavelength
assert wavelength_A > 0
# os.system("nvidia-smi") # printout might severely impact performance
# use crystal structure to initialize Fhkl array
N = crystal.number_of_cells(sfall_channels[0].unit_cell())
#SIM = nanoBragg(detpixels_slowfast=(2000,2000),pixel_size_mm=0.11,Ncells_abc=(5,5,5),verbose=0)
SIM = nanoBragg(
detpixels_slowfast=(3840, 3840),
pixel_size_mm=0.088,
Ncells_abc=(N, N, N),
# workaround for problem with wavelength array, specify it separately in constructor.
wavelength_A=wavelength_A,
verbose=0)
SIM.adc_offset_adu = 0 # Do not offset by 40
#SIM.adc_offset_adu = 10 # Do not offset by 40
SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev
# mosaic_domains setter MUST come after mosaic_spread_deg setter
SIM.mosaic_domains = int(os.environ.get("MOS_DOM", "25"))
print("MOSAIC", SIM.mosaic_domains)
SIM.distance_mm = 141.7
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(mean=0,
sigma=SIM.mosaic_spread_deg *
math.pi / 180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(SIM.mosaic_domains)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False))
SIM.set_mosaic_blocks(UMAT_nm)
if params.attenuation:
SIM.detector_thick_mm = 0.032 # = 0 for Rayonix
SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD
SIM.detector_attenuation_length_mm = 0.017 # default is silicon
# get same noise each time this test is run
SIM.seed = 1
SIM.oversample = 1
SIM.wavelength_A = wavelength_A
SIM.polarization = 1
# this will become F000, marking the beam center
SIM.default_F = 0
SIM.Fhkl = sfall_channels[0] # instead of sfall_main
Amatrix_rot = (rotation * sqr(
sfall_channels[0].unit_cell().orthogonalization_matrix())).transpose()
SIM.Amatrix_RUB = Amatrix_rot
#workaround for failing init_cell, use custom written Amatrix setter
print("unit_cell_Adeg=", SIM.unit_cell_Adeg)
print("unit_cell_tuple=", SIM.unit_cell_tuple)
Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM
from cctbx import crystal_orientation
Ori = crystal_orientation.crystal_orientation(
Amat, crystal_orientation.basis_type.reciprocal)
# fastest option, least realistic
SIM.xtal_shape = shapetype.Gauss_argchk # both crystal & RLP are Gaussian
# only really useful for long runs
SIM.progress_meter = False
# prints out value of one pixel only. will not render full image!
# flux is always in photons/s
SIM.flux = 1e12
SIM.exposure_s = 1.0 # so total fluence is e12
# assumes round beam
SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense
temp = SIM.Ncells_abc
SIM.Ncells_abc = temp
# rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor
water_bg = flex.vec2_double([(0, 2.57), (0.0365, 2.58), (0.07, 2.8),
(0.12, 5), (0.162, 8), (0.18, 7.32),
(0.2, 6.75), (0.216, 6.75), (0.236, 6.5),
(0.28, 4.5), (0.3, 4.3), (0.345, 4.36),
(0.436, 3.77), (0.5, 3.17)])
assert [a[0] for a in water_bg] == sorted([a[0] for a in water_bg])
# rough approximation to air
air_bg = flex.vec2_double([(0, 14.1), (0.045, 13.5), (0.174, 8.35),
(0.35, 4.78), (0.5, 4.22)])
assert [a[0] for a in air_bg] == sorted([a[0] for a in air_bg])
# simulated crystal is only 125 unit cells (25 nm wide)
# amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume)
SIM.raw_pixels *= crystal.domains_per_crystal
# must calculate the correct scale!
QQ = Profiler("nanoBragg Bragg spots rank %d" % (rank))
if True:
#something new
devices_per_node = int(os.environ["DEVICES_PER_NODE"])
SIM.device_Id = rank % devices_per_node
assert gpu_channels_singleton.get_deviceID() == SIM.device_Id
if gpu_channels_singleton.get_nchannels() == 0: # if uninitialized
P = Profiler("Initialize the channels singleton rank %d" % (rank))
for x in range(len(flux)):
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, sfall_channels[x].indices(), sfall_channels[x].data())
del P
import time
print("datetime for channels singleton rank %d" % (rank),
time.time())
exascale_api = get_exascale("exascale_api", params.context)
gpud = get_exascale("gpu_detector", params.context)
gpu_simulation = exascale_api(nanoBragg=SIM)
gpu_simulation.allocate()
gpu_detector = gpud(deviceId=SIM.device_Id, nanoBragg=SIM)
gpu_detector.each_image_allocate()
# loop over energies
for x in range(len(flux)):
P = Profiler("USE_EXASCALE_API nanoBragg Python and C++ rank %d" %
(rank))
print("USE_EXASCALE_API+++++++++++++++++++++++ Wavelength", x)
# from channel_pixels function
SIM.wavelength_A = wavlen[x]
SIM.flux = flux[x]
gpu_simulation.add_energy_channel_from_gpu_amplitudes(
x, gpu_channels_singleton, gpu_detector)
del P
gpu_detector.scale_in_place(
crystal.domains_per_crystal) # apply scale directly on GPU
SIM.wavelength_A = wavelength_A # return to canonical energy for subsequent background
if add_background_algorithm == "cuda":
QQ = Profiler("nanoBragg background rank %d" % (rank))
SIM.Fbg_vs_stol = water_bg
SIM.amorphous_sample_thick_mm = 0.1
SIM.amorphous_density_gcm3 = 1
SIM.amorphous_molecular_weight_Da = 18
SIM.flux = 1e12
SIM.beamsize_mm = 0.003 # square (not user specified)
SIM.exposure_s = 1.0 # multiplies flux x exposure
gpu_simulation.add_background(gpu_detector)
SIM.Fbg_vs_stol = air_bg
SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator
SIM.amorphous_density_gcm3 = 1.2e-3
SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2
gpu_simulation.add_background(gpu_detector)
# deallocate GPU arrays
gpu_detector.write_raw_pixels(SIM) # updates SIM.raw_pixels from GPU
gpu_detector.each_image_free()
SIM.Amatrix_RUB = Amatrix_rot # return to canonical orientation
del QQ
if add_background_algorithm in ["jh", "sort_stable"]:
QQ = Profiler("nanoBragg background rank %d" % (rank))
SIM.Fbg_vs_stol = water_bg
SIM.amorphous_sample_thick_mm = 0.1
SIM.amorphous_density_gcm3 = 1
SIM.amorphous_molecular_weight_Da = 18
SIM.flux = 1e12
SIM.beamsize_mm = 0.003 # square (not user specified)
SIM.exposure_s = 1.0 # multiplies flux x exposure
SIM.add_background(
sort_stable=(add_background_algorithm == "sort_stable"))
SIM.Fbg_vs_stol = air_bg
SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator
SIM.amorphous_density_gcm3 = 1.2e-3
SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2
SIM.add_background(
sort_stable=(add_background_algorithm == "sort_stable"))
del QQ
if params.psf:
SIM.detector_psf_kernel_radius_pixels = 10
SIM.detector_psf_type = shapetype.Fiber # for Rayonix
SIM.detector_psf_fwhm_mm = 0.08
#SIM.apply_psf() # the actual application is called within the C++ SIM.add_noise()
else:
#SIM.detector_psf_kernel_radius_pixels=5;
SIM.detector_psf_type = shapetype.Unknown # for CSPAD
SIM.detector_psf_fwhm_mm = 0
if params.noise:
SIM.adc_offset_adu = 10 # Do not offset by 40
SIM.detector_calibration_noise_pct = 1.0
SIM.readout_noise_adu = 1.
QQ = Profiler("nanoBragg noise rank %d" % (rank))
if params.noise or params.psf:
from LS49.sim.step6_pad import estimate_gain
print("quantum_gain=",
SIM.quantum_gain) #defaults to 1. converts photons to ADU
print("adc_offset_adu=", SIM.adc_offset_adu)
print("detector_calibration_noise_pct=",
SIM.detector_calibration_noise_pct)
print("flicker_noise_pct=", SIM.flicker_noise_pct)
print("readout_noise_adu=", SIM.readout_noise_adu
) # gaussian random number to add to every pixel (0 for PAD)
# apply Poissonion correction, then scale to ADU, then adc_offset.
# should be 10 for most Rayonix, Pilatus should be 0, CSPAD should be 0.
print("detector_psf_type=", SIM.detector_psf_type)
print("detector_psf_fwhm_mm=", SIM.detector_psf_fwhm_mm)
print("detector_psf_kernel_radius_pixels=",
SIM.detector_psf_kernel_radius_pixels)
#estimate_gain(SIM.raw_pixels,offset=0)
SIM.add_noise() #converts photons to ADU.
#estimate_gain(SIM.raw_pixels,offset=SIM.adc_offset_adu,algorithm="slow")
#estimate_gain(SIM.raw_pixels,offset=SIM.adc_offset_adu,algorithm="kabsch")
del QQ
print("FULLY")
print(burst_buffer_expand_dir, params.logger.outdir)
print(burst_buffer_fileout, smv_fileout)
extra = "PREFIX=%s;\nRANK=%d;\n" % (prefix, rank)
SIM.to_smv_format_py(fileout=burst_buffer_fileout,
intfile_scale=1,
rotmat=True,
extra=extra,
gz=True)
SIM.free_all()
def __init__(self, rotation,sfall_main, N,
mosaic_domains=25,
mosaic_spread_deg=0.05,
SEED=1,
randomize=False):
"""
:param rotation:
:param sfall_main:
:param N:
:param mosaic_domains:
:param mosaic_spread_deg:
:param SEED:
:param randomize:
"""
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(
mean=0,
sigma=mosaic_spread_deg * math.pi/180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(mosaic_domains)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) )
Amatrix_rot = (rotation * sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose()
self.SIM = nanoBragg(
detpixels_slowfast=(3000,3000),
pixel_size_mm=0.11,
Ncells_abc=(N, N, N),
wavelength_A=1, # default we will update later@
verbose=0)
self.SIM.seed = SEED
self.SIM.adc_offset_adu = 10 # Do not offset by 40
self.SIM.mosaic_spread_deg = mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev
self.SIM.mosaic_domains = mosaic_domains # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image
# mosaic_domains setter must come after mosaic_spread_deg setter
self.SIM.distance_mm=141.7
self.SIM.set_mosaic_blocks(UMAT_nm)
self.SIM.oversample=1
self.SIM.polarization=1
self.SIM.default_F=1e5 # TODO: in the future we will init the energy dependent F_HKL here
#self.SIM.Fhkl = energy_independent_F
self.SIM.Amatrix_RUB = Amatrix_rot
self.SIM.xtal_shape=shapetype.Gauss # both crystal & RLP are Gaussian
self.SIM.progress_meter=False
self.SIM.exposure_s=1.0 # so total fluence is e12
self.SIM.beamsize_mm=0.003 #cannot make this 3 microns; spots are too intense
if randomize:
self.SIM.random_orientation()
temp=self.SIM.Ncells_abc
print("Ncells_abc=",self.SIM.Ncells_abc)
self.SIM.Ncells_abc=temp
# FIXME: add the CUDA init script here
#initialize_GPU_variables()
self.raw_pixels = self.SIM.raw_pixels # FIXME: this will be on GPU
def run_sim2smv(img_prefix=None,
simparams=None,
pdb_lines=None,
crystal=None,
spectra=None,
rotation=None,
rank=None,
fsave=None,
sfall_cluster=None,
quick=False):
smv_fileout = fsave
direct_algo_res_limit = simparams.direct_algo_res_limit
wavlen, flux, real_wavelength_A = next(
spectra) # list of lambdas, list of fluxes, average wavelength
real_flux = flex.sum(flux)
assert real_wavelength_A > 0
# print(rank, " ## real_wavelength_A/real_flux = ", real_wavelength_A, real_flux*1.0/simparams.flux)
if quick:
wavlen = flex.double([real_wavelength_A])
flux = flex.double([real_flux])
# GF = gen_fmodel(resolution=simparams.direct_algo_res_limit,pdb_text=pdb_lines,algorithm=simparams.fmodel_algorithm,wavelength=real_wavelength_A)
# GF.set_k_sol(simparams.k_sol)
# GF.make_P1_primitive()
sfall_main = sfall_cluster["main"] #GF.get_amplitudes()
# use crystal structure to initialize Fhkl array
# sfall_main.show_summary(prefix = "Amplitudes used ")
N = crystal.number_of_cells(sfall_main.unit_cell())
#print("## number of N = ", N)
SIM = nanoBragg(detpixels_slowfast=(simparams.detector_size_ny,simparams.detector_size_nx),pixel_size_mm=simparams.pixel_size_mm,\
Ncells_abc=(N,N,N),wavelength_A=real_wavelength_A,verbose=0)
# workaround for problem with wavelength array, specify it separately in constructor.
# SIM.adc_offset_adu = 0 # Do not offset by 40
SIM.adc_offset_adu = 10 # Do not offset by 40
SIM.seed = 0
# SIM.randomize_orientation()
SIM.mosaic_spread_deg = simparams.mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = simparams.mosaic_domains # 77 seconds.
SIM.distance_mm = simparams.distance_mm
## setup the mosaicity
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(mean=0,
sigma=SIM.mosaic_spread_deg *
math.pi / 180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(SIM.mosaic_domains)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False))
SIM.set_mosaic_blocks(UMAT_nm)
######################
SIM.beamcenter_convention = convention.ADXV
SIM.beam_center_mm = (simparams.beam_center_x_mm,
simparams.beam_center_y_mm) # 95.975 96.855
######################
# get same noise each time this test is run
SIM.seed = 0
SIM.oversample = simparams.oversample
SIM.wavelength_A = real_wavelength_A
SIM.polarization = simparams.polarization
# this will become F000, marking the beam center
SIM.default_F = simparams.default_F
#SIM.missets_deg= (10,20,30)
SIM.Fhkl = sfall_main
Amatrix_rot = (
rotation *
sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose()
SIM.Amatrix_RUB = Amatrix_rot
#workaround for failing init_cell, use custom written Amatrix setter
# print("## inside run_sim2smv, Amat_rot = ", Amatrix_rot)
Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM
Ori = crystal_orientation.crystal_orientation(
Amat, crystal_orientation.basis_type.reciprocal)
print(fsave, "Amatrix_rot", Amatrix_rot)
print(fsave, "Amat", Amat)
print(fsave, "Ori", Ori)
print(fsave, "rotation", rotation)
print(fsave, "sqr(sfall_main.unit_cell().orthogonalization_matrix())",
sqr(sfall_main.unit_cell().orthogonalization_matrix()))
SIM.free_all()
def __init__(self, model_parameterisations, sigma):
"""model_parameterisations is a list of CrystalUnitCellParameterisations
sigma is a sequence of 6 elements giving the 'sigma' for each of the
unit cell parameters, from which weights for the residuals will be
calculated. Values of zero in sigma will remove the restraint for the
cell parameter at that position"""
self._xlucp = model_parameterisations
self._nxls = len(model_parameterisations)
# common factors used in gradient calculations
self._meangradfac = 1.0 / self._nxls
self._gradfac = 1.0 - self._meangradfac
self._weights = []
# initially want to calculate all gradients
self._sel = [True] * 6
# identify any cell dimensions constrained to be equal. If any are and a
# restraint has been requested for that cell dimension, remove the restraint
# for all crystals and warn in the log
msg = ("Unit cell similarity restraints were requested for both the "
"{0} and {1} dimensions, however for the crystal in experiment "
"{2} these are constrained to be equal. Only the strongest "
"of these restraints will be retained for all crystals in "
"the restrained group.")
for ixl, xlucp in enumerate(self._xlucp):
B = xlucp.get_state()
dB_dp = flex.mat3_double(xlucp.get_ds_dp())
ccg = CalculateCellGradients(B, dB_dp)
grads = [
ccg.da_dp(),
ccg.db_dp(),
ccg.dc_dp(),
ccg.daa_dp(),
ccg.dbb_dp(),
ccg.dcc_dp(),
]
a, b, c, aa, bb, cc = xlucp.get_model().get_unit_cell().parameters(
)
if abs(a - b) < 1e-10:
grad_diff = [
abs(e1 - e2) for (e1, e2) in zip(grads[0], grads[1])
]
if max(grad_diff) < 1e-10:
# a and b are equal for this crystal, therefore keep only the
# strongest requested restraint
if sigma[0] > 0.0 and sigma[1] > 0.0:
logger.debug(
msg.format("a", "b",
xlucp.get_experiment_ids()[0]))
strong, weak = sorted([sigma[0], sigma[1]])
sigma[0] = strong
sigma[1] = 0.0
if abs(a - c) < 1e-10:
grad_diff = [
abs(e1 - e2) for (e1, e2) in zip(grads[0], grads[2])
]
if max(grad_diff) < 1e-10:
# a and c are equal for this crystal, therefore keep only the
# strongest requested restraint
if sigma[0] > 0.0 and sigma[2] > 0.0:
logger.debug(
msg.format("a", "c",
xlucp.get_experiment_ids()[0]))
strong, weak = sorted([sigma[0], sigma[2]])
sigma[0] = strong
sigma[2] = 0.0
if abs(b - c) < 1e-10:
grad_diff = [
abs(e1 - e2) for (e1, e2) in zip(grads[1], grads[2])
]
if max(grad_diff) < 1e-10:
# b and c are equal for this crystal, therefore keep only the
# strongest requested restraint
if sigma[1] > 0.0 and sigma[2] > 0.0:
logger.debug(
msg.format("b", "c",
xlucp.get_experiment_ids()[0]))
strong, weak = sorted([sigma[1], sigma[2]])
sigma[1] = strong
sigma[2] = 0.0
# A gradient of zero indicates that cell parameter is constrained and thus
# to be ignored in restraints
# _sigma = []
msg = (
"Unit cell similarity restraints were requested for the {0} "
"parameter, however for the crystal in experiment {1}, {0} is "
"constrained. This restraint will be removed for all crystals in "
"the restrained group.")
for i, (grad, pname) in enumerate(
zip(grads, ["a", "b", "c", "alpha", "beta", "gamma"])):
tst = (abs(g) <= 1.0e-10 for g in grad)
if all(tst):
# this parameter is constrained, so remove any requested restraints
# at this position
if sigma[i] > 0.0:
logger.debug(
msg.format(pname,
xlucp.get_experiment_ids()[0]))
sigma[i] = 0.0
# set the selection for gradient calculations to the unconstrained parameters
self._sel = [s > 0.0 for s in sigma]
self.nrestraints_per_cell = self._sel.count(True)
# repeat the weights for each unit cell being restrained
weights = [1.0 / s**2 for s in sigma if s > 0.0]
weights = [flex.double(self._nxls, w) for w in weights]
self._weights = weights[0]
for w in weights[1:]:
self._weights.extend(w)
def _crystal_properties(self):
if self.crystal is None:
return
self.D.xtal_shape = self.crystal.xtal_shape
self.update_Fhkl_tuple()
## TODO: am I unnecessary?
#self.D.unit_cell_tuple = self.crystal.dxtbx_crystal.get_unit_cell().parameters()
if self.using_diffBragg_spots:
self.D.Omatrix = self.crystal.Omatrix
self.D.Bmatrix = self.crystal.dxtbx_crystal.get_B() #
self.D.Umatrix = self.crystal.dxtbx_crystal.get_U()
if self.crystal.isotropic_ncells:
self.D.Ncells_abc = self.crystal.Ncells_abc[0]
else:
self.D.Ncells_abc_aniso = self.crystal.Ncells_abc
if self.crystal.Ncells_def is not None:
self.D.Ncells_def = self.crystal.Ncells_def
if self.crystal.anisotropic_mos_spread_deg is not None:
mosaicity = self.crystal.anisotropic_mos_spread_deg
self.Umats_method = 3 if 3 == len(mosaicity) else 4
crystal = self.crystal.dxtbx_crystal
else:
mosaicity = self.crystal.mos_spread_deg
self.Umats_method = 2
crystal = None
self.update_umats(mosaicity, self.crystal.n_mos_domains, crystal)
else:
# umat implementation for the exascale api
self.D.xtal_shape = self.crystal.xtal_shape
self.update_Fhkl_tuple()
self.D.Amatrix = Amatrix_dials2nanoBragg(
self.crystal.dxtbx_crystal)
#Nabc = tuple([int(round(x)) for x in self.crystal.Ncells_abc])
Nabc = self.crystal.Ncells_abc
if len(Nabc) == 1:
Nabc = Nabc[0], Nabc[0], Nabc[0]
self.D.Ncells_abc = Nabc
if self.umat_maker is None:
# initialize the parameterized distribution
assert self.crystal.n_mos_domains % 2 == 0
from simtbx.nanoBragg.tst_anisotropic_mosaicity import AnisoUmats as AnisoUmats_exa
self.umat_maker = AnisoUmats_exa(num_random_samples=2 *
self.crystal.n_mos_domains)
if self.crystal.anisotropic_mos_spread_deg is None:
# isotropic case
self.D.mosaic_spread_deg = self.crystal.mos_spread_deg
self.D.mosaic_domains = self.crystal.n_mos_domains
if self.Umats_method == 0: # double_random legacy method, exafel tests
Umats = SimData.Umats(self.crystal.mos_spread_deg,
self.crystal.n_mos_domains,
seed=self.mosaic_seeds[0],
norm_dist_seed=self.mosaic_seeds[1])
elif self.Umats_method == 5: # double_uniform isotropic
Umats, Umats_prime, Umats_dbl_prime = self.umat_maker.generate_isotropic_Umats(
self.crystal.mos_spread_deg, compute_derivs=False)
else:
raise ValueError(
"Invalid method for nanoBragg isotropic case")
#SimData.plot_isotropic_umats(Umats, self.crystal.mos_spread_deg)
else:
# anisotropic case with diagonal elements
eta_a, eta_b, eta_c = self.crystal.anisotropic_mos_spread_deg
eta_tensor = [eta_a, 0, 0, 0, eta_b, 0, 0, 0, eta_c]
Umats, Umats_prime, Umats_dbl_prime = self.umat_maker.generate_Umats(
eta_tensor,
compute_derivs=False,
crystal=self.crystal.dxtbx_crystal,
how=1)
self.exascale_mos_blocks = flex.mat3_double(Umats)
self.D.set_mosaic_blocks(self.exascale_mos_blocks)
def run_sim2smv(prefix,
crystal,
spectra,
rotation,
rank,
gpu_channels_singleton,
params,
quick=False,
save_bragg=False,
sfall_channels=None):
smv_fileout = prefix + ".img"
burst_buffer_expand_dir = os.path.expandvars(params.logger.outdir)
burst_buffer_fileout = os.path.join(burst_buffer_expand_dir, smv_fileout)
reference_fileout = os.path.join(".", smv_fileout)
if not quick:
if not write_safe(reference_fileout):
print("File %s already exists, skipping in rank %d" %
(reference_fileout, rank))
return
direct_algo_res_limit = 1.7
wavlen, flux, wavelength_A = next(
spectra) # list of lambdas, list of fluxes, average wavelength
assert wavelength_A > 0
# use crystal structure to initialize Fhkl array
N = crystal.number_of_cells(sfall_channels[0].unit_cell())
#SIM = nanoBragg(detpixels_slowfast=(2000,2000),pixel_size_mm=0.11,Ncells_abc=(5,5,5),verbose=0)
SIM = nanoBragg(
detpixels_slowfast=(3000, 3000),
pixel_size_mm=0.11,
Ncells_abc=(N, N, N),
# workaround for problem with wavelength array, specify it separately in constructor.
wavelength_A=wavelength_A,
verbose=0)
SIM.adc_offset_adu = 0 # Do not offset by 40
SIM.adc_offset_adu = 10 # Do not offset by 40
import sys
if len(sys.argv) > 2:
SIM.seed = -int(sys.argv[2])
if len(sys.argv) > 1:
if sys.argv[1] == "random": SIM.randomize_orientation()
SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = 25 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image
# 3000000 images would be 100000 hours on a 60-core machine (dials), or 11.4 years
# using 2 nodes, 5.7 years. Do this at SLAC? NERSC? combination of all?
# SLAC downtimes: Tues Dec 5 (24 hrs), Mon Dec 11 (72 hrs), Mon Dec 18 light use, 24 days
# mosaic_domains setter must come after mosaic_spread_deg setter
SIM.distance_mm = 141.7
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(mean=0,
sigma=SIM.mosaic_spread_deg *
math.pi / 180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(SIM.mosaic_domains)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False))
SIM.set_mosaic_blocks(UMAT_nm)
#SIM.detector_thick_mm = 0.5 # = 0 for Rayonix
#SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD
#SIM.detector_attenuation_length_mm = default is silicon
# get same noise each time this test is run
SIM.seed = 1
SIM.oversample = 1
SIM.wavelength_A = wavelength_A
SIM.polarization = 1
# this will become F000, marking the beam center
SIM.default_F = 0
SIM.Fhkl = sfall_channels[0] # instead of sfall_main
Amatrix_rot = (rotation * sqr(
sfall_channels[0].unit_cell().orthogonalization_matrix())).transpose()
SIM.Amatrix_RUB = Amatrix_rot
#workaround for failing init_cell, use custom written Amatrix setter
print("unit_cell_Adeg=", SIM.unit_cell_Adeg)
print("unit_cell_tuple=", SIM.unit_cell_tuple)
Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM
from cctbx import crystal_orientation
Ori = crystal_orientation.crystal_orientation(
Amat, crystal_orientation.basis_type.reciprocal)
# fastest option, least realistic
SIM.xtal_shape = shapetype.Gauss_argchk # both crystal & RLP are Gaussian
# only really useful for long runs
SIM.progress_meter = False
# prints out value of one pixel only. will not render full image!
# flux is always in photons/s
SIM.flux = 1e12
SIM.exposure_s = 1.0 # so total fluence is e12
# assumes round beam
SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense
temp = SIM.Ncells_abc
SIM.Ncells_abc = temp
# rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor
water_bg = flex.vec2_double([(0, 2.57), (0.0365, 2.58), (0.07, 2.8),
(0.12, 5), (0.162, 8), (0.18, 7.32),
(0.2, 6.75), (0.216, 6.75), (0.236, 6.5),
(0.28, 4.5), (0.3, 4.3), (0.345, 4.36),
(0.436, 3.77), (0.5, 3.17)])
assert [a[0] for a in water_bg] == sorted([a[0] for a in water_bg])
# rough approximation to air
air_bg = flex.vec2_double([(0, 14.1), (0.045, 13.5), (0.174, 8.35),
(0.35, 4.78), (0.5, 4.22)])
assert [a[0] for a in air_bg] == sorted([a[0] for a in air_bg])
# simulated crystal is only 125 unit cells (25 nm wide)
# amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume)
SIM.raw_pixels *= crystal.domains_per_crystal
# must calculate the correct scale!
use_exascale_api = os.environ.get("USE_EXASCALE_API", "True")
cache_fhkl_on_gpu = os.environ.get("CACHE_FHKL_ON_GPU", "True")
assert use_exascale_api in ["True", "False"
] and cache_fhkl_on_gpu in ["True", "False"]
use_exascale_api = (use_exascale_api == "True")
cache_fhkl_on_gpu = (cache_fhkl_on_gpu == "True")
QQ = Profiler("nanoBragg Bragg spots rank %d" % (rank))
if use_exascale_api:
#something new
devices_per_node = int(os.environ["DEVICES_PER_NODE"])
SIM.device_Id = rank % devices_per_node
assert gpu_channels_singleton.get_deviceID() == SIM.device_Id
if cache_fhkl_on_gpu: #flag to switch on GPU energy channels
if gpu_channels_singleton.get_nchannels() == 0: # if uninitialized
P = Profiler("Initialize the channels singleton rank %d" %
(rank))
for x in range(len(flux)):
gpu_channels_singleton.structure_factors_to_GPU_direct(
x, sfall_channels[x].indices(),
sfall_channels[x].data())
del P
import time
print("datetime for channels singleton rank %d" % (rank),
time.time())
# allocate GPU arrays
SIM.allocate()
# loop over energies
for x in range(len(flux)):
P = Profiler("USE_EXASCALE_API nanoBragg Python and C++ rank %d" %
(rank))
print("USE_EXASCALE_API+++++++++++++++++++++++ Wavelength", x)
# from channel_pixels function
SIM.wavelength_A = wavlen[x]
SIM.flux = flux[x]
if cache_fhkl_on_gpu: # new interface, use singleton to store all-image energy channels
SIM.add_energy_channel_from_gpu_amplitudes(
x, gpu_channels_singleton)
else: # old interface, host-to-device each energy channel, each image
SIM.Fhkl = sfall_channels[x]
SIM.add_energy_channel_cuda()
del P
SIM.scale_in_place(
crystal.domains_per_crystal) # apply scale directly on GPU
SIM.wavelength_A = wavelength_A # return to canonical energy for subsequent background
if add_background_algorithm == "cuda":
QQ = Profiler("nanoBragg background rank %d" % (rank))
SIM.Fbg_vs_stol = water_bg
SIM.amorphous_sample_thick_mm = 0.1
SIM.amorphous_density_gcm3 = 1
SIM.amorphous_molecular_weight_Da = 18
SIM.flux = 1e12
SIM.beamsize_mm = 0.003 # square (not user specified)
SIM.exposure_s = 1.0 # multiplies flux x exposure
SIM.add_background()
SIM.Fbg_vs_stol = air_bg
SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator
SIM.amorphous_density_gcm3 = 1.2e-3
SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2
SIM.add_background()
# deallocate GPU arrays
SIM.get_raw_pixels() # updates SIM.raw_pixels from GPU
SIM.deallocate()
SIM.Amatrix_RUB = Amatrix_rot # return to canonical orientation
del QQ
else:
for x in range(len(flux)):
if rank in [0, 7]:
P = Profiler("nanoBragg Python and C++ rank %d" % (rank))
if rank in [0, 7]:
print("+++++++++++++++++++++++++++++++++++++++ Wavelength", x)
CH = channel_pixels(wavlen[x], flux[x], N, UMAT_nm, Amatrix_rot,
rank, sfall_channels[x])
SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal
CHDBG_singleton.extract(channel_no=x, data=CH.raw_pixels)
CH.free_all()
if rank in [0, 7]: del P
if add_background_algorithm in ["jh", "sort_stable"]:
QQ = Profiler("nanoBragg background rank %d" % (rank))
SIM.Fbg_vs_stol = water_bg
SIM.amorphous_sample_thick_mm = 0.1
SIM.amorphous_density_gcm3 = 1
SIM.amorphous_molecular_weight_Da = 18
SIM.flux = 1e12
SIM.beamsize_mm = 0.003 # square (not user specified)
SIM.exposure_s = 1.0 # multiplies flux x exposure
SIM.add_background(
sort_stable=(add_background_algorithm == "sort_stable"))
SIM.Fbg_vs_stol = air_bg
SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator
SIM.amorphous_density_gcm3 = 1.2e-3
SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2
SIM.add_background(
sort_stable=(add_background_algorithm == "sort_stable"))
del QQ
SIM.detector_psf_kernel_radius_pixels = 5
SIM.detector_psf_type = shapetype.Unknown # for CSPAD
SIM.detector_psf_fwhm_mm = 0
#SIM.apply_psf()
QQ = Profiler("nanoBragg noise rank %d" % (rank))
#SIM.add_noise() #converts phtons to ADU.
del QQ
extra = "PREFIX=%s;\nRANK=%d;\n" % (prefix, rank)
SIM.to_smv_format_py(fileout=burst_buffer_fileout,
intfile_scale=1,
rotmat=True,
extra=extra,
gz=True)
SIM.free_all()
def test_mat3():
fo = flex.mat3_double(10)
as_np = flumpy.to_numpy(fo)
for i in range(10):
fo[i] = [1, i, 0, 0, 1, 0, i, 0, 1]
assert (as_np.reshape(10, 9) == fo).all()
def run_sim2smv(ROI,prefix,crystal,spectra,rotation,rank,quick=False):
smv_fileout = prefix + ".img"
direct_algo_res_limit = 1.7
wavlen, flux, wavelength_A = next(spectra) # list of lambdas, list of fluxes, average wavelength
tophat_spectrum = True
if tophat_spectrum:
sum_flux = flex.sum(flux)
#from IPython import embed; embed()
ave_flux = sum_flux/60. # 60 energy channels
for ix in range(len(wavlen)):
energy = 12398.425 / wavlen[ix]
if energy>=7090 and energy <=7150:
flux[ix]=ave_flux
else:
flux[ix]=0.
if quick:
wavlen = flex.double([wavelength_A]);
flux = flex.double([flex.sum(flux)])
print("Quick sim, lambda=%f, flux=%f"%(wavelength_A,flux[0]))
GF = gen_fmodel(resolution=direct_algo_res_limit,pdb_text=pdb_lines,algorithm="fft",wavelength=wavelength_A)
GF.set_k_sol(0.435)
GF.make_P1_primitive()
sfall_main = GF.get_amplitudes()
# use crystal structure to initialize Fhkl array
sfall_main.show_summary(prefix = "Amplitudes used ")
N = crystal.number_of_cells(sfall_main.unit_cell())
#SIM = nanoBragg(detpixels_slowfast=(2000,2000),pixel_size_mm=0.11,Ncells_abc=(5,5,5),verbose=0)
SIM = nanoBragg(detpixels_slowfast=(3000,3000),pixel_size_mm=0.11,Ncells_abc=(N,N,N),
# workaround for problem with wavelength array, specify it separately in constructor.
wavelength_A=wavelength_A,verbose=0)
SIM.adc_offset_adu = 0 # Do not offset by 40
SIM.adc_offset_adu = 10 # Do not offset by 40
import sys
if len(sys.argv)>2:
SIM.seed = -int(sys.argv[2])
print("GOTHERE seed=",SIM.seed)
if len(sys.argv)>1:
if sys.argv[1]=="random" : SIM.randomize_orientation()
SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = 50 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image
# 3000000 images would be 100000 hours on a 60-core machine (dials), or 11.4 years
# using 2 nodes, 5.7 years. Do this at SLAC? NERSC? combination of all?
# SLAC downtimes: Tues Dec 5 (24 hrs), Mon Dec 11 (72 hrs), Mon Dec 18 light use, 24 days
# mosaic_domains setter must come after mosaic_spread_deg setter
SIM.distance_mm=141.7
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi/180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(SIM.mosaic_domains)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) )
SIM.set_mosaic_blocks(UMAT_nm)
#SIM.detector_thick_mm = 0.5 # = 0 for Rayonix
#SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD
#SIM.detector_attenuation_length_mm = default is silicon
# get same noise each time this test is run
SIM.seed = 1
SIM.oversample=1
SIM.wavelength_A = wavelength_A
SIM.polarization=1
# this will become F000, marking the beam center
SIM.default_F=0
#SIM.missets_deg= (10,20,30)
print("mosaic_seed=",SIM.mosaic_seed)
print("seed=",SIM.seed)
print("calib_seed=",SIM.calib_seed)
print("missets_deg =", SIM.missets_deg)
SIM.Fhkl=sfall_main
print("Determinant",rotation.determinant())
Amatrix_rot = (rotation * sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose()
print("RAND_ORI", prefix, end=' ')
for i in Amatrix_rot: print(i, end=' ')
print()
SIM.Amatrix_RUB = Amatrix_rot
#workaround for failing init_cell, use custom written Amatrix setter
print("unit_cell_Adeg=",SIM.unit_cell_Adeg)
print("unit_cell_tuple=",SIM.unit_cell_tuple)
Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM
from cctbx import crystal_orientation
Ori = crystal_orientation.crystal_orientation(Amat, crystal_orientation.basis_type.reciprocal)
print("Python unit cell from SIM state",Ori.unit_cell())
# fastest option, least realistic
#SIM.xtal_shape=shapetype.Tophat # RLP = hard sphere
#SIM.xtal_shape=shapetype.Square # gives fringes
SIM.xtal_shape=shapetype.Gauss # both crystal & RLP are Gaussian
#SIM.xtal_shape=shapetype.Round # Crystal is a hard sphere
# only really useful for long runs
SIM.progress_meter=False
# prints out value of one pixel only. will not render full image!
#SIM.printout_pixel_fastslow=(500,500)
#SIM.printout=True
SIM.show_params()
# flux is always in photons/s
SIM.flux=1e12
SIM.exposure_s=1.0 # so total fluence is e12
# assumes round beam
SIM.beamsize_mm=0.003 #cannot make this 3 microns; spots are too intense
temp=SIM.Ncells_abc
print("Ncells_abc=",SIM.Ncells_abc)
SIM.Ncells_abc=temp
print("Ncells_abc=",SIM.Ncells_abc)
print("xtal_size_mm=",SIM.xtal_size_mm)
print("unit_cell_Adeg=",SIM.unit_cell_Adeg)
print("unit_cell_tuple=",SIM.unit_cell_tuple)
print("missets_deg=",SIM.missets_deg)
print("Amatrix=",SIM.Amatrix)
print("beam_center_mm=",SIM.beam_center_mm)
print("XDS_ORGXY=",SIM.XDS_ORGXY)
print("detector_pivot=",SIM.detector_pivot)
print("xtal_shape=",SIM.xtal_shape)
print("beamcenter_convention=",SIM.beamcenter_convention)
print("fdet_vector=",SIM.fdet_vector)
print("sdet_vector=",SIM.sdet_vector)
print("odet_vector=",SIM.odet_vector)
print("beam_vector=",SIM.beam_vector)
print("polar_vector=",SIM.polar_vector)
print("spindle_axis=",SIM.spindle_axis)
print("twotheta_axis=",SIM.twotheta_axis)
print("distance_meters=",SIM.distance_meters)
print("distance_mm=",SIM.distance_mm)
print("close_distance_mm=",SIM.close_distance_mm)
print("detector_twotheta_deg=",SIM.detector_twotheta_deg)
print("detsize_fastslow_mm=",SIM.detsize_fastslow_mm)
print("detpixels_fastslow=",SIM.detpixels_fastslow)
print("detector_rot_deg=",SIM.detector_rot_deg)
print("curved_detector=",SIM.curved_detector)
print("pixel_size_mm=",SIM.pixel_size_mm)
print("point_pixel=",SIM.point_pixel)
print("polarization=",SIM.polarization)
print("nopolar=",SIM.nopolar)
print("oversample=",SIM.oversample)
print("region_of_interest=",SIM.region_of_interest)
print("wavelength_A=",SIM.wavelength_A)
print("energy_eV=",SIM.energy_eV)
print("fluence=",SIM.fluence)
print("flux=",SIM.flux)
print("exposure_s=",SIM.exposure_s)
print("beamsize_mm=",SIM.beamsize_mm)
print("dispersion_pct=",SIM.dispersion_pct)
print("dispsteps=",SIM.dispsteps)
print("divergence_hv_mrad=",SIM.divergence_hv_mrad)
print("divsteps_hv=",SIM.divsteps_hv)
print("divstep_hv_mrad=",SIM.divstep_hv_mrad)
print("round_div=",SIM.round_div)
print("phi_deg=",SIM.phi_deg)
print("osc_deg=",SIM.osc_deg)
print("phisteps=",SIM.phisteps)
print("phistep_deg=",SIM.phistep_deg)
print("detector_thick_mm=",SIM.detector_thick_mm)
print("detector_thicksteps=",SIM.detector_thicksteps)
print("detector_thickstep_mm=",SIM.detector_thickstep_mm)
print("***mosaic_spread_deg=",SIM.mosaic_spread_deg)
print("***mosaic_domains=",SIM.mosaic_domains)
print("indices=",SIM.indices)
print("amplitudes=",SIM.amplitudes)
print("Fhkl_tuple=",SIM.Fhkl_tuple)
print("default_F=",SIM.default_F)
print("interpolate=",SIM.interpolate)
print("integral_form=",SIM.integral_form)
from libtbx.development.timers import Profiler
P = Profiler("nanoBragg")
# now actually burn up some CPU
#SIM.add_nanoBragg_spots()
del P
# simulated crystal is only 125 unit cells (25 nm wide)
# amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume)
print(crystal.domains_per_crystal)
SIM.raw_pixels *= crystal.domains_per_crystal; # must calculate the correct scale!
output = StringIO() # open("myfile","w")
for x in range(0,100,2): #len(flux)):
if flux[x]==0.0:continue
print("+++++++++++++++++++++++++++++++++++++++ Wavelength",x)
CH = channel_pixels(ROI,wavlen[x],flux[x],N,UMAT_nm,Amatrix_rot,GF,output)
SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal;
print(SIM.raw_pixels)
CH.free_all()
message = output.getvalue().split()
miller = (int(message[4]),int(message[5]),int(message[6]))
intensity = float(message[9]);
#SIM.to_smv_format(fileout=prefix + "_intimage_001.img")
pixels = SIM.raw_pixels
roi_pixels = pixels[ROI[1][0]:ROI[1][1], ROI[0][0]:ROI[0][1]]
print("Reducing full shape of",pixels.focus(),"to ROI of",roi_pixels.focus())
SIM.free_all()
return dict(roi_pixels=roi_pixels,miller=miller,intensity=intensity)
def run_sim2smv(img_prefix=None, simparams=None,pdb_lines=None,crystal=None,spectra=None,rotation=None,rank=None,fsave=None,sfall_cluster=None,quick=False):
smv_fileout = fsave
direct_algo_res_limit = simparams.direct_algo_res_limit
wavlen, flux, real_wavelength_A = next(spectra) # list of lambdas, list of fluxes, average wavelength
real_flux = flex.sum(flux)
assert real_wavelength_A > 0
# print(rank, " ## real_wavelength_A/real_flux = ", real_wavelength_A, real_flux*1.0/simparams.flux)
if quick:
wavlen = flex.double([real_wavelength_A])
flux = flex.double([real_flux])
# GF = gen_fmodel(resolution=simparams.direct_algo_res_limit,pdb_text=pdb_lines,algorithm=simparams.fmodel_algorithm,wavelength=real_wavelength_A)
# GF.set_k_sol(simparams.k_sol)
# GF.make_P1_primitive()
sfall_main = sfall_cluster["main"] #GF.get_amplitudes()
# use crystal structure to initialize Fhkl array
# sfall_main.show_summary(prefix = "Amplitudes used ")
N = crystal.number_of_cells(sfall_main.unit_cell())
#print("## number of N = ", N)
SIM = nanoBragg(detpixels_slowfast=(simparams.detector_size_ny,simparams.detector_size_nx),pixel_size_mm=simparams.pixel_size_mm,\
Ncells_abc=(N,N,N),wavelength_A=real_wavelength_A,verbose=0)
# workaround for problem with wavelength array, specify it separately in constructor.
# SIM.adc_offset_adu = 0 # Do not offset by 40
SIM.adc_offset_adu = 10 # Do not offset by 40
SIM.seed = 0
# SIM.randomize_orientation()
SIM.mosaic_spread_deg = simparams.mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = simparams.mosaic_domains # 77 seconds.
SIM.distance_mm = simparams.distance_mm
## setup the mosaicity
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi/180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(SIM.mosaic_domains)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) )
SIM.set_mosaic_blocks(UMAT_nm)
######################
SIM.beamcenter_convention=convention.ADXV
SIM.beam_center_mm=(simparams.beam_center_x_mm, simparams.beam_center_y_mm) # 95.975 96.855
######################
# get same noise each time this test is run
SIM.seed = 0
SIM.oversample=simparams.oversample
SIM.wavelength_A = real_wavelength_A
SIM.polarization=simparams.polarization
# this will become F000, marking the beam center
SIM.default_F=simparams.default_F
#SIM.missets_deg= (10,20,30)
SIM.Fhkl=sfall_main
Amatrix_rot = (rotation * sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose()
SIM.Amatrix_RUB = Amatrix_rot
#workaround for failing init_cell, use custom written Amatrix setter
# print("## inside run_sim2smv, Amat_rot = ", Amatrix_rot)
Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM
Ori = crystal_orientation.crystal_orientation(Amat, crystal_orientation.basis_type.reciprocal)
SIM.xtal_shape=shapetype.Gauss # both crystal & RLP are Gaussian
SIM.progress_meter=False
# SIM.show_params()
# flux is always in photons/s
SIM.flux=real_flux
SIM.exposure_s=simparams.exposure_s
# assumes round beam
SIM.beamsize_mm=simparams.beamsize_mm #cannot make this 3 microns; spots are too intense
temp=SIM.Ncells_abc
SIM.Ncells_abc=temp
# print("## domains_per_crystal = ", crystal.domains_per_crystal)
SIM.raw_pixels *= crystal.domains_per_crystal # must calculate the correct scale!
# print("## Initial raw_pixels = ", flex.sum(SIM.raw_pixels))
for x in range(len(flux)):
# CH = channel_pixels(wavlen[x],flux[x],N,UMAT_nm,Amatrix_rot,sfall_cluster[x],rank)
# print("## in loop wavlen/flux/real_wavelength_A = ", wavlen[x], flux[x]/real_flux, real_wavelength_A)
CH = channel_pixels(simparams=simparams,single_wavelength_A=wavlen[x],single_flux=flux[x],N=N,UMAT_nm=UMAT_nm, \
Amatrix_rot=Amatrix_rot,sfall_channel=sfall_cluster[x],rank=rank)
SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal
CHDBG_singleton.extract(channel_no=x, data=CH.raw_pixels)
CH.free_all()
# print("## sum raw_pixels after ", x, "is", flex.sum(SIM.raw_pixels))
# rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor
bg = flex.vec2_double([(0,2.57),(0.0365,2.58),(0.07,2.8),(0.12,5),(0.162,8),(0.2,6.75),(0.18,7.32),(0.216,6.75),(0.236,6.5),(0.28,4.5),(0.3,4.3),(0.345,4.36),(0.436,3.77),(0.5,3.17)])
SIM.Fbg_vs_stol = bg
SIM.amorphous_sample_thick_mm = simparams.water_sample_thick_mm
SIM.amorphous_density_gcm3 = simparams.water_density_gcm3
SIM.amorphous_molecular_weight_Da = simparams.water_molecular_weight_Da
SIM.flux=real_flux
SIM.beamsize_mm=simparams.beamsize_mm # square (not user specified)
SIM.exposure_s=simparams.exposure_s # multiplies flux x exposure
SIM.add_background()
# rough approximation to air
bg = flex.vec2_double([(0,14.1),(0.045,13.5),(0.174,8.35),(0.35,4.78),(0.5,4.22)])
SIM.Fbg_vs_stol = bg
SIM.amorphous_sample_thick_mm = simparams.air_sample_thick_mm # between beamstop and collimator
SIM.amorphous_density_gcm3 = simparams.air_density_gcm3
SIM.amorphous_sample_molecular_weight_Da = simparams.air_molecular_weight_Da # nitrogen = N2
SIM.add_background()
#apply beamstop mask here
# settings for CCD
SIM.detector_psf_kernel_radius_pixels=simparams.detector_psf_kernel_radius_pixels
SIM.detector_psf_type=shapetype.Unknown # for CSPAD
SIM.detector_psf_fwhm_mm=simparams.detector_psf_fwhm_mm
SIM.apply_psf()
SIM.add_noise()
extra = "PREFIX=%s;\nRANK=%d;\n"%(img_prefix,rank)
SIM.to_smv_format_py(fileout=smv_fileout,intfile_scale=1,rotmat=True,extra=extra,gz=True)
SIM.free_all()
def run_sim2smv(prefix,
crystal,
spectra,
rotation,
rank,
sfall_main,
quick=False,
save_bragg=False,
save_smv=True,
save_h5=False,
return_pixels=False):
smv_fileout = prefix + ".img"
#if not quick:
# if not write_safe(smv_fileout):
# print("File %s already exists, skipping in rank %d"%(smv_fileout,rank))
# return
direct_algo_res_limit = 1.7
wavlen, flux, wavelength_A = next(
spectra) # list of lambdas, list of fluxes, average wavelength
assert wavelength_A > 0
assert (len(wavlen) == len(flux) == len(sfall_main))
if quick:
wavlen = flex.double([wavelength_A])
flux = flex.double([flex.sum(flux)])
print("Quick sim, lambda=%f, flux=%f" % (wavelength_A, flux[0]))
# use crystal structure to initialize Fhkl array
#sfall_main[0].show_summary(prefix = "Amplitudes used ")
N = crystal.number_of_cells(sfall_main[0].unit_cell())
if use_microcrystal:
global Ncells_abc
Ncells_abc = (N, N, N)
SIM = nanoBragg(detpixels_slowfast=detpixels_slowfast,
pixel_size_mm=pixsize_mm,
Ncells_abc=Ncells_abc,
wavelength_A=wavelength_A,
verbose=verbose)
SIM.adc_offset_adu = offset_adu # Do not offset by 40
SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev
SIM.mosaic_domains = mos_doms
SIM.distance_mm = distance_mm
UMAT_nm = flex.mat3_double()
mersenne_twister = flex.mersenne_twister(seed=0)
scitbx.random.set_random_seed(1234)
rand_norm = scitbx.random.normal_distribution(mean=0,
sigma=SIM.mosaic_spread_deg *
math.pi / 180.)
g = scitbx.random.variate(rand_norm)
mosaic_rotation = g(SIM.mosaic_domains)
for m in mosaic_rotation:
site = col(mersenne_twister.random_double_point_on_sphere())
UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False))
SIM.set_mosaic_blocks(UMAT_nm)
# get same noise each time this test is run
SIM.seed = 1
SIM.oversample = 1
SIM.wavelength_A = wavelength_A
SIM.polarization = 1
# this will become F000, marking the beam center
SIM.default_F = 0
#SIM.missets_deg= (10,20,30)
if verbose:
print("mosaic_seed=", SIM.mosaic_seed)
print("seed=", SIM.seed)
print("calib_seed=", SIM.calib_seed)
print("missets_deg =", SIM.missets_deg)
SIM.Fhkl = sfall_main[0].amplitudes()
if verbose: print("Determinant", rotation.determinant())
Amatrix_rot = (
rotation *
sqr(sfall_main[0].unit_cell().orthogonalization_matrix())).transpose()
if verbose:
print("RAND_ORI", prefix, end=' ')
for i in Amatrix_rot:
print(i, end=' ')
print()
SIM.Amatrix_RUB = Amatrix_rot
#workaround for failing init_cell, use custom written Amatrix setter
if verbose:
print("unit_cell_Adeg=", SIM.unit_cell_Adeg)
print("unit_cell_tuple=", SIM.unit_cell_tuple)
Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM
Ori = crystal_orientation.crystal_orientation(
Amat, crystal_orientation.basis_type.reciprocal)
if verbose: print("Python unit cell from SIM state", Ori.unit_cell())
SIM.xtal_shape = shapetype.Gauss
SIM.progress_meter = False
#SIM.show_params()
SIM.flux = flux_ave
SIM.exposure_s = exposure_s
SIM.beamsize_mm = beam_size_mm
temp = SIM.Ncells_abc
if verbose: print("Ncells_abc=", SIM.Ncells_abc)
SIM.Ncells_abc = temp
if verbose:
print("Ncells_abc=", SIM.Ncells_abc)
print("xtal_size_mm=", SIM.xtal_size_mm)
print("unit_cell_Adeg=", SIM.unit_cell_Adeg)
print("unit_cell_tuple=", SIM.unit_cell_tuple)
print("missets_deg=", SIM.missets_deg)
print("Amatrix=", SIM.Amatrix)
print("beam_center_mm=", SIM.beam_center_mm)
print("XDS_ORGXY=", SIM.XDS_ORGXY)
print("detector_pivot=", SIM.detector_pivot)
print("xtal_shape=", SIM.xtal_shape)
print("beamcenter_convention=", SIM.beamcenter_convention)
print("fdet_vector=", SIM.fdet_vector)
print("sdet_vector=", SIM.sdet_vector)
print("odet_vector=", SIM.odet_vector)
print("beam_vector=", SIM.beam_vector)
print("polar_vector=", SIM.polar_vector)
print("spindle_axis=", SIM.spindle_axis)
print("twotheta_axis=", SIM.twotheta_axis)
print("distance_meters=", SIM.distance_meters)
print("distance_mm=", SIM.distance_mm)
print("close_distance_mm=", SIM.close_distance_mm)
print("detector_twotheta_deg=", SIM.detector_twotheta_deg)
print("detsize_fastslow_mm=", SIM.detsize_fastslow_mm)
print("detpixels_fastslow=", SIM.detpixels_fastslow)
print("detector_rot_deg=", SIM.detector_rot_deg)
print("curved_detector=", SIM.curved_detector)
print("pixel_size_mm=", SIM.pixel_size_mm)
print("point_pixel=", SIM.point_pixel)
print("polarization=", SIM.polarization)
print("nopolar=", SIM.nopolar)
print("oversample=", SIM.oversample)
print("region_of_interest=", SIM.region_of_interest)
print("wavelength_A=", SIM.wavelength_A)
print("energy_eV=", SIM.energy_eV)
print("fluence=", SIM.fluence)
print("flux=", SIM.flux)
print("exposure_s=", SIM.exposure_s)
print("beamsize_mm=", SIM.beamsize_mm)
print("dispersion_pct=", SIM.dispersion_pct)
print("dispsteps=", SIM.dispsteps)
print("divergence_hv_mrad=", SIM.divergence_hv_mrad)
print("divsteps_hv=", SIM.divsteps_hv)
print("divstep_hv_mrad=", SIM.divstep_hv_mrad)
print("round_div=", SIM.round_div)
print("phi_deg=", SIM.phi_deg)
print("osc_deg=", SIM.osc_deg)
print("phisteps=", SIM.phisteps)
print("phistep_deg=", SIM.phistep_deg)
print("detector_thick_mm=", SIM.detector_thick_mm)
print("detector_thicksteps=", SIM.detector_thicksteps)
print("detector_thickstep_mm=", SIM.detector_thickstep_mm)
print("***mosaic_spread_deg=", SIM.mosaic_spread_deg)
print("***mosaic_domains=", SIM.mosaic_domains)
print("indices=", SIM.indices)
print("amplitudes=", SIM.amplitudes)
print("Fhkl_tuple=", SIM.Fhkl_tuple)
print("default_F=", SIM.default_F)
print("interpolate=", SIM.interpolate)
print("integral_form=", SIM.integral_form)
# simulated crystal is only 125 unit cells (25 nm wide)
# amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume)
if verbose: print(crystal.domains_per_crystal)
SIM.raw_pixels *= crystal.domains_per_crystal
# must calculate the correct scale!
Nflux = len(flux)
for x in range(Nflux):
#P = Profiler("nanoBragg Python and C++ rank %d"%(rank))
if x % 10 == 0:
print(
"+++++++++++++++++++++++++++++++++++++++ Wavelength %d / %d" %
(x + 1, Nflux),
end="\r")
if flux[x] == 0:
continue
CH = channel_pixels(wavlen[x], flux[x], UMAT_nm, Amatrix_rot, rank,
sfall_main[x])
if use_microcrystal:
SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal
else:
SIM.raw_pixels += CH.raw_pixels
CH.free_all()
#del P
print()
# image 1: crystal Bragg scatter
if quick or save_bragg:
SIM.to_smv_format(fileout=prefix + "_intimage_001.img")
if save_bragg:
raw_to_pickle(SIM.raw_pixels, fileout=prefix + "_dblprec_001.pickle")
SIM.raw_pixels = SIM.raw_pixels + background
#SIM.raw_pixels = SIM.raw_pixels*0
## rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor
#bg = flex.vec2_double([(0,2.57),(0.0365,2.58),(0.07,2.8),(0.12,5),(0.162,8),(0.2,6.75),(0.18,7.32),(0.216,6.75),(0.236,6.5),(0.28,4.5),(0.3,4.3),(0.345,4.36),(0.436,3.77),(0.5,3.17)])
#SIM.Fbg_vs_stol = bg
#SIM.amorphous_sample_thick_mm = sample_thick_mm
#SIM.amorphous_density_gcm3 = 1
#SIM.amorphous_molecular_weight_Da = 18
#SIM.flux=flux_ave
#SIM.beamsize_mm=beam_size_mm
#SIM.exposure_s=exposure_s
#SIM.add_background()
#if quick: SIM.to_smv_format(fileout=prefix + "_intimage_002.img")
## rough approximation to air
#bg = flex.vec2_double([(0,14.1),(0.045,13.5),(0.174,8.35),(0.35,4.78),(0.5,4.22)])
#SIM.Fbg_vs_stol = bg
#SIM.amorphous_sample_thick_mm = air_thick_mm # between beamstop and collimator
#SIM.amorphous_density_gcm3 = 1.2e-3
#SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2
#print("amorphous_sample_size_mm=",SIM.amorphous_sample_size_mm)
#print("amorphous_sample_thick_mm=",SIM.amorphous_sample_thick_mm)
#print("amorphous_density_gcm3=",SIM.amorphous_density_gcm3)
#print("amorphous_molecular_weight_Da=",SIM.amorphous_molecular_weight_Da)
#SIM.add_background()
##apply beamstop mask here
## set this to 0 or -1 to trigger automatic radius. could be very slow with bright images
## settings for CCD
#utils.save_flex(SIM.raw_pixels, "background")
SIM.detector_psf_kernel_radius_pixels = 5
#SIM.detector_psf_fwhm_mm=0.08;
#SIM.detector_psf_type=shapetype.Fiber # rayonix=Fiber, CSPAD=None (or small Gaussian)
SIM.detector_psf_type = shapetype.Unknown # for CSPAD
SIM.detector_psf_fwhm_mm = 0
SIM.quantum_gain = 28.
#SIM.apply_psf()
if verbose: print("One pixel-->", SIM.raw_pixels[500000])
# at this point we scale the raw pixels so that the output array is on an scale from 0 to 50000.
# that is the default behavior (intfile_scale<=0), otherwise it applies intfile_scale as a multiplier on an abs scale.
if quick: SIM.to_smv_format(fileout=prefix + "_intimage_003.img")
if verbose:
print("quantum_gain=",
SIM.quantum_gain) #defaults to 1. converts photons to ADU
print("adc_offset_adu=", SIM.adc_offset_adu)
print("detector_calibration_noise_pct=",
SIM.detector_calibration_noise_pct)
print("flicker_noise_pct=", SIM.flicker_noise_pct)
print("readout_noise_adu=", SIM.readout_noise_adu
) # gaussian random number to add to every pixel (0 for PAD)
# apply Poissonion correction, then scale to ADU, then adc_offset.
# should be 10 for most Rayonix, Pilatus should be 0, CSPAD should be 0.
print("detector_psf_type=", SIM.detector_psf_type)
print("detector_psf_fwhm_mm=", SIM.detector_psf_fwhm_mm)
print("detector_psf_kernel_radius_pixels=",
SIM.detector_psf_kernel_radius_pixels)
SIM.add_noise() #converts phtons to ADU.
if verbose: print("raw_pixels=", SIM.raw_pixels)
extra = "PREFIX=%s;\nRANK=%d;\n" % (prefix, rank)
out = SIM.raw_pixels.as_numpy_array()
if save_smv:
SIM.to_smv_format_py(fileout=smv_fileout,
intfile_scale=1,
rotmat=True,
extra=extra,
gz=True)
elif save_h5:
h5_fileout = smv_fileout + ".h5"
f = h5py.File(h5_fileout, "w")
f.create_dataset("data",
data=SIM.raw_pixels.as_numpy_array().astype(
np.uint16),
compression="lzf")
f.close()
SIM.free_all()
