def run_uniform(eta_angle, sample_size=20000, verbose=True): UMAT = flex.mat3_double() d_UMAT_d_eta = flex.mat3_double() # the axis is sampled randomly on a sphere. # Alternately it could have been calculated on a regular hemispheric grid # but regular grid is not needed; the only goal is to have the ability to compute gradient mersenne_twister = flex.mersenne_twister( seed=0) # set seed, get reproducible results # for each axis, sample both the + and - angle assert sample_size % 2 == 0 # the angle is sampled uniformly from its distribution mosaic_rotation0 = np.array(range(sample_size // 2)) mosaic_rotation1 = special.erfinv(mosaic_rotation0 / (sample_size // 2)) d_theta_d_eta = math.sqrt(2.0) * flex.double(mosaic_rotation1) mosaic_rotation = (math.pi / 180.) * eta_angle * d_theta_d_eta for m, d in zip(mosaic_rotation, d_theta_d_eta): site = col(mersenne_twister.random_double_point_on_sphere()) UMAT.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) UMAT.append(site.axis_and_angle_as_r3_rotation_matrix(-m, deg=False)) d_umat = site.axis_and_angle_as_r3_derivative_wrt_angle(m, deg=False) d_UMAT_d_eta.append((math.pi / 180.) * d * d_umat) d_umat = site.axis_and_angle_as_r3_derivative_wrt_angle(-m, deg=False) d_UMAT_d_eta.append((math.pi / 180.) * -d * d_umat) #sanity check on the gaussian distribution if verbose: nm_angles = check_distributions.get_angular_rotation(UMAT) nm_rms_angle = math.sqrt(flex.mean(nm_angles * nm_angles)) print("Normal rms angle is ", nm_rms_angle) return UMAT, d_UMAT_d_eta
def Umats(mos_spread_deg, n_mos_doms, isotropic=True, seed=777, norm_dist_seed=777): import scitbx from scitbx.matrix import col import math UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=seed) scitbx.random.set_random_seed(norm_dist_seed) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=(mos_spread_deg * math.pi / 180.0)) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(n_mos_doms) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) if mos_spread_deg > 0: UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) else: UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(0, deg=False)) if isotropic and mos_spread_deg > 0: UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix((-m), deg=False)) return UMAT_nm
def gradients(self): """A generator function to return the gradients dR/dp for all the restraints referring to a particular crystal's cell parameters. The return value is a list of sparse matrices, one for each of the 6 cell parameters being restrained. Each sparse matrix has as many columns as the crystal unit cell parameterisation has parameters, and as many rows as there are crystals being restrained. Gradients of zero are detected and not set in the sparse matrices to save memory.""" for i, xlucp in enumerate(self._xlucp): B = xlucp.get_state() dB_dp = flex.mat3_double(xlucp.get_ds_dp()) # Use C++ function for speed ccg = CalculateCellGradients(B, dB_dp) dRdp = [] if self._sel[0]: dRdp.append(self._construct_grad_block(ccg.da_dp(), i)) if self._sel[1]: dRdp.append(self._construct_grad_block(ccg.db_dp(), i)) if self._sel[2]: dRdp.append(self._construct_grad_block(ccg.dc_dp(), i)) if self._sel[3]: dRdp.append(self._construct_grad_block(ccg.daa_dp(), i)) if self._sel[4]: dRdp.append(self._construct_grad_block(ccg.dbb_dp(), i)) if self._sel[5]: dRdp.append(self._construct_grad_block(ccg.dcc_dp(), i)) yield dRdp
def channel_wavelength_fmodel(create): N_mosaic_domains = 25 mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=mosaic_spread_deg * math.pi/180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(N_mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) ) if create: # write the reference for the first time cPickle.dump(UMAT_nm, open(os.path.join(ls49_big_data,"reference",filename),"wb"),cPickle.HIGHEST_PROTOCOL) else: # read the reference and assert sameness to production run import six if six.PY3: UMAT_ref = cPickle.load(open(os.path.join(ls49_big_data,"reference",filename),"rb"),encoding="bytes") else: UMAT_ref = cPickle.load(open(os.path.join(ls49_big_data,"reference",filename),"rb")) for x in range(len(UMAT_nm)): print(x," ".join( ["%18.15f"%UMAT_ref[x][z] for z in range(9)] )) assert UMAT_nm[x] == UMAT_ref[x] expected_output = """
def _init_nanoBragg_umats(self, model_mosaic_spread): # umat implementation for the exascale api if model_mosaic_spread and self.umat_maker is None: # initialize the parameterized distribution assert self.crystal.n_mos_domains % 2 == 0 self.umat_maker = AnisoUmats(num_random_samples=2*self.crystal.n_mos_domains) if self.crystal.anisotropic_mos_spread_deg is None: # isotropic case self.D.mosaic_spread_deg = self.crystal.mos_spread_deg self.D.mosaic_domains = self.crystal.n_mos_domains if self.Umats_method == 0: # double_random legacy method, exafel tests Umats = SimData.Umats(self.crystal.mos_spread_deg, self.crystal.n_mos_domains, seed=self.mosaic_seeds[0], norm_dist_seed=self.mosaic_seeds[1]) elif self.Umats_method == 2: # double_uniform isotropic, NOTE: if Umats_method was set as 5, it was modified to be 2 Umats, _,_ = self.umat_maker.generate_Umats( self.crystal.mos_spread_deg, compute_derivs=False, crystal=None, how=2) else: raise ValueError("Invalid method for nanoBragg isotropic case") #SimData.plot_isotropic_umats(Umats, self.crystal.mos_spread_deg) else: # anisotropic case with diagonal elements Umats, _,_ = self.umat_maker.generate_Umats( self.crystal.anisotropic_mos_spread_deg, compute_derivs=False, crystal=self.crystal.dxtbx_crystal, how=1) self.exascale_mos_blocks = flex.mat3_double(Umats) self.D.set_mosaic_blocks(self.exascale_mos_blocks)
def run_sim2smv(fileout): SIM = nanoBragg(detpixels_slowfast=(1000, 1000), pixel_size_mm=0.1, Ncells_abc=(5, 5, 5), verbose=0) SIM.mosaic_spread_deg = MOSAIC_SPREAD # apparently this is half width SIM.mosaic_domains = SAMPLE_SIZE SIM.distance_mm = 100 # this triggers the generation of mosaic distribution UMAT_th = SIM.get_mosaic_blocks() # extract top-hat distributed U-mats #sanity checks on the top hat distribution th_angles = check_distributions.get_angular_rotation(UMAT_th) max_angle = flex.max(th_angles) assert max_angle <= MOSAIC_SPREAD + 0.0000001 # need to allow a small epsilon assert max_angle > 0.99 * MOSAIC_SPREAD # insist that max angle is near the limit we gave it rms_angle = math.sqrt(flex.mean(th_angles * th_angles)) print(rms_angle) assert rms_angle < MOSAIC_SPREAD import scitbx # compute an array of normally-distributed U-mats into the simulator UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister( seed=0) # set seed, get reproducible results scitbx.random.set_random_seed(4321) # set seed, get reproducibe results rand_norm = scitbx.random.normal_distribution(mean=0, sigma=rms_angle * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SAMPLE_SIZE) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) # set the normally-distributed U-mats into the simulator: SIM.set_mosaic_blocks(UMAT_nm) # get them back new_UMAT_nm = SIM.get_mosaic_blocks() #double check that simulator does not alter the UMATs for iumat in range(0, SAMPLE_SIZE, 100): assert UMAT_nm[iumat] == new_UMAT_nm[iumat] #sanity check on the gaussian distribution nm_angles = check_distributions.get_angular_rotation(new_UMAT_nm) nm_rms_angle = math.sqrt(flex.mean(nm_angles * nm_angles)) print(nm_rms_angle) assert approx_equal(rms_angle,nm_rms_angle,eps=1e-03), \ "The top hat and gaussian models should have similar standard deviations" return UMAT_th, new_UMAT_nm
def test_flex_looping_vecs(): # We want to try and avoid recursive nesting, but don't want to # return the original object if attempting to miscast vec<->numeric # however.... this means lots of conditions to check, so ignore now fo = flex.vec3_double(5) npo = flumpy.to_numpy(fo) assert flumpy.vec_from_numpy(npo) is fo # Don't be dumb when casting flex_nonvec = flex.double((9, 3)) assert not flumpy.vec_from_numpy( flumpy.to_numpy(flex_nonvec)) is flex_nonvec # mat3 fo = flex.mat3_double(5) npo = flumpy.to_numpy(fo) assert flumpy.mat3_from_numpy(npo) is fo
def _unit_cell_surplus_reflections(self, p): F_dbdp = flex.mat3_double(p.get_ds_dp()) min_nref = self._options.min_nref_per_parameter reflections = self.reflection_manager.get_obs() # if no free parameters, do as _surplus_reflections if len(F_dbdp) == 0: exp_ids = p.get_experiment_ids() isel = flex.size_t() for exp_id in exp_ids: isel.extend((reflections["id"] == exp_id).iselection()) return len(isel) return (uc_surpl( reflections["id"], reflections["miller_index"], p.get_experiment_ids(), F_dbdp, ).result - min_nref)
def run_sim2smv(fileout): SIM = nanoBragg(detpixels_slowfast=(1000,1000),pixel_size_mm=0.1,Ncells_abc=(5,5,5),verbose=0) SIM.mosaic_domains = 10000 SIM.mosaic_spread_deg = 2.0 SIM.distance_mm=100 # this triggers the generation of mosaic distribution UMAT_th = SIM.get_mosaic_blocks() import scitbx UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=math.pi/180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(10000) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) ) return UMAT_th, UMAT_nm
def _calculate_uc_gradients(self, sel=[True] * 6): """Calculate gradients of the unit cell parameters with respect to each of the parameters of the crystal unit cell model parameterisation""" B = self._xlucp.get_state() dB_dp = flex.mat3_double(self._xlucp.get_ds_dp()) # Use C++ function for speed ccg = CalculateCellGradients(B, dB_dp) nparam = len(dB_dp) da = list(ccg.da_dp()) if sel[0] else [0.0] * nparam db = list(ccg.db_dp()) if sel[1] else [0.0] * nparam dc = list(ccg.dc_dp()) if sel[2] else [0.0] * nparam daa = list(ccg.daa_dp()) if sel[3] else [0.0] * nparam dbb = list(ccg.dbb_dp()) if sel[4] else [0.0] * nparam dcc = list(ccg.dcc_dp()) if sel[5] else [0.0] * nparam return (da, db, dc, daa, dbb, dcc)
def _calculate_uc_gradients(self, sel=[True]*6): '''Calculate gradients of the unit cell parameters with respect to each of the parameters of the crystal unit cell model parameterisation''' B = self._xlucp.get_state() dB_dp = flex.mat3_double(self._xlucp.get_ds_dp()) # Use C++ function for speed ccg = CalculateCellGradients(B, dB_dp) nparam = len(dB_dp) da = list(ccg.da_dp()) if sel[0] else [0.0] * nparam db = list(ccg.db_dp()) if sel[1] else [0.0] * nparam dc = list(ccg.dc_dp()) if sel[2] else [0.0] * nparam daa = list(ccg.daa_dp()) if sel[3] else [0.0] * nparam dbb = list(ccg.dbb_dp()) if sel[4] else [0.0] * nparam dcc = list(ccg.dcc_dp()) if sel[5] else [0.0] * nparam return (da, db, dc, daa, dbb, dcc)
def run_sim2smv(fileout): SIM = nanoBragg(detpixels_slowfast=(1000, 1000), pixel_size_mm=0.1, Ncells_abc=(5, 5, 5), verbose=0) SIM.mosaic_domains = 10000 SIM.mosaic_spread_deg = MOSAIC_SPREAD SIM.distance_mm = 100 # this triggers the generation of mosaic distribution UMAT_th = SIM.get_mosaic_blocks() # extract top-hat distributed U-mats #sanity checks on the top hat distribution th_angles = check_distributions.get_angular_rotation(UMAT_th) max_angle = flex.max(th_angles) assert max_angle <= MOSAIC_SPREAD + 0.0000001 # need to allow a small epsilon assert max_angle > 0.99 * MOSAIC_SPREAD # insist that max angle is near the limit we gave it rms_angle = math.sqrt(flex.mean(th_angles * th_angles)) print rms_angle import scitbx # computer an array of normally-distributed U-mats into the simulator UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=rms_angle * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(10000) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) # set the normally-distributed U-mats into the simulator: SIM.set_mosaic_blocks(UMAT_nm) # get them back new_UMAT_nm = SIM.get_mosaic_blocks() #sanity check on the gaussian distribution nm_angles = check_distributions.get_angular_rotation(new_UMAT_nm) nm_rms_angle = math.sqrt(flex.mean(nm_angles * nm_angles)) print nm_rms_angle return UMAT_th, new_UMAT_nm
def ref_gen_varying(experiments): """Generate some reflections using the scan varying predictor""" beam = experiments[0].beam crystal = experiments[0].crystal detector = experiments[0].detector scan = experiments[0].scan # We need a UB matrix at the beginning of every image, and at the end of the # last image. These are all the same - we want to compare the scan-varying # predictor with the scan-static one for a flat scan. ar_range = scan.get_array_range() UBlist = [crystal.get_A() for t in range(ar_range[0], ar_range[1] + 1)] dmin = detector.get_max_resolution(beam.get_s0()) from dials.algorithms.spot_prediction import ScanVaryingReflectionPredictor sv_predictor = ScanVaryingReflectionPredictor(experiments[0], dmin=dmin) refs = sv_predictor.for_ub(flex.mat3_double(UBlist)) return refs
def ref_gen_varying(experiments): """Generate some reflections using the scan varying predictor""" beam = experiments[0].beam crystal = experiments[0].crystal goniometer = experiments[0].goniometer detector = experiments[0].detector scan = experiments[0].scan # We need a UB matrix at the beginning of every image, and at the end of the # last image. These are all the same - we want to compare the scan-varying # predictor with the scan-static one for a flat scan. ar_range = scan.get_array_range() UBlist = [crystal.get_A() for t in range(ar_range[0], ar_range[1]+1)] dmin = detector.get_max_resolution(beam.get_s0()) from dials.algorithms.spot_prediction import ScanVaryingReflectionPredictor sv_predictor = ScanVaryingReflectionPredictor(experiments[0], dmin=dmin) refs = sv_predictor.for_ub(flex.mat3_double(UBlist)) return refs
def __init__(self, mosaic_domains=25, mosaic_spread_deg=0.05): local_data = data() direct_algo_res_limit = 1.7 wavelength_A = 1.3 # Angstroms, dummy value GF = gen_fmodel(resolution=direct_algo_res_limit, pdb_text=local_data.get("pdb_lines"), algorithm="fft", wavelength=wavelength_A) GF.set_k_sol(0.435) GF.make_P1_primitive() self.sfall_main = GF.get_amplitudes() SIM = nanoBragg( detpixels_slowfast=(3000, 3000), pixel_size_mm=0.11, Ncells_abc=(10, 10, 10), # workaround for problem with wavelength array, specify it separately in constructor. wavelength_A=wavelength_A, verbose=0) SIM.mosaic_spread_deg = mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = mosaic_domains # mosaic_domains setter must come after mosaic_spread_deg setter UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution( mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) SIM.set_mosaic_blocks(UMAT_nm) self.SIM = SIM
def run_sim2smv(prefix, crystal, spectra, rotation, rank, quick=False): local_data = data() smv_fileout = prefix + ".img" if quick is not True: if not write_safe(smv_fileout): print("File %s already exists, skipping in rank %d" % (smv_fileout, rank)) return direct_algo_res_limit = 1.7 wavlen, flux, wavelength_A = next( spectra) # list of lambdas, list of fluxes, average wavelength if quick: wavlen = flex.double([wavelength_A]) flux = flex.double([flex.sum(flux)]) print("Quick sim, lambda=%f, flux=%f" % (wavelength_A, flux[0])) GF = gen_fmodel(resolution=direct_algo_res_limit, pdb_text=local_data.get("pdb_lines"), algorithm="fft", wavelength=wavelength_A) GF.set_k_sol(0.435) GF.make_P1_primitive() sfall_main = GF.get_amplitudes() # use crystal structure to initialize Fhkl array sfall_main.show_summary(prefix="Amplitudes used ") N = crystal.number_of_cells(sfall_main.unit_cell()) #SIM = nanoBragg(detpixels_slowfast=(2000,2000),pixel_size_mm=0.11,Ncells_abc=(5,5,5),verbose=0) SIM = nanoBragg( detpixels_slowfast=(3000, 3000), pixel_size_mm=0.11, Ncells_abc=(N, N, N), # workaround for problem with wavelength array, specify it separately in constructor. wavelength_A=wavelength_A, verbose=0) SIM.adc_offset_adu = 0 # Do not offset by 40 SIM.adc_offset_adu = 10 # Do not offset by 40 import sys if len(sys.argv) > 2: SIM.seed = -int(sys.argv[2]) print("GOTHERE seed=", SIM.seed) if len(sys.argv) > 1: if sys.argv[1] == "random": SIM.randomize_orientation() SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = n_mosaic_domains # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image # 3000000 images would be 100000 hours on a 60-core machine (dials), or 11.4 years # using 2 nodes, 5.7 years. Do this at SLAC? NERSC? combination of all? # SLAC downtimes: Tues Dec 5 (24 hrs), Mon Dec 11 (72 hrs), Mon Dec 18 light use, 24 days # mosaic_domains setter must come after mosaic_spread_deg setter SIM.distance_mm = 141.7 UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) SIM.set_mosaic_blocks(UMAT_nm) #SIM.detector_thick_mm = 0.5 # = 0 for Rayonix #SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD #SIM.detector_attenuation_length_mm = default is silicon # get same noise each time this test is run SIM.seed = 1 SIM.oversample = 1 SIM.wavelength_A = wavelength_A SIM.polarization = 1 # this will become F000, marking the beam center SIM.default_F = 0 #SIM.missets_deg= (10,20,30) print("mosaic_seed=", SIM.mosaic_seed) print("seed=", SIM.seed) print("calib_seed=", SIM.calib_seed) print("missets_deg =", SIM.missets_deg) SIM.Fhkl = sfall_main print("Determinant", rotation.determinant()) Amatrix_rot = ( rotation * sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose() print("RAND_ORI", prefix, end=' ') for i in Amatrix_rot: print(i, end=' ') print() SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter print("unit_cell_Adeg=", SIM.unit_cell_Adeg) print("unit_cell_tuple=", SIM.unit_cell_tuple) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM from cctbx import crystal_orientation Ori = crystal_orientation.crystal_orientation( Amat, crystal_orientation.basis_type.reciprocal) print("Python unit cell from SIM state", Ori.unit_cell()) # fastest option, least realistic #SIM.xtal_shape=shapetype.Tophat # RLP = hard sphere #SIM.xtal_shape=shapetype.Square # gives fringes SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian #SIM.xtal_shape=shapetype.Round # Crystal is a hard sphere # only really useful for long runs SIM.progress_meter = False # prints out value of one pixel only. will not render full image! #SIM.printout_pixel_fastslow=(500,500) #SIM.printout=True SIM.show_params() # flux is always in photons/s SIM.flux = 1e12 SIM.exposure_s = 1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense temp = SIM.Ncells_abc print("Ncells_abc=", SIM.Ncells_abc) SIM.Ncells_abc = temp print("Ncells_abc=", SIM.Ncells_abc) print("xtal_size_mm=", SIM.xtal_size_mm) print("unit_cell_Adeg=", SIM.unit_cell_Adeg) print("unit_cell_tuple=", SIM.unit_cell_tuple) print("missets_deg=", SIM.missets_deg) print("Amatrix=", SIM.Amatrix) print("beam_center_mm=", SIM.beam_center_mm) print("XDS_ORGXY=", SIM.XDS_ORGXY) print("detector_pivot=", SIM.detector_pivot) print("xtal_shape=", SIM.xtal_shape) print("beamcenter_convention=", SIM.beamcenter_convention) print("fdet_vector=", SIM.fdet_vector) print("sdet_vector=", SIM.sdet_vector) print("odet_vector=", SIM.odet_vector) print("beam_vector=", SIM.beam_vector) print("polar_vector=", SIM.polar_vector) print("spindle_axis=", SIM.spindle_axis) print("twotheta_axis=", SIM.twotheta_axis) print("distance_meters=", SIM.distance_meters) print("distance_mm=", SIM.distance_mm) print("close_distance_mm=", SIM.close_distance_mm) print("detector_twotheta_deg=", SIM.detector_twotheta_deg) print("detsize_fastslow_mm=", SIM.detsize_fastslow_mm) print("detpixels_fastslow=", SIM.detpixels_fastslow) print("detector_rot_deg=", SIM.detector_rot_deg) print("curved_detector=", SIM.curved_detector) print("pixel_size_mm=", SIM.pixel_size_mm) print("point_pixel=", SIM.point_pixel) print("polarization=", SIM.polarization) print("nopolar=", SIM.nopolar) print("oversample=", SIM.oversample) print("region_of_interest=", SIM.region_of_interest) print("wavelength_A=", SIM.wavelength_A) print("energy_eV=", SIM.energy_eV) print("fluence=", SIM.fluence) print("flux=", SIM.flux) print("exposure_s=", SIM.exposure_s) print("beamsize_mm=", SIM.beamsize_mm) print("dispersion_pct=", SIM.dispersion_pct) print("dispsteps=", SIM.dispsteps) print("divergence_hv_mrad=", SIM.divergence_hv_mrad) print("divsteps_hv=", SIM.divsteps_hv) print("divstep_hv_mrad=", SIM.divstep_hv_mrad) print("round_div=", SIM.round_div) print("phi_deg=", SIM.phi_deg) print("osc_deg=", SIM.osc_deg) print("phisteps=", SIM.phisteps) print("phistep_deg=", SIM.phistep_deg) print("detector_thick_mm=", SIM.detector_thick_mm) print("detector_thicksteps=", SIM.detector_thicksteps) print("detector_thickstep_mm=", SIM.detector_thickstep_mm) print("***mosaic_spread_deg=", SIM.mosaic_spread_deg) print("***mosaic_domains=", SIM.mosaic_domains) print("indices=", SIM.indices) print("amplitudes=", SIM.amplitudes) print("Fhkl_tuple=", SIM.Fhkl_tuple) print("default_F=", SIM.default_F) print("interpolate=", SIM.interpolate) print("integral_form=", SIM.integral_form) from libtbx.development.timers import Profiler P = Profiler("nanoBragg") # now actually burn up some CPU #SIM.add_nanoBragg_spots() del P # simulated crystal is only 125 unit cells (25 nm wide) # amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume) print(crystal.domains_per_crystal) SIM.raw_pixels *= crystal.domains_per_crystal # must calculate the correct scale! # Use single wavelength for all energy channels for the purpose of Fcalc wavelength_hi_remote = wavlen[-1] GF.reset_wavelength(wavelength_hi_remote) GF.reset_specific_at_wavelength(label_has="FE1", tables=local_data.get("Fe_oxidized_model"), newvalue=wavelength_hi_remote) GF.reset_specific_at_wavelength(label_has="FE2", tables=local_data.get("Fe_reduced_model"), newvalue=wavelength_hi_remote) sfall_channel = GF.get_amplitudes() # sources channel_source_XYZ = flex.vec3_double(len(flux), SIM.xray_source_XYZ[0]) CP = channel_pixels( wavlen, flux, channel_source_XYZ) # class interface for multi-wavelength print("+++++++++++++++++++++++++++++++++++++++ Multiwavelength call") CH = CP(N=N, UMAT_nm=UMAT_nm, Amatrix_rot=Amatrix_rot, sfall_channel=sfall_channel) SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal CH.free_all() if quick: SIM.to_smv_format(fileout=prefix + "_intimage_001.img") # rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor bg = flex.vec2_double([(0, 2.57), (0.0365, 2.58), (0.07, 2.8), (0.12, 5), (0.162, 8), (0.2, 6.75), (0.18, 7.32), (0.216, 6.75), (0.236, 6.5), (0.28, 4.5), (0.3, 4.3), (0.345, 4.36), (0.436, 3.77), (0.5, 3.17)]) SIM.Fbg_vs_stol = bg SIM.amorphous_sample_thick_mm = 0.1 SIM.amorphous_density_gcm3 = 1 SIM.amorphous_molecular_weight_Da = 18 SIM.flux = 1e12 SIM.beamsize_mm = 0.003 # square (not user specified) SIM.exposure_s = 1.0 # multiplies flux x exposure SIM.add_background() if quick: SIM.to_smv_format(fileout=prefix + "_intimage_002.img") # rough approximation to air bg = flex.vec2_double([(0, 14.1), (0.045, 13.5), (0.174, 8.35), (0.35, 4.78), (0.5, 4.22)]) SIM.Fbg_vs_stol = bg #SIM.amorphous_sample_thick_mm = 35 # between beamstop and collimator SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator SIM.amorphous_density_gcm3 = 1.2e-3 SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2 print("amorphous_sample_size_mm=", SIM.amorphous_sample_size_mm) print("amorphous_sample_thick_mm=", SIM.amorphous_sample_thick_mm) print("amorphous_density_gcm3=", SIM.amorphous_density_gcm3) print("amorphous_molecular_weight_Da=", SIM.amorphous_molecular_weight_Da) SIM.add_background() #apply beamstop mask here # set this to 0 or -1 to trigger automatic radius. could be very slow with bright images # settings for CCD SIM.detector_psf_kernel_radius_pixels = 5 #SIM.detector_psf_fwhm_mm=0.08; #SIM.detector_psf_type=shapetype.Fiber # rayonix=Fiber, CSPAD=None (or small Gaussian) SIM.detector_psf_type = shapetype.Unknown # for CSPAD SIM.detector_psf_fwhm_mm = 0 #SIM.apply_psf() print("One pixel-->", SIM.raw_pixels[500000]) # at this point we scale the raw pixels so that the output array is on an scale from 0 to 50000. # that is the default behavior (intfile_scale<=0), otherwise it applies intfile_scale as a multiplier on an abs scale. if quick: SIM.to_smv_format(fileout=prefix + "_intimage_003.img") print("quantum_gain=", SIM.quantum_gain) #defaults to 1. converts photons to ADU print("adc_offset_adu=", SIM.adc_offset_adu) print("detector_calibration_noise_pct=", SIM.detector_calibration_noise_pct) print("flicker_noise_pct=", SIM.flicker_noise_pct) print("readout_noise_adu=", SIM.readout_noise_adu ) # gaussian random number to add to every pixel (0 for PAD) # apply Poissonion correction, then scale to ADU, then adc_offset. # should be 10 for most Rayonix, Pilatus should be 0, CSPAD should be 0. print("detector_psf_type=", SIM.detector_psf_type) print("detector_psf_fwhm_mm=", SIM.detector_psf_fwhm_mm) print("detector_psf_kernel_radius_pixels=", SIM.detector_psf_kernel_radius_pixels) SIM.add_noise() #converts phtons to ADU. print("raw_pixels=", SIM.raw_pixels) extra = "PREFIX=%s;\nRANK=%d;\n" % (prefix, rank) SIM.to_smv_format_py(fileout=smv_fileout, intfile_scale=1, rotmat=True, extra=extra, gz=True) # try to write as CBF if False: import dxtbx from dxtbx.format.FormatCBFMiniPilatus import FormatCBFMiniPilatus img = dxtbx.load(prefix + ".img") print(img) FormatCBFMiniPilatus.as_file(detector=img.get_detector(), beam=img.get_beam(), gonio=img.get_goniometer(), scan=img.get_scan(), data=img.get_raw_data(), path=prefix + ".cbf") SIM.free_all()
def run_sim2smv(Nshot_max, odir, prefix, rank, n_jobs, save_bragg=False, save_smv=True, save_h5 =False, return_pixels=False): from six.moves import range, StringIO from six.moves import cPickle as pickle from cxid9114.sim import sim_utils import os import h5py import math import sys import numpy as np from IPython import embed from cxid9114.bigsim.bigsim_geom import DET,BEAM import scitbx from scitbx.array_family import flex from scitbx.matrix import sqr,col from simtbx.nanoBragg import shapetype from simtbx.nanoBragg import nanoBragg import libtbx.load_env # possibly implicit from libtbx.development.timers import Profiler from cctbx import crystal,crystal_orientation from LS49.sim.step4_pad import microcrystal from cxid9114 import utils from cxid9114.parameters import ENERGY_CONV, ENERGY_HIGH, ENERGY_LOW from cxid9114.bigsim import sim_spectra odir_j = os.path.join( odir, "job%d" % rank) if not os.path.exists(odir_j): os.makedirs(odir_j) add_noise = False add_background = False overwrite = True #$False sample_thick_mm = 0.005 # 50 micron GDVN nozzle makes a ~5ish micron jet air_thick_mm =0 # mostly vacuum, maybe helium layer of 1 micron flux_ave=2e11 add_spots_algorithm="cuda" big_data = "." # directory location for reference files detpixels_slowfast = (1800,1800) pixsize_mm=0.11 distance_mm = 125 offset_adu=30 mos_spread_deg=0.015 mos_doms=1000 beam_size_mm=0.001 exposure_s=1 use_microcrystal=True #False Ncells_abc=(120,120,120) Deff_A = 2200 length_um = 2.2 timelog = False background = utils.open_flex("background") crystal = microcrystal(Deff_A = Deff_A, length_um = length_um, beam_diameter_um = beam_size_mm*1000, verbose=False) spec_file = h5py.File("simMe_data_run62.h5", "r") spec_data = spec_file["hist_spec"] Umat_data = spec_file["Umats"] en_chans = spec_file["energy_bins"][()] ilow = np.abs(en_chans - ENERGY_LOW).argmin() ihigh = np.abs(en_chans - ENERGY_HIGH).argmin() wave_chans = ENERGY_CONV/en_chans sfall_main = sim_spectra.load_spectra("test_sfall.h5") Nshot = spec_data.shape[0] idx_range = np.array_split(np.arange(Nshot), n_jobs) Nshot_per_job = len(idx_range[rank]) if Nshot_max > 0 : Nshot_per_job = min( Nshot_max, Nshot_per_job) print ("Job %d: Simulating %d shots" % (rank, Nshot_per_job)) istart = idx_range[rank][0] istop = istart + Nshot_per_job smi_stride = 10 for idx in range( istart, istop): print ("<><><><><><><><><><><><><><>") print ("Job %d; Image %d (%d - %d)" % (rank, idx+1, istart, istop)) print ("<><><><><><><><><><><><><><>") smv_fileout = os.path.join( odir_j, prefix % idx + ".img") h5_fileout = smv_fileout + ".h5" if os.path.exists(smv_fileout) and not overwrite and save_smv: print("Shot %s exists: moving on" % smv_fileout) continue if os.path.exists(h5_fileout) and not overwrite and save_h5: print("Shot %s exists: moving on" % h5_fileout) continue if (rank==0 and idx % smi_stride==0): print("GPU status") os.system("nvidia-smi") print("\n\n") print("CPU memory usage") mem_usg= """ps -U dermen --no-headers -o rss | awk '{ sum+=$1} END {print int(sum/1024) "MB consumed by CPU user"}'""" os.system(mem_usg) spec = spec_data[2] rotation = sqr(Umat_data[2]) wavelength_A = np.mean(wave_chans) spectra = iter([(wave_chans, spec, wavelength_A)]) direct_algo_res_limit = 1.7 wavlen, flux, wavelength_A = next(spectra) # list of lambdas, list of fluxes, average wavelength assert wavelength_A > 0 assert (len(wavlen)==len(flux)==len(sfall_main)) N = crystal.number_of_cells(sfall_main[0].unit_cell()) if use_mcrocrystal: Ncells_abc = (N,N,N) flux *= 0 flux[ilow] = 1e12 flux[ihigh]=1e12 UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=mos_spread_deg * math.pi/180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(mos_doms) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) ) Amatrix_rot = (rotation * sqr(sfall_main[0].unit_cell().orthogonalization_matrix())).transpose() #SIM = nanoBragg(detpixels_slowfast=detpixels_slowfast, # pixel_size_mm=pixsize_mm,Ncells_abc=Ncells_abc, # wavelength_A=wavelength_A,verbose=verbose) SIM = nanoBragg(detector=DET, beam=BEAM, panel_id=0,verbose=verbose) SIM.adc_offset_adu = offset_adu # Do not offset by 40 SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = mos_doms SIM.distance_mm=distance_mm SIM.set_mosaic_blocks(UMAT_nm) SIM.seed = 1 SIM.oversample=1 SIM.wavelength_A = wavelength_A SIM.polarization=1 SIM.default_F=0 SIM.Fhkl=sfall_main[0].amplitudes() SIM.progress_meter=False SIM.flux=flux_ave SIM.exposure_s=exposure_s SIM.beamsize_mm=beam_size_mm SIM.Ncells_abc=Ncells_abc SIM.xtal_shape=shapetype.Gauss idxpath = "try3_idx2/job0/dump_0_data.pkl" Amat = sim_utils.Amatrix_dials2nanoBragg(utils.open_flex(idxpath)["crystalAB"]) #SIM.Umatrix = rotation.elems #SIM.unit_cell_Adeg = sfall_main[0].unit_cell() #SIM2 = nanoBragg(detector=DET, beam=BEAM, panel_id=0,verbose=verbose) #SIM2.adc_offset_adu = offset_adu # Do not offset by 40 #SIM2.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev #SIM2.mosaic_domains = mos_doms #SIM2.distance_mm=distance_mm #SIM2.set_mosaic_blocks(UMAT_nm) #SIM2.seed = 1 #SIM2.oversample=1 #SIM2.wavelength_A = wavelength_A #SIM2.polarization=1 #SIM2.default_F=0 #SIM2.Fhkl=sfall_main[0].amplitudes() #SIM2.progress_meter=False #SIM2.flux=flux_ave #SIM2.exposure_s=exposure_s #SIM2.beamsize_mm=beam_size_mm #SIM2.Ncells_abc=Ncells_abc #SIM2.xtal_shape=shapetype.Gauss #SIM2.Umatrix = rotation.elems #SIM2.unit_cell_Adeg = sfall_main[0].unit_cell() #SIM.Amatrix_RUB = Amatrix_rot #SIM.Amatrix = Amatrix_rot.inverse() raw_pixel_sum = flex.double(len(SIM.raw_pixels)) Nflux = len(flux) print("Beginning the loop") for x in range(Nflux): if x % 10==0: print("+++++++++++++++++++++++++++++++++++++++ Wavelength %d / %d" % (x+1, Nflux) , end="\r") if flux[x] ==0: continue #SIM.Amatrix = Amatrix_rot.inverse() print (SIM.Ncells_abc) SIM.wavelength_A=wavlen[x] SIM.flux=flux[x] SIM.Fhkl=sfall_main[x].amplitudes() SIM.Amatrix = Amat#Amatrix_rot.inverse() #sim_utils.compare_sims(SIM, SIM2) #SIM.Ncells_abc=Ncells_abc #SIM.adc_offset_adu = offset_adu #SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev #SIM.mosaic_domains = mos_doms # #SIM.distance_mm=distance_mm #SIM.set_mosaic_blocks(UMAT_nm) #SIM.seed = 1 #SIM.polarization=1 #SIM.default_F=0 #SIM.xtal_shape=shapetype.Gauss #SIM.progress_meter=False #SIM.exposure_s = exposure_s #SIM.beamsize_mm=beam_size_mm SIM.timelog=timelog SIM.device_Id=rank SIM.raw_pixels *= 0 # just in case! SIM.add_nanoBragg_spots_cuda() if use_microcrystal: raw_pixel_sum += SIM.raw_pixels * crystal.domains_per_crystal print() SIM.raw_pixels = raw_pixel_sum if add_background: SIM.raw_pixels = SIM.raw_pixels + background SIM.detector_psf_kernel_radius_pixels=5; #SIM.detector_psf_fwhm_mm=0.08; #SIM.detector_psf_type=shapetype.Fiber # rayonix=Fiber, CSPAD=None (or small Gaussian) SIM.detector_psf_type=shapetype.Unknown # for CSPAD SIM.detector_psf_fwhm_mm=0 SIM.quantum_gain = 28. #SIM.apply_psf() if add_noise: SIM.add_noise() #converts phtons to ADU. extra = "PREFIX=%s;\nRANK=%d;\n"%(prefix,rank) out = SIM.raw_pixels.as_numpy_array() if save_smv: SIM.to_smv_format_py(fileout=smv_fileout,intfile_scale=1,rotmat=True,extra=extra,gz=True) elif save_h5: f = h5py.File(h5_fileout, "w") f.create_dataset("bigsim_d9114", data=SIM.raw_pixels.as_numpy_array().astype(np.uint16).reshape(detpixels_slowfast), compression="lzf") f.close() if npout is not None: np.save(npout, SIM.raw_pixels.as_numpy_array()) SIM.free_all()
def perform_simulations(self, spectrum, crystal, tophat_spectrum=True): #borrow code from util_partiality. #def run_sim2smv(ROI,prefix,crystal,spectra,rotation,rank,tophat_spectrum=True,quick=False): direct_algo_res_limit = 1.7 self.wavlen, self.flux, self.wavelength_A = next( spectrum) # list of lambdas, list of fluxes, average wavelength if tophat_spectrum: sum_flux = flex.sum(self.flux) ave_flux = sum_flux / 60. # 60 energy channels for ix in range(len(self.wavlen)): energy = 12398.425 / self.wavlen[ix] if energy >= 7090 and energy <= 7150: self.flux[ix] = ave_flux else: self.flux[ix] = 0. # use crystal structure to initialize Fhkl array self.sfall_main.show_summary(prefix="Amplitudes used ") N = crystal.number_of_cells(self.sfall_main.unit_cell()) self.crystal = crystal # delete later self.N = N # delete later SIM = nanoBragg( detpixels_slowfast=(3000, 3000), pixel_size_mm=0.11, Ncells_abc=(N, N, N), # workaround for problem with wavelength array, specify it separately in constructor. wavelength_A=self.wavelength_A, verbose=0) self.SIM = SIM SIM.adc_offset_adu = 10 # Do not offset by 40 SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = 50 # mosaic_domains setter must come after mosaic_spread_deg setter #manuscript says 200, June 15 abc_cov was done with 50 SIM.distance_mm = 141.7 UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution( mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) SIM.set_mosaic_blocks(UMAT_nm) self.UMAT_nm = UMAT_nm # delete later # get same noise each time this test is run SIM.seed = 1 SIM.oversample = 1 SIM.wavelength_A = self.wavelength_A SIM.polarization = 1 # this will become F000, marking the beam center SIM.default_F = 0 #SIM.Fhkl=self.sfall_main # no it turns out we don't use these calculations for abc_coverage # use local file with (open(something,"wb")) as F: with (open("confirm_sfall_P1_7122_amplitudes.pickle", "rb")) as F: sfall_P1_7122_amplitudes = pickle.load(F) SIM.Fhkl = sfall_P1_7122_amplitudes SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian SIM.progress_meter = False SIM.show_params() # flux is always in photons/s SIM.flux = 1e12 SIM.exposure_s = 1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense temp = SIM.Ncells_abc print("Ncells_abc=", SIM.Ncells_abc) SIM.Ncells_abc = temp
def __init__(self, model_parameterisations, sigma): """model_parameterisations is a list of CrystalUnitCellParameterisations sigma is a sequence of 6 elements giving the 'sigma' for each of the unit cell parameters, from which weights for the residuals will be calculated. Values of zero in sigma will remove the restraint for the cell parameter at that position""" self._xlucp = model_parameterisations self._nxls = len(model_parameterisations) # common factors used in gradient calculations self._meangradfac = 1./self._nxls self._gradfac = (1. - self._meangradfac) self._weights = [] # initially want to calculate all gradients self._sel = [True] * 6 # identify any cell dimensions constrained to be equal. If any are and a # restraint has been requested for that cell dimension, remove the restraint # for all crystals and warn in the log msg = ('Unit cell similarity restraints were requested for both the ' '{0} and {1} dimensions, however for the crystal in experiment ' '{2} these are constrained to be equal. Only the strongest ' 'of these restraints will be retained for all crystals in ' 'the restrained group.') for ixl, xlucp in enumerate(self._xlucp): B = xlucp.get_state() dB_dp = flex.mat3_double(xlucp.get_ds_dp()) ccg = CalculateCellGradients(B, dB_dp) grads = [ccg.da_dp(), ccg.db_dp(), ccg.dc_dp()] a, b, c, aa, bb, cc = xlucp.get_model().get_unit_cell().parameters() if abs(a - b) < 1e-10: grad_diff = [abs(e1 - e2) for (e1, e2) in zip(grads[0], grads[1])] if max(grad_diff) < 1e-10: # a and b are equal for this crystal, therefore keep only the # strongest requested restraint if sigma[0] > 0.0 and sigma[1] > 0.0: warning(msg.format('a', 'b', xlucp.get_experiment_ids()[0])) strong, weak = sorted([sigma[0], sigma[1]]) sigma[0] = strong sigma[1] = 0.0 if abs(a - c) < 1e-10: grad_diff = [abs(e1 - e2) for (e1, e2) in zip(grads[0], grads[2])] if max(grad_diff) < 1e-10: # a and c are equal for this crystal, therefore keep only the # strongest requested restraint if sigma[0] > 0.0 and sigma[2] > 0.0: warning(msg.format('a', 'c', xlucp.get_experiment_ids()[0])) strong, weak = sorted([sigma[0], sigma[2]]) sigma[0] = strong sigma[2] = 0.0 if abs(b - c) < 1e-10: grad_diff = [abs(e1 - e2) for (e1, e2) in zip(grads[1], grads[2])] if max(grad_diff) < 1e-10: # b and c are equal for this crystal, therefore keep only the # strongest requested restraint if sigma[1] > 0.0 and sigma[2] > 0.0: strong, weak = sorted([sigma[1], sigma[2]]) sigma[1] = strong sigma[2] = 0.0 # A gradient of zero indicates that cell parameter is constrained and thus # to be ignored in restraints #_sigma = [] msg = ('Unit cell similarity restraints were requested for the {0} ' 'parameter, however for the crystal in experiment {1}, {0} is ' 'constrained. This restraint will be removed for all crystals in ' 'the restrained group.') for i, (grad, pname) in enumerate(zip(grads, ['a', 'b', 'c', 'alpha', 'beta', 'gamma'])): tst = [abs(g) <= 1.e-10 for g in grad] if all(tst): # this parameter is constrained, so remove any requested restraints # at this position if sigma[i] > 0.0: warning(msg.format(xlucp.get_experiment_ids()[0], pname[i])) sigma[i] = 0.0 # set the selection for gradient calculations to the unconstrained parameters self._sel = [s > 0.0 for s in sigma] self.nrestraints_per_cell = self._sel.count(True) # repeat the weights for each unit cell being restrained weights = [1./s**2 for s in sigma if s > 0.0] weights = [flex.double(self._nxls, w) for w in weights] self._weights = weights[0] for w in weights[1:]: self._weights.extend(w) return
def run_sim2smv(prefix, crystal, spectra, rotation, rank, gpu_channels_singleton, params, quick=False, save_bragg=False, sfall_channels=None): smv_fileout = prefix + ".img" burst_buffer_expand_dir = os.path.expandvars(params.logger.outdir) burst_buffer_fileout = os.path.join(burst_buffer_expand_dir, smv_fileout) reference_fileout = os.path.join(".", smv_fileout) if not quick: if not write_safe(reference_fileout): print("File %s already exists, skipping in rank %d" % (reference_fileout, rank)) return direct_algo_res_limit = 1.7 wavlen, flux, wavelength_A = next( spectra) # list of lambdas, list of fluxes, average wavelength assert wavelength_A > 0 # os.system("nvidia-smi") # printout might severely impact performance # use crystal structure to initialize Fhkl array N = crystal.number_of_cells(sfall_channels[0].unit_cell()) #SIM = nanoBragg(detpixels_slowfast=(2000,2000),pixel_size_mm=0.11,Ncells_abc=(5,5,5),verbose=0) SIM = nanoBragg( detpixels_slowfast=(3840, 3840), pixel_size_mm=0.088, Ncells_abc=(N, N, N), # workaround for problem with wavelength array, specify it separately in constructor. wavelength_A=wavelength_A, verbose=0) SIM.adc_offset_adu = 0 # Do not offset by 40 #SIM.adc_offset_adu = 10 # Do not offset by 40 SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev # mosaic_domains setter MUST come after mosaic_spread_deg setter SIM.mosaic_domains = int(os.environ.get("MOS_DOM", "25")) print("MOSAIC", SIM.mosaic_domains) SIM.distance_mm = 141.7 UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) SIM.set_mosaic_blocks(UMAT_nm) if params.attenuation: SIM.detector_thick_mm = 0.032 # = 0 for Rayonix SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD SIM.detector_attenuation_length_mm = 0.017 # default is silicon # get same noise each time this test is run SIM.seed = 1 SIM.oversample = 1 SIM.wavelength_A = wavelength_A SIM.polarization = 1 # this will become F000, marking the beam center SIM.default_F = 0 SIM.Fhkl = sfall_channels[0] # instead of sfall_main Amatrix_rot = (rotation * sqr( sfall_channels[0].unit_cell().orthogonalization_matrix())).transpose() SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter print("unit_cell_Adeg=", SIM.unit_cell_Adeg) print("unit_cell_tuple=", SIM.unit_cell_tuple) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM from cctbx import crystal_orientation Ori = crystal_orientation.crystal_orientation( Amat, crystal_orientation.basis_type.reciprocal) # fastest option, least realistic SIM.xtal_shape = shapetype.Gauss_argchk # both crystal & RLP are Gaussian # only really useful for long runs SIM.progress_meter = False # prints out value of one pixel only. will not render full image! # flux is always in photons/s SIM.flux = 1e12 SIM.exposure_s = 1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense temp = SIM.Ncells_abc SIM.Ncells_abc = temp # rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor water_bg = flex.vec2_double([(0, 2.57), (0.0365, 2.58), (0.07, 2.8), (0.12, 5), (0.162, 8), (0.18, 7.32), (0.2, 6.75), (0.216, 6.75), (0.236, 6.5), (0.28, 4.5), (0.3, 4.3), (0.345, 4.36), (0.436, 3.77), (0.5, 3.17)]) assert [a[0] for a in water_bg] == sorted([a[0] for a in water_bg]) # rough approximation to air air_bg = flex.vec2_double([(0, 14.1), (0.045, 13.5), (0.174, 8.35), (0.35, 4.78), (0.5, 4.22)]) assert [a[0] for a in air_bg] == sorted([a[0] for a in air_bg]) # simulated crystal is only 125 unit cells (25 nm wide) # amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume) SIM.raw_pixels *= crystal.domains_per_crystal # must calculate the correct scale! QQ = Profiler("nanoBragg Bragg spots rank %d" % (rank)) if True: #something new devices_per_node = int(os.environ["DEVICES_PER_NODE"]) SIM.device_Id = rank % devices_per_node assert gpu_channels_singleton.get_deviceID() == SIM.device_Id if gpu_channels_singleton.get_nchannels() == 0: # if uninitialized P = Profiler("Initialize the channels singleton rank %d" % (rank)) for x in range(len(flux)): gpu_channels_singleton.structure_factors_to_GPU_direct( x, sfall_channels[x].indices(), sfall_channels[x].data()) del P import time print("datetime for channels singleton rank %d" % (rank), time.time()) exascale_api = get_exascale("exascale_api", params.context) gpud = get_exascale("gpu_detector", params.context) gpu_simulation = exascale_api(nanoBragg=SIM) gpu_simulation.allocate() gpu_detector = gpud(deviceId=SIM.device_Id, nanoBragg=SIM) gpu_detector.each_image_allocate() # loop over energies for x in range(len(flux)): P = Profiler("USE_EXASCALE_API nanoBragg Python and C++ rank %d" % (rank)) print("USE_EXASCALE_API+++++++++++++++++++++++ Wavelength", x) # from channel_pixels function SIM.wavelength_A = wavlen[x] SIM.flux = flux[x] gpu_simulation.add_energy_channel_from_gpu_amplitudes( x, gpu_channels_singleton, gpu_detector) del P gpu_detector.scale_in_place( crystal.domains_per_crystal) # apply scale directly on GPU SIM.wavelength_A = wavelength_A # return to canonical energy for subsequent background if add_background_algorithm == "cuda": QQ = Profiler("nanoBragg background rank %d" % (rank)) SIM.Fbg_vs_stol = water_bg SIM.amorphous_sample_thick_mm = 0.1 SIM.amorphous_density_gcm3 = 1 SIM.amorphous_molecular_weight_Da = 18 SIM.flux = 1e12 SIM.beamsize_mm = 0.003 # square (not user specified) SIM.exposure_s = 1.0 # multiplies flux x exposure gpu_simulation.add_background(gpu_detector) SIM.Fbg_vs_stol = air_bg SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator SIM.amorphous_density_gcm3 = 1.2e-3 SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2 gpu_simulation.add_background(gpu_detector) # deallocate GPU arrays gpu_detector.write_raw_pixels(SIM) # updates SIM.raw_pixels from GPU gpu_detector.each_image_free() SIM.Amatrix_RUB = Amatrix_rot # return to canonical orientation del QQ if add_background_algorithm in ["jh", "sort_stable"]: QQ = Profiler("nanoBragg background rank %d" % (rank)) SIM.Fbg_vs_stol = water_bg SIM.amorphous_sample_thick_mm = 0.1 SIM.amorphous_density_gcm3 = 1 SIM.amorphous_molecular_weight_Da = 18 SIM.flux = 1e12 SIM.beamsize_mm = 0.003 # square (not user specified) SIM.exposure_s = 1.0 # multiplies flux x exposure SIM.add_background( sort_stable=(add_background_algorithm == "sort_stable")) SIM.Fbg_vs_stol = air_bg SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator SIM.amorphous_density_gcm3 = 1.2e-3 SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2 SIM.add_background( sort_stable=(add_background_algorithm == "sort_stable")) del QQ if params.psf: SIM.detector_psf_kernel_radius_pixels = 10 SIM.detector_psf_type = shapetype.Fiber # for Rayonix SIM.detector_psf_fwhm_mm = 0.08 #SIM.apply_psf() # the actual application is called within the C++ SIM.add_noise() else: #SIM.detector_psf_kernel_radius_pixels=5; SIM.detector_psf_type = shapetype.Unknown # for CSPAD SIM.detector_psf_fwhm_mm = 0 if params.noise: SIM.adc_offset_adu = 10 # Do not offset by 40 SIM.detector_calibration_noise_pct = 1.0 SIM.readout_noise_adu = 1. QQ = Profiler("nanoBragg noise rank %d" % (rank)) if params.noise or params.psf: from LS49.sim.step6_pad import estimate_gain print("quantum_gain=", SIM.quantum_gain) #defaults to 1. converts photons to ADU print("adc_offset_adu=", SIM.adc_offset_adu) print("detector_calibration_noise_pct=", SIM.detector_calibration_noise_pct) print("flicker_noise_pct=", SIM.flicker_noise_pct) print("readout_noise_adu=", SIM.readout_noise_adu ) # gaussian random number to add to every pixel (0 for PAD) # apply Poissonion correction, then scale to ADU, then adc_offset. # should be 10 for most Rayonix, Pilatus should be 0, CSPAD should be 0. print("detector_psf_type=", SIM.detector_psf_type) print("detector_psf_fwhm_mm=", SIM.detector_psf_fwhm_mm) print("detector_psf_kernel_radius_pixels=", SIM.detector_psf_kernel_radius_pixels) #estimate_gain(SIM.raw_pixels,offset=0) SIM.add_noise() #converts photons to ADU. #estimate_gain(SIM.raw_pixels,offset=SIM.adc_offset_adu,algorithm="slow") #estimate_gain(SIM.raw_pixels,offset=SIM.adc_offset_adu,algorithm="kabsch") del QQ print("FULLY") print(burst_buffer_expand_dir, params.logger.outdir) print(burst_buffer_fileout, smv_fileout) extra = "PREFIX=%s;\nRANK=%d;\n" % (prefix, rank) SIM.to_smv_format_py(fileout=burst_buffer_fileout, intfile_scale=1, rotmat=True, extra=extra, gz=True) SIM.free_all()
def __init__(self, rotation,sfall_main, N, mosaic_domains=25, mosaic_spread_deg=0.05, SEED=1, randomize=False): """ :param rotation: :param sfall_main: :param N: :param mosaic_domains: :param mosaic_spread_deg: :param SEED: :param randomize: """ UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution( mean=0, sigma=mosaic_spread_deg * math.pi/180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) ) Amatrix_rot = (rotation * sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose() self.SIM = nanoBragg( detpixels_slowfast=(3000,3000), pixel_size_mm=0.11, Ncells_abc=(N, N, N), wavelength_A=1, # default we will update later@ verbose=0) self.SIM.seed = SEED self.SIM.adc_offset_adu = 10 # Do not offset by 40 self.SIM.mosaic_spread_deg = mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev self.SIM.mosaic_domains = mosaic_domains # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image # mosaic_domains setter must come after mosaic_spread_deg setter self.SIM.distance_mm=141.7 self.SIM.set_mosaic_blocks(UMAT_nm) self.SIM.oversample=1 self.SIM.polarization=1 self.SIM.default_F=1e5 # TODO: in the future we will init the energy dependent F_HKL here #self.SIM.Fhkl = energy_independent_F self.SIM.Amatrix_RUB = Amatrix_rot self.SIM.xtal_shape=shapetype.Gauss # both crystal & RLP are Gaussian self.SIM.progress_meter=False self.SIM.exposure_s=1.0 # so total fluence is e12 self.SIM.beamsize_mm=0.003 #cannot make this 3 microns; spots are too intense if randomize: self.SIM.random_orientation() temp=self.SIM.Ncells_abc print("Ncells_abc=",self.SIM.Ncells_abc) self.SIM.Ncells_abc=temp # FIXME: add the CUDA init script here #initialize_GPU_variables() self.raw_pixels = self.SIM.raw_pixels # FIXME: this will be on GPU
def run_sim2smv(img_prefix=None, simparams=None, pdb_lines=None, crystal=None, spectra=None, rotation=None, rank=None, fsave=None, sfall_cluster=None, quick=False): smv_fileout = fsave direct_algo_res_limit = simparams.direct_algo_res_limit wavlen, flux, real_wavelength_A = next( spectra) # list of lambdas, list of fluxes, average wavelength real_flux = flex.sum(flux) assert real_wavelength_A > 0 # print(rank, " ## real_wavelength_A/real_flux = ", real_wavelength_A, real_flux*1.0/simparams.flux) if quick: wavlen = flex.double([real_wavelength_A]) flux = flex.double([real_flux]) # GF = gen_fmodel(resolution=simparams.direct_algo_res_limit,pdb_text=pdb_lines,algorithm=simparams.fmodel_algorithm,wavelength=real_wavelength_A) # GF.set_k_sol(simparams.k_sol) # GF.make_P1_primitive() sfall_main = sfall_cluster["main"] #GF.get_amplitudes() # use crystal structure to initialize Fhkl array # sfall_main.show_summary(prefix = "Amplitudes used ") N = crystal.number_of_cells(sfall_main.unit_cell()) #print("## number of N = ", N) SIM = nanoBragg(detpixels_slowfast=(simparams.detector_size_ny,simparams.detector_size_nx),pixel_size_mm=simparams.pixel_size_mm,\ Ncells_abc=(N,N,N),wavelength_A=real_wavelength_A,verbose=0) # workaround for problem with wavelength array, specify it separately in constructor. # SIM.adc_offset_adu = 0 # Do not offset by 40 SIM.adc_offset_adu = 10 # Do not offset by 40 SIM.seed = 0 # SIM.randomize_orientation() SIM.mosaic_spread_deg = simparams.mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = simparams.mosaic_domains # 77 seconds. SIM.distance_mm = simparams.distance_mm ## setup the mosaicity UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) SIM.set_mosaic_blocks(UMAT_nm) ###################### SIM.beamcenter_convention = convention.ADXV SIM.beam_center_mm = (simparams.beam_center_x_mm, simparams.beam_center_y_mm) # 95.975 96.855 ###################### # get same noise each time this test is run SIM.seed = 0 SIM.oversample = simparams.oversample SIM.wavelength_A = real_wavelength_A SIM.polarization = simparams.polarization # this will become F000, marking the beam center SIM.default_F = simparams.default_F #SIM.missets_deg= (10,20,30) SIM.Fhkl = sfall_main Amatrix_rot = ( rotation * sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose() SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter # print("## inside run_sim2smv, Amat_rot = ", Amatrix_rot) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM Ori = crystal_orientation.crystal_orientation( Amat, crystal_orientation.basis_type.reciprocal) print(fsave, "Amatrix_rot", Amatrix_rot) print(fsave, "Amat", Amat) print(fsave, "Ori", Ori) print(fsave, "rotation", rotation) print(fsave, "sqr(sfall_main.unit_cell().orthogonalization_matrix())", sqr(sfall_main.unit_cell().orthogonalization_matrix())) SIM.free_all()
def __init__(self, model_parameterisations, sigma): """model_parameterisations is a list of CrystalUnitCellParameterisations sigma is a sequence of 6 elements giving the 'sigma' for each of the unit cell parameters, from which weights for the residuals will be calculated. Values of zero in sigma will remove the restraint for the cell parameter at that position""" self._xlucp = model_parameterisations self._nxls = len(model_parameterisations) # common factors used in gradient calculations self._meangradfac = 1.0 / self._nxls self._gradfac = 1.0 - self._meangradfac self._weights = [] # initially want to calculate all gradients self._sel = [True] * 6 # identify any cell dimensions constrained to be equal. If any are and a # restraint has been requested for that cell dimension, remove the restraint # for all crystals and warn in the log msg = ("Unit cell similarity restraints were requested for both the " "{0} and {1} dimensions, however for the crystal in experiment " "{2} these are constrained to be equal. Only the strongest " "of these restraints will be retained for all crystals in " "the restrained group.") for ixl, xlucp in enumerate(self._xlucp): B = xlucp.get_state() dB_dp = flex.mat3_double(xlucp.get_ds_dp()) ccg = CalculateCellGradients(B, dB_dp) grads = [ ccg.da_dp(), ccg.db_dp(), ccg.dc_dp(), ccg.daa_dp(), ccg.dbb_dp(), ccg.dcc_dp(), ] a, b, c, aa, bb, cc = xlucp.get_model().get_unit_cell().parameters( ) if abs(a - b) < 1e-10: grad_diff = [ abs(e1 - e2) for (e1, e2) in zip(grads[0], grads[1]) ] if max(grad_diff) < 1e-10: # a and b are equal for this crystal, therefore keep only the # strongest requested restraint if sigma[0] > 0.0 and sigma[1] > 0.0: logger.debug( msg.format("a", "b", xlucp.get_experiment_ids()[0])) strong, weak = sorted([sigma[0], sigma[1]]) sigma[0] = strong sigma[1] = 0.0 if abs(a - c) < 1e-10: grad_diff = [ abs(e1 - e2) for (e1, e2) in zip(grads[0], grads[2]) ] if max(grad_diff) < 1e-10: # a and c are equal for this crystal, therefore keep only the # strongest requested restraint if sigma[0] > 0.0 and sigma[2] > 0.0: logger.debug( msg.format("a", "c", xlucp.get_experiment_ids()[0])) strong, weak = sorted([sigma[0], sigma[2]]) sigma[0] = strong sigma[2] = 0.0 if abs(b - c) < 1e-10: grad_diff = [ abs(e1 - e2) for (e1, e2) in zip(grads[1], grads[2]) ] if max(grad_diff) < 1e-10: # b and c are equal for this crystal, therefore keep only the # strongest requested restraint if sigma[1] > 0.0 and sigma[2] > 0.0: logger.debug( msg.format("b", "c", xlucp.get_experiment_ids()[0])) strong, weak = sorted([sigma[1], sigma[2]]) sigma[1] = strong sigma[2] = 0.0 # A gradient of zero indicates that cell parameter is constrained and thus # to be ignored in restraints # _sigma = [] msg = ( "Unit cell similarity restraints were requested for the {0} " "parameter, however for the crystal in experiment {1}, {0} is " "constrained. This restraint will be removed for all crystals in " "the restrained group.") for i, (grad, pname) in enumerate( zip(grads, ["a", "b", "c", "alpha", "beta", "gamma"])): tst = (abs(g) <= 1.0e-10 for g in grad) if all(tst): # this parameter is constrained, so remove any requested restraints # at this position if sigma[i] > 0.0: logger.debug( msg.format(pname, xlucp.get_experiment_ids()[0])) sigma[i] = 0.0 # set the selection for gradient calculations to the unconstrained parameters self._sel = [s > 0.0 for s in sigma] self.nrestraints_per_cell = self._sel.count(True) # repeat the weights for each unit cell being restrained weights = [1.0 / s**2 for s in sigma if s > 0.0] weights = [flex.double(self._nxls, w) for w in weights] self._weights = weights[0] for w in weights[1:]: self._weights.extend(w)
def _crystal_properties(self): if self.crystal is None: return self.D.xtal_shape = self.crystal.xtal_shape self.update_Fhkl_tuple() ## TODO: am I unnecessary? #self.D.unit_cell_tuple = self.crystal.dxtbx_crystal.get_unit_cell().parameters() if self.using_diffBragg_spots: self.D.Omatrix = self.crystal.Omatrix self.D.Bmatrix = self.crystal.dxtbx_crystal.get_B() # self.D.Umatrix = self.crystal.dxtbx_crystal.get_U() if self.crystal.isotropic_ncells: self.D.Ncells_abc = self.crystal.Ncells_abc[0] else: self.D.Ncells_abc_aniso = self.crystal.Ncells_abc if self.crystal.Ncells_def is not None: self.D.Ncells_def = self.crystal.Ncells_def if self.crystal.anisotropic_mos_spread_deg is not None: mosaicity = self.crystal.anisotropic_mos_spread_deg self.Umats_method = 3 if 3 == len(mosaicity) else 4 crystal = self.crystal.dxtbx_crystal else: mosaicity = self.crystal.mos_spread_deg self.Umats_method = 2 crystal = None self.update_umats(mosaicity, self.crystal.n_mos_domains, crystal) else: # umat implementation for the exascale api self.D.xtal_shape = self.crystal.xtal_shape self.update_Fhkl_tuple() self.D.Amatrix = Amatrix_dials2nanoBragg( self.crystal.dxtbx_crystal) #Nabc = tuple([int(round(x)) for x in self.crystal.Ncells_abc]) Nabc = self.crystal.Ncells_abc if len(Nabc) == 1: Nabc = Nabc[0], Nabc[0], Nabc[0] self.D.Ncells_abc = Nabc if self.umat_maker is None: # initialize the parameterized distribution assert self.crystal.n_mos_domains % 2 == 0 from simtbx.nanoBragg.tst_anisotropic_mosaicity import AnisoUmats as AnisoUmats_exa self.umat_maker = AnisoUmats_exa(num_random_samples=2 * self.crystal.n_mos_domains) if self.crystal.anisotropic_mos_spread_deg is None: # isotropic case self.D.mosaic_spread_deg = self.crystal.mos_spread_deg self.D.mosaic_domains = self.crystal.n_mos_domains if self.Umats_method == 0: # double_random legacy method, exafel tests Umats = SimData.Umats(self.crystal.mos_spread_deg, self.crystal.n_mos_domains, seed=self.mosaic_seeds[0], norm_dist_seed=self.mosaic_seeds[1]) elif self.Umats_method == 5: # double_uniform isotropic Umats, Umats_prime, Umats_dbl_prime = self.umat_maker.generate_isotropic_Umats( self.crystal.mos_spread_deg, compute_derivs=False) else: raise ValueError( "Invalid method for nanoBragg isotropic case") #SimData.plot_isotropic_umats(Umats, self.crystal.mos_spread_deg) else: # anisotropic case with diagonal elements eta_a, eta_b, eta_c = self.crystal.anisotropic_mos_spread_deg eta_tensor = [eta_a, 0, 0, 0, eta_b, 0, 0, 0, eta_c] Umats, Umats_prime, Umats_dbl_prime = self.umat_maker.generate_Umats( eta_tensor, compute_derivs=False, crystal=self.crystal.dxtbx_crystal, how=1) self.exascale_mos_blocks = flex.mat3_double(Umats) self.D.set_mosaic_blocks(self.exascale_mos_blocks)
def run_sim2smv(prefix, crystal, spectra, rotation, rank, gpu_channels_singleton, params, quick=False, save_bragg=False, sfall_channels=None): smv_fileout = prefix + ".img" burst_buffer_expand_dir = os.path.expandvars(params.logger.outdir) burst_buffer_fileout = os.path.join(burst_buffer_expand_dir, smv_fileout) reference_fileout = os.path.join(".", smv_fileout) if not quick: if not write_safe(reference_fileout): print("File %s already exists, skipping in rank %d" % (reference_fileout, rank)) return direct_algo_res_limit = 1.7 wavlen, flux, wavelength_A = next( spectra) # list of lambdas, list of fluxes, average wavelength assert wavelength_A > 0 # use crystal structure to initialize Fhkl array N = crystal.number_of_cells(sfall_channels[0].unit_cell()) #SIM = nanoBragg(detpixels_slowfast=(2000,2000),pixel_size_mm=0.11,Ncells_abc=(5,5,5),verbose=0) SIM = nanoBragg( detpixels_slowfast=(3000, 3000), pixel_size_mm=0.11, Ncells_abc=(N, N, N), # workaround for problem with wavelength array, specify it separately in constructor. wavelength_A=wavelength_A, verbose=0) SIM.adc_offset_adu = 0 # Do not offset by 40 SIM.adc_offset_adu = 10 # Do not offset by 40 import sys if len(sys.argv) > 2: SIM.seed = -int(sys.argv[2]) if len(sys.argv) > 1: if sys.argv[1] == "random": SIM.randomize_orientation() SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = 25 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image # 3000000 images would be 100000 hours on a 60-core machine (dials), or 11.4 years # using 2 nodes, 5.7 years. Do this at SLAC? NERSC? combination of all? # SLAC downtimes: Tues Dec 5 (24 hrs), Mon Dec 11 (72 hrs), Mon Dec 18 light use, 24 days # mosaic_domains setter must come after mosaic_spread_deg setter SIM.distance_mm = 141.7 UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) SIM.set_mosaic_blocks(UMAT_nm) #SIM.detector_thick_mm = 0.5 # = 0 for Rayonix #SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD #SIM.detector_attenuation_length_mm = default is silicon # get same noise each time this test is run SIM.seed = 1 SIM.oversample = 1 SIM.wavelength_A = wavelength_A SIM.polarization = 1 # this will become F000, marking the beam center SIM.default_F = 0 SIM.Fhkl = sfall_channels[0] # instead of sfall_main Amatrix_rot = (rotation * sqr( sfall_channels[0].unit_cell().orthogonalization_matrix())).transpose() SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter print("unit_cell_Adeg=", SIM.unit_cell_Adeg) print("unit_cell_tuple=", SIM.unit_cell_tuple) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM from cctbx import crystal_orientation Ori = crystal_orientation.crystal_orientation( Amat, crystal_orientation.basis_type.reciprocal) # fastest option, least realistic SIM.xtal_shape = shapetype.Gauss_argchk # both crystal & RLP are Gaussian # only really useful for long runs SIM.progress_meter = False # prints out value of one pixel only. will not render full image! # flux is always in photons/s SIM.flux = 1e12 SIM.exposure_s = 1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense temp = SIM.Ncells_abc SIM.Ncells_abc = temp # rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor water_bg = flex.vec2_double([(0, 2.57), (0.0365, 2.58), (0.07, 2.8), (0.12, 5), (0.162, 8), (0.18, 7.32), (0.2, 6.75), (0.216, 6.75), (0.236, 6.5), (0.28, 4.5), (0.3, 4.3), (0.345, 4.36), (0.436, 3.77), (0.5, 3.17)]) assert [a[0] for a in water_bg] == sorted([a[0] for a in water_bg]) # rough approximation to air air_bg = flex.vec2_double([(0, 14.1), (0.045, 13.5), (0.174, 8.35), (0.35, 4.78), (0.5, 4.22)]) assert [a[0] for a in air_bg] == sorted([a[0] for a in air_bg]) # simulated crystal is only 125 unit cells (25 nm wide) # amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume) SIM.raw_pixels *= crystal.domains_per_crystal # must calculate the correct scale! use_exascale_api = os.environ.get("USE_EXASCALE_API", "True") cache_fhkl_on_gpu = os.environ.get("CACHE_FHKL_ON_GPU", "True") assert use_exascale_api in ["True", "False" ] and cache_fhkl_on_gpu in ["True", "False"] use_exascale_api = (use_exascale_api == "True") cache_fhkl_on_gpu = (cache_fhkl_on_gpu == "True") QQ = Profiler("nanoBragg Bragg spots rank %d" % (rank)) if use_exascale_api: #something new devices_per_node = int(os.environ["DEVICES_PER_NODE"]) SIM.device_Id = rank % devices_per_node assert gpu_channels_singleton.get_deviceID() == SIM.device_Id if cache_fhkl_on_gpu: #flag to switch on GPU energy channels if gpu_channels_singleton.get_nchannels() == 0: # if uninitialized P = Profiler("Initialize the channels singleton rank %d" % (rank)) for x in range(len(flux)): gpu_channels_singleton.structure_factors_to_GPU_direct( x, sfall_channels[x].indices(), sfall_channels[x].data()) del P import time print("datetime for channels singleton rank %d" % (rank), time.time()) # allocate GPU arrays SIM.allocate() # loop over energies for x in range(len(flux)): P = Profiler("USE_EXASCALE_API nanoBragg Python and C++ rank %d" % (rank)) print("USE_EXASCALE_API+++++++++++++++++++++++ Wavelength", x) # from channel_pixels function SIM.wavelength_A = wavlen[x] SIM.flux = flux[x] if cache_fhkl_on_gpu: # new interface, use singleton to store all-image energy channels SIM.add_energy_channel_from_gpu_amplitudes( x, gpu_channels_singleton) else: # old interface, host-to-device each energy channel, each image SIM.Fhkl = sfall_channels[x] SIM.add_energy_channel_cuda() del P SIM.scale_in_place( crystal.domains_per_crystal) # apply scale directly on GPU SIM.wavelength_A = wavelength_A # return to canonical energy for subsequent background if add_background_algorithm == "cuda": QQ = Profiler("nanoBragg background rank %d" % (rank)) SIM.Fbg_vs_stol = water_bg SIM.amorphous_sample_thick_mm = 0.1 SIM.amorphous_density_gcm3 = 1 SIM.amorphous_molecular_weight_Da = 18 SIM.flux = 1e12 SIM.beamsize_mm = 0.003 # square (not user specified) SIM.exposure_s = 1.0 # multiplies flux x exposure SIM.add_background() SIM.Fbg_vs_stol = air_bg SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator SIM.amorphous_density_gcm3 = 1.2e-3 SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2 SIM.add_background() # deallocate GPU arrays SIM.get_raw_pixels() # updates SIM.raw_pixels from GPU SIM.deallocate() SIM.Amatrix_RUB = Amatrix_rot # return to canonical orientation del QQ else: for x in range(len(flux)): if rank in [0, 7]: P = Profiler("nanoBragg Python and C++ rank %d" % (rank)) if rank in [0, 7]: print("+++++++++++++++++++++++++++++++++++++++ Wavelength", x) CH = channel_pixels(wavlen[x], flux[x], N, UMAT_nm, Amatrix_rot, rank, sfall_channels[x]) SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal CHDBG_singleton.extract(channel_no=x, data=CH.raw_pixels) CH.free_all() if rank in [0, 7]: del P if add_background_algorithm in ["jh", "sort_stable"]: QQ = Profiler("nanoBragg background rank %d" % (rank)) SIM.Fbg_vs_stol = water_bg SIM.amorphous_sample_thick_mm = 0.1 SIM.amorphous_density_gcm3 = 1 SIM.amorphous_molecular_weight_Da = 18 SIM.flux = 1e12 SIM.beamsize_mm = 0.003 # square (not user specified) SIM.exposure_s = 1.0 # multiplies flux x exposure SIM.add_background( sort_stable=(add_background_algorithm == "sort_stable")) SIM.Fbg_vs_stol = air_bg SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator SIM.amorphous_density_gcm3 = 1.2e-3 SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2 SIM.add_background( sort_stable=(add_background_algorithm == "sort_stable")) del QQ SIM.detector_psf_kernel_radius_pixels = 5 SIM.detector_psf_type = shapetype.Unknown # for CSPAD SIM.detector_psf_fwhm_mm = 0 #SIM.apply_psf() QQ = Profiler("nanoBragg noise rank %d" % (rank)) #SIM.add_noise() #converts phtons to ADU. del QQ extra = "PREFIX=%s;\nRANK=%d;\n" % (prefix, rank) SIM.to_smv_format_py(fileout=burst_buffer_fileout, intfile_scale=1, rotmat=True, extra=extra, gz=True) SIM.free_all()
def test_mat3(): fo = flex.mat3_double(10) as_np = flumpy.to_numpy(fo) for i in range(10): fo[i] = [1, i, 0, 0, 1, 0, i, 0, 1] assert (as_np.reshape(10, 9) == fo).all()
def run_sim2smv(ROI,prefix,crystal,spectra,rotation,rank,quick=False): smv_fileout = prefix + ".img" direct_algo_res_limit = 1.7 wavlen, flux, wavelength_A = next(spectra) # list of lambdas, list of fluxes, average wavelength tophat_spectrum = True if tophat_spectrum: sum_flux = flex.sum(flux) #from IPython import embed; embed() ave_flux = sum_flux/60. # 60 energy channels for ix in range(len(wavlen)): energy = 12398.425 / wavlen[ix] if energy>=7090 and energy <=7150: flux[ix]=ave_flux else: flux[ix]=0. if quick: wavlen = flex.double([wavelength_A]); flux = flex.double([flex.sum(flux)]) print("Quick sim, lambda=%f, flux=%f"%(wavelength_A,flux[0])) GF = gen_fmodel(resolution=direct_algo_res_limit,pdb_text=pdb_lines,algorithm="fft",wavelength=wavelength_A) GF.set_k_sol(0.435) GF.make_P1_primitive() sfall_main = GF.get_amplitudes() # use crystal structure to initialize Fhkl array sfall_main.show_summary(prefix = "Amplitudes used ") N = crystal.number_of_cells(sfall_main.unit_cell()) #SIM = nanoBragg(detpixels_slowfast=(2000,2000),pixel_size_mm=0.11,Ncells_abc=(5,5,5),verbose=0) SIM = nanoBragg(detpixels_slowfast=(3000,3000),pixel_size_mm=0.11,Ncells_abc=(N,N,N), # workaround for problem with wavelength array, specify it separately in constructor. wavelength_A=wavelength_A,verbose=0) SIM.adc_offset_adu = 0 # Do not offset by 40 SIM.adc_offset_adu = 10 # Do not offset by 40 import sys if len(sys.argv)>2: SIM.seed = -int(sys.argv[2]) print("GOTHERE seed=",SIM.seed) if len(sys.argv)>1: if sys.argv[1]=="random" : SIM.randomize_orientation() SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = 50 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image # 3000000 images would be 100000 hours on a 60-core machine (dials), or 11.4 years # using 2 nodes, 5.7 years. Do this at SLAC? NERSC? combination of all? # SLAC downtimes: Tues Dec 5 (24 hrs), Mon Dec 11 (72 hrs), Mon Dec 18 light use, 24 days # mosaic_domains setter must come after mosaic_spread_deg setter SIM.distance_mm=141.7 UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi/180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) ) SIM.set_mosaic_blocks(UMAT_nm) #SIM.detector_thick_mm = 0.5 # = 0 for Rayonix #SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD #SIM.detector_attenuation_length_mm = default is silicon # get same noise each time this test is run SIM.seed = 1 SIM.oversample=1 SIM.wavelength_A = wavelength_A SIM.polarization=1 # this will become F000, marking the beam center SIM.default_F=0 #SIM.missets_deg= (10,20,30) print("mosaic_seed=",SIM.mosaic_seed) print("seed=",SIM.seed) print("calib_seed=",SIM.calib_seed) print("missets_deg =", SIM.missets_deg) SIM.Fhkl=sfall_main print("Determinant",rotation.determinant()) Amatrix_rot = (rotation * sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose() print("RAND_ORI", prefix, end=' ') for i in Amatrix_rot: print(i, end=' ') print() SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter print("unit_cell_Adeg=",SIM.unit_cell_Adeg) print("unit_cell_tuple=",SIM.unit_cell_tuple) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM from cctbx import crystal_orientation Ori = crystal_orientation.crystal_orientation(Amat, crystal_orientation.basis_type.reciprocal) print("Python unit cell from SIM state",Ori.unit_cell()) # fastest option, least realistic #SIM.xtal_shape=shapetype.Tophat # RLP = hard sphere #SIM.xtal_shape=shapetype.Square # gives fringes SIM.xtal_shape=shapetype.Gauss # both crystal & RLP are Gaussian #SIM.xtal_shape=shapetype.Round # Crystal is a hard sphere # only really useful for long runs SIM.progress_meter=False # prints out value of one pixel only. will not render full image! #SIM.printout_pixel_fastslow=(500,500) #SIM.printout=True SIM.show_params() # flux is always in photons/s SIM.flux=1e12 SIM.exposure_s=1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm=0.003 #cannot make this 3 microns; spots are too intense temp=SIM.Ncells_abc print("Ncells_abc=",SIM.Ncells_abc) SIM.Ncells_abc=temp print("Ncells_abc=",SIM.Ncells_abc) print("xtal_size_mm=",SIM.xtal_size_mm) print("unit_cell_Adeg=",SIM.unit_cell_Adeg) print("unit_cell_tuple=",SIM.unit_cell_tuple) print("missets_deg=",SIM.missets_deg) print("Amatrix=",SIM.Amatrix) print("beam_center_mm=",SIM.beam_center_mm) print("XDS_ORGXY=",SIM.XDS_ORGXY) print("detector_pivot=",SIM.detector_pivot) print("xtal_shape=",SIM.xtal_shape) print("beamcenter_convention=",SIM.beamcenter_convention) print("fdet_vector=",SIM.fdet_vector) print("sdet_vector=",SIM.sdet_vector) print("odet_vector=",SIM.odet_vector) print("beam_vector=",SIM.beam_vector) print("polar_vector=",SIM.polar_vector) print("spindle_axis=",SIM.spindle_axis) print("twotheta_axis=",SIM.twotheta_axis) print("distance_meters=",SIM.distance_meters) print("distance_mm=",SIM.distance_mm) print("close_distance_mm=",SIM.close_distance_mm) print("detector_twotheta_deg=",SIM.detector_twotheta_deg) print("detsize_fastslow_mm=",SIM.detsize_fastslow_mm) print("detpixels_fastslow=",SIM.detpixels_fastslow) print("detector_rot_deg=",SIM.detector_rot_deg) print("curved_detector=",SIM.curved_detector) print("pixel_size_mm=",SIM.pixel_size_mm) print("point_pixel=",SIM.point_pixel) print("polarization=",SIM.polarization) print("nopolar=",SIM.nopolar) print("oversample=",SIM.oversample) print("region_of_interest=",SIM.region_of_interest) print("wavelength_A=",SIM.wavelength_A) print("energy_eV=",SIM.energy_eV) print("fluence=",SIM.fluence) print("flux=",SIM.flux) print("exposure_s=",SIM.exposure_s) print("beamsize_mm=",SIM.beamsize_mm) print("dispersion_pct=",SIM.dispersion_pct) print("dispsteps=",SIM.dispsteps) print("divergence_hv_mrad=",SIM.divergence_hv_mrad) print("divsteps_hv=",SIM.divsteps_hv) print("divstep_hv_mrad=",SIM.divstep_hv_mrad) print("round_div=",SIM.round_div) print("phi_deg=",SIM.phi_deg) print("osc_deg=",SIM.osc_deg) print("phisteps=",SIM.phisteps) print("phistep_deg=",SIM.phistep_deg) print("detector_thick_mm=",SIM.detector_thick_mm) print("detector_thicksteps=",SIM.detector_thicksteps) print("detector_thickstep_mm=",SIM.detector_thickstep_mm) print("***mosaic_spread_deg=",SIM.mosaic_spread_deg) print("***mosaic_domains=",SIM.mosaic_domains) print("indices=",SIM.indices) print("amplitudes=",SIM.amplitudes) print("Fhkl_tuple=",SIM.Fhkl_tuple) print("default_F=",SIM.default_F) print("interpolate=",SIM.interpolate) print("integral_form=",SIM.integral_form) from libtbx.development.timers import Profiler P = Profiler("nanoBragg") # now actually burn up some CPU #SIM.add_nanoBragg_spots() del P # simulated crystal is only 125 unit cells (25 nm wide) # amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume) print(crystal.domains_per_crystal) SIM.raw_pixels *= crystal.domains_per_crystal; # must calculate the correct scale! output = StringIO() # open("myfile","w") for x in range(0,100,2): #len(flux)): if flux[x]==0.0:continue print("+++++++++++++++++++++++++++++++++++++++ Wavelength",x) CH = channel_pixels(ROI,wavlen[x],flux[x],N,UMAT_nm,Amatrix_rot,GF,output) SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal; print(SIM.raw_pixels) CH.free_all() message = output.getvalue().split() miller = (int(message[4]),int(message[5]),int(message[6])) intensity = float(message[9]); #SIM.to_smv_format(fileout=prefix + "_intimage_001.img") pixels = SIM.raw_pixels roi_pixels = pixels[ROI[1][0]:ROI[1][1], ROI[0][0]:ROI[0][1]] print("Reducing full shape of",pixels.focus(),"to ROI of",roi_pixels.focus()) SIM.free_all() return dict(roi_pixels=roi_pixels,miller=miller,intensity=intensity)
def run_sim2smv(img_prefix=None, simparams=None,pdb_lines=None,crystal=None,spectra=None,rotation=None,rank=None,fsave=None,sfall_cluster=None,quick=False): smv_fileout = fsave direct_algo_res_limit = simparams.direct_algo_res_limit wavlen, flux, real_wavelength_A = next(spectra) # list of lambdas, list of fluxes, average wavelength real_flux = flex.sum(flux) assert real_wavelength_A > 0 # print(rank, " ## real_wavelength_A/real_flux = ", real_wavelength_A, real_flux*1.0/simparams.flux) if quick: wavlen = flex.double([real_wavelength_A]) flux = flex.double([real_flux]) # GF = gen_fmodel(resolution=simparams.direct_algo_res_limit,pdb_text=pdb_lines,algorithm=simparams.fmodel_algorithm,wavelength=real_wavelength_A) # GF.set_k_sol(simparams.k_sol) # GF.make_P1_primitive() sfall_main = sfall_cluster["main"] #GF.get_amplitudes() # use crystal structure to initialize Fhkl array # sfall_main.show_summary(prefix = "Amplitudes used ") N = crystal.number_of_cells(sfall_main.unit_cell()) #print("## number of N = ", N) SIM = nanoBragg(detpixels_slowfast=(simparams.detector_size_ny,simparams.detector_size_nx),pixel_size_mm=simparams.pixel_size_mm,\ Ncells_abc=(N,N,N),wavelength_A=real_wavelength_A,verbose=0) # workaround for problem with wavelength array, specify it separately in constructor. # SIM.adc_offset_adu = 0 # Do not offset by 40 SIM.adc_offset_adu = 10 # Do not offset by 40 SIM.seed = 0 # SIM.randomize_orientation() SIM.mosaic_spread_deg = simparams.mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = simparams.mosaic_domains # 77 seconds. SIM.distance_mm = simparams.distance_mm ## setup the mosaicity UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi/180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) ) SIM.set_mosaic_blocks(UMAT_nm) ###################### SIM.beamcenter_convention=convention.ADXV SIM.beam_center_mm=(simparams.beam_center_x_mm, simparams.beam_center_y_mm) # 95.975 96.855 ###################### # get same noise each time this test is run SIM.seed = 0 SIM.oversample=simparams.oversample SIM.wavelength_A = real_wavelength_A SIM.polarization=simparams.polarization # this will become F000, marking the beam center SIM.default_F=simparams.default_F #SIM.missets_deg= (10,20,30) SIM.Fhkl=sfall_main Amatrix_rot = (rotation * sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose() SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter # print("## inside run_sim2smv, Amat_rot = ", Amatrix_rot) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM Ori = crystal_orientation.crystal_orientation(Amat, crystal_orientation.basis_type.reciprocal) SIM.xtal_shape=shapetype.Gauss # both crystal & RLP are Gaussian SIM.progress_meter=False # SIM.show_params() # flux is always in photons/s SIM.flux=real_flux SIM.exposure_s=simparams.exposure_s # assumes round beam SIM.beamsize_mm=simparams.beamsize_mm #cannot make this 3 microns; spots are too intense temp=SIM.Ncells_abc SIM.Ncells_abc=temp # print("## domains_per_crystal = ", crystal.domains_per_crystal) SIM.raw_pixels *= crystal.domains_per_crystal # must calculate the correct scale! # print("## Initial raw_pixels = ", flex.sum(SIM.raw_pixels)) for x in range(len(flux)): # CH = channel_pixels(wavlen[x],flux[x],N,UMAT_nm,Amatrix_rot,sfall_cluster[x],rank) # print("## in loop wavlen/flux/real_wavelength_A = ", wavlen[x], flux[x]/real_flux, real_wavelength_A) CH = channel_pixels(simparams=simparams,single_wavelength_A=wavlen[x],single_flux=flux[x],N=N,UMAT_nm=UMAT_nm, \ Amatrix_rot=Amatrix_rot,sfall_channel=sfall_cluster[x],rank=rank) SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal CHDBG_singleton.extract(channel_no=x, data=CH.raw_pixels) CH.free_all() # print("## sum raw_pixels after ", x, "is", flex.sum(SIM.raw_pixels)) # rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor bg = flex.vec2_double([(0,2.57),(0.0365,2.58),(0.07,2.8),(0.12,5),(0.162,8),(0.2,6.75),(0.18,7.32),(0.216,6.75),(0.236,6.5),(0.28,4.5),(0.3,4.3),(0.345,4.36),(0.436,3.77),(0.5,3.17)]) SIM.Fbg_vs_stol = bg SIM.amorphous_sample_thick_mm = simparams.water_sample_thick_mm SIM.amorphous_density_gcm3 = simparams.water_density_gcm3 SIM.amorphous_molecular_weight_Da = simparams.water_molecular_weight_Da SIM.flux=real_flux SIM.beamsize_mm=simparams.beamsize_mm # square (not user specified) SIM.exposure_s=simparams.exposure_s # multiplies flux x exposure SIM.add_background() # rough approximation to air bg = flex.vec2_double([(0,14.1),(0.045,13.5),(0.174,8.35),(0.35,4.78),(0.5,4.22)]) SIM.Fbg_vs_stol = bg SIM.amorphous_sample_thick_mm = simparams.air_sample_thick_mm # between beamstop and collimator SIM.amorphous_density_gcm3 = simparams.air_density_gcm3 SIM.amorphous_sample_molecular_weight_Da = simparams.air_molecular_weight_Da # nitrogen = N2 SIM.add_background() #apply beamstop mask here # settings for CCD SIM.detector_psf_kernel_radius_pixels=simparams.detector_psf_kernel_radius_pixels SIM.detector_psf_type=shapetype.Unknown # for CSPAD SIM.detector_psf_fwhm_mm=simparams.detector_psf_fwhm_mm SIM.apply_psf() SIM.add_noise() extra = "PREFIX=%s;\nRANK=%d;\n"%(img_prefix,rank) SIM.to_smv_format_py(fileout=smv_fileout,intfile_scale=1,rotmat=True,extra=extra,gz=True) SIM.free_all()
def run_sim2smv(prefix, crystal, spectra, rotation, rank, sfall_main, quick=False, save_bragg=False, save_smv=True, save_h5=False, return_pixels=False): smv_fileout = prefix + ".img" #if not quick: # if not write_safe(smv_fileout): # print("File %s already exists, skipping in rank %d"%(smv_fileout,rank)) # return direct_algo_res_limit = 1.7 wavlen, flux, wavelength_A = next( spectra) # list of lambdas, list of fluxes, average wavelength assert wavelength_A > 0 assert (len(wavlen) == len(flux) == len(sfall_main)) if quick: wavlen = flex.double([wavelength_A]) flux = flex.double([flex.sum(flux)]) print("Quick sim, lambda=%f, flux=%f" % (wavelength_A, flux[0])) # use crystal structure to initialize Fhkl array #sfall_main[0].show_summary(prefix = "Amplitudes used ") N = crystal.number_of_cells(sfall_main[0].unit_cell()) if use_microcrystal: global Ncells_abc Ncells_abc = (N, N, N) SIM = nanoBragg(detpixels_slowfast=detpixels_slowfast, pixel_size_mm=pixsize_mm, Ncells_abc=Ncells_abc, wavelength_A=wavelength_A, verbose=verbose) SIM.adc_offset_adu = offset_adu # Do not offset by 40 SIM.mosaic_spread_deg = mos_spread_deg # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = mos_doms SIM.distance_mm = distance_mm UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) SIM.set_mosaic_blocks(UMAT_nm) # get same noise each time this test is run SIM.seed = 1 SIM.oversample = 1 SIM.wavelength_A = wavelength_A SIM.polarization = 1 # this will become F000, marking the beam center SIM.default_F = 0 #SIM.missets_deg= (10,20,30) if verbose: print("mosaic_seed=", SIM.mosaic_seed) print("seed=", SIM.seed) print("calib_seed=", SIM.calib_seed) print("missets_deg =", SIM.missets_deg) SIM.Fhkl = sfall_main[0].amplitudes() if verbose: print("Determinant", rotation.determinant()) Amatrix_rot = ( rotation * sqr(sfall_main[0].unit_cell().orthogonalization_matrix())).transpose() if verbose: print("RAND_ORI", prefix, end=' ') for i in Amatrix_rot: print(i, end=' ') print() SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter if verbose: print("unit_cell_Adeg=", SIM.unit_cell_Adeg) print("unit_cell_tuple=", SIM.unit_cell_tuple) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM Ori = crystal_orientation.crystal_orientation( Amat, crystal_orientation.basis_type.reciprocal) if verbose: print("Python unit cell from SIM state", Ori.unit_cell()) SIM.xtal_shape = shapetype.Gauss SIM.progress_meter = False #SIM.show_params() SIM.flux = flux_ave SIM.exposure_s = exposure_s SIM.beamsize_mm = beam_size_mm temp = SIM.Ncells_abc if verbose: print("Ncells_abc=", SIM.Ncells_abc) SIM.Ncells_abc = temp if verbose: print("Ncells_abc=", SIM.Ncells_abc) print("xtal_size_mm=", SIM.xtal_size_mm) print("unit_cell_Adeg=", SIM.unit_cell_Adeg) print("unit_cell_tuple=", SIM.unit_cell_tuple) print("missets_deg=", SIM.missets_deg) print("Amatrix=", SIM.Amatrix) print("beam_center_mm=", SIM.beam_center_mm) print("XDS_ORGXY=", SIM.XDS_ORGXY) print("detector_pivot=", SIM.detector_pivot) print("xtal_shape=", SIM.xtal_shape) print("beamcenter_convention=", SIM.beamcenter_convention) print("fdet_vector=", SIM.fdet_vector) print("sdet_vector=", SIM.sdet_vector) print("odet_vector=", SIM.odet_vector) print("beam_vector=", SIM.beam_vector) print("polar_vector=", SIM.polar_vector) print("spindle_axis=", SIM.spindle_axis) print("twotheta_axis=", SIM.twotheta_axis) print("distance_meters=", SIM.distance_meters) print("distance_mm=", SIM.distance_mm) print("close_distance_mm=", SIM.close_distance_mm) print("detector_twotheta_deg=", SIM.detector_twotheta_deg) print("detsize_fastslow_mm=", SIM.detsize_fastslow_mm) print("detpixels_fastslow=", SIM.detpixels_fastslow) print("detector_rot_deg=", SIM.detector_rot_deg) print("curved_detector=", SIM.curved_detector) print("pixel_size_mm=", SIM.pixel_size_mm) print("point_pixel=", SIM.point_pixel) print("polarization=", SIM.polarization) print("nopolar=", SIM.nopolar) print("oversample=", SIM.oversample) print("region_of_interest=", SIM.region_of_interest) print("wavelength_A=", SIM.wavelength_A) print("energy_eV=", SIM.energy_eV) print("fluence=", SIM.fluence) print("flux=", SIM.flux) print("exposure_s=", SIM.exposure_s) print("beamsize_mm=", SIM.beamsize_mm) print("dispersion_pct=", SIM.dispersion_pct) print("dispsteps=", SIM.dispsteps) print("divergence_hv_mrad=", SIM.divergence_hv_mrad) print("divsteps_hv=", SIM.divsteps_hv) print("divstep_hv_mrad=", SIM.divstep_hv_mrad) print("round_div=", SIM.round_div) print("phi_deg=", SIM.phi_deg) print("osc_deg=", SIM.osc_deg) print("phisteps=", SIM.phisteps) print("phistep_deg=", SIM.phistep_deg) print("detector_thick_mm=", SIM.detector_thick_mm) print("detector_thicksteps=", SIM.detector_thicksteps) print("detector_thickstep_mm=", SIM.detector_thickstep_mm) print("***mosaic_spread_deg=", SIM.mosaic_spread_deg) print("***mosaic_domains=", SIM.mosaic_domains) print("indices=", SIM.indices) print("amplitudes=", SIM.amplitudes) print("Fhkl_tuple=", SIM.Fhkl_tuple) print("default_F=", SIM.default_F) print("interpolate=", SIM.interpolate) print("integral_form=", SIM.integral_form) # simulated crystal is only 125 unit cells (25 nm wide) # amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume) if verbose: print(crystal.domains_per_crystal) SIM.raw_pixels *= crystal.domains_per_crystal # must calculate the correct scale! Nflux = len(flux) for x in range(Nflux): #P = Profiler("nanoBragg Python and C++ rank %d"%(rank)) if x % 10 == 0: print( "+++++++++++++++++++++++++++++++++++++++ Wavelength %d / %d" % (x + 1, Nflux), end="\r") if flux[x] == 0: continue CH = channel_pixels(wavlen[x], flux[x], UMAT_nm, Amatrix_rot, rank, sfall_main[x]) if use_microcrystal: SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal else: SIM.raw_pixels += CH.raw_pixels CH.free_all() #del P print() # image 1: crystal Bragg scatter if quick or save_bragg: SIM.to_smv_format(fileout=prefix + "_intimage_001.img") if save_bragg: raw_to_pickle(SIM.raw_pixels, fileout=prefix + "_dblprec_001.pickle") SIM.raw_pixels = SIM.raw_pixels + background #SIM.raw_pixels = SIM.raw_pixels*0 ## rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor #bg = flex.vec2_double([(0,2.57),(0.0365,2.58),(0.07,2.8),(0.12,5),(0.162,8),(0.2,6.75),(0.18,7.32),(0.216,6.75),(0.236,6.5),(0.28,4.5),(0.3,4.3),(0.345,4.36),(0.436,3.77),(0.5,3.17)]) #SIM.Fbg_vs_stol = bg #SIM.amorphous_sample_thick_mm = sample_thick_mm #SIM.amorphous_density_gcm3 = 1 #SIM.amorphous_molecular_weight_Da = 18 #SIM.flux=flux_ave #SIM.beamsize_mm=beam_size_mm #SIM.exposure_s=exposure_s #SIM.add_background() #if quick: SIM.to_smv_format(fileout=prefix + "_intimage_002.img") ## rough approximation to air #bg = flex.vec2_double([(0,14.1),(0.045,13.5),(0.174,8.35),(0.35,4.78),(0.5,4.22)]) #SIM.Fbg_vs_stol = bg #SIM.amorphous_sample_thick_mm = air_thick_mm # between beamstop and collimator #SIM.amorphous_density_gcm3 = 1.2e-3 #SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2 #print("amorphous_sample_size_mm=",SIM.amorphous_sample_size_mm) #print("amorphous_sample_thick_mm=",SIM.amorphous_sample_thick_mm) #print("amorphous_density_gcm3=",SIM.amorphous_density_gcm3) #print("amorphous_molecular_weight_Da=",SIM.amorphous_molecular_weight_Da) #SIM.add_background() ##apply beamstop mask here ## set this to 0 or -1 to trigger automatic radius. could be very slow with bright images ## settings for CCD #utils.save_flex(SIM.raw_pixels, "background") SIM.detector_psf_kernel_radius_pixels = 5 #SIM.detector_psf_fwhm_mm=0.08; #SIM.detector_psf_type=shapetype.Fiber # rayonix=Fiber, CSPAD=None (or small Gaussian) SIM.detector_psf_type = shapetype.Unknown # for CSPAD SIM.detector_psf_fwhm_mm = 0 SIM.quantum_gain = 28. #SIM.apply_psf() if verbose: print("One pixel-->", SIM.raw_pixels[500000]) # at this point we scale the raw pixels so that the output array is on an scale from 0 to 50000. # that is the default behavior (intfile_scale<=0), otherwise it applies intfile_scale as a multiplier on an abs scale. if quick: SIM.to_smv_format(fileout=prefix + "_intimage_003.img") if verbose: print("quantum_gain=", SIM.quantum_gain) #defaults to 1. converts photons to ADU print("adc_offset_adu=", SIM.adc_offset_adu) print("detector_calibration_noise_pct=", SIM.detector_calibration_noise_pct) print("flicker_noise_pct=", SIM.flicker_noise_pct) print("readout_noise_adu=", SIM.readout_noise_adu ) # gaussian random number to add to every pixel (0 for PAD) # apply Poissonion correction, then scale to ADU, then adc_offset. # should be 10 for most Rayonix, Pilatus should be 0, CSPAD should be 0. print("detector_psf_type=", SIM.detector_psf_type) print("detector_psf_fwhm_mm=", SIM.detector_psf_fwhm_mm) print("detector_psf_kernel_radius_pixels=", SIM.detector_psf_kernel_radius_pixels) SIM.add_noise() #converts phtons to ADU. if verbose: print("raw_pixels=", SIM.raw_pixels) extra = "PREFIX=%s;\nRANK=%d;\n" % (prefix, rank) out = SIM.raw_pixels.as_numpy_array() if save_smv: SIM.to_smv_format_py(fileout=smv_fileout, intfile_scale=1, rotmat=True, extra=extra, gz=True) elif save_h5: h5_fileout = smv_fileout + ".h5" f = h5py.File(h5_fileout, "w") f.create_dataset("data", data=SIM.raw_pixels.as_numpy_array().astype( np.uint16), compression="lzf") f.close() SIM.free_all()