def exercise_basic(): identity = (1,0,0,0,1,0,0,0,1) I = crystal_orientation(identity,False) #direct space orthorhombic = (1,0,0,0,0.5,0.,0.,0.,0.25) R = crystal_orientation(orthorhombic,True) #reciprocal space assert R.direct_matrix() == (1.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 4.0) assert R.reciprocal_matrix() == orthorhombic inverse = (-1,0,0,0,-1,0,0,0,-1) negative = crystal_orientation(inverse,False) assert I == negative.make_positive() assert R.unit_cell().parameters() == (1.0,2.0,4.0,90.,90.,90.) assert approx_equal( R.unit_cell_inverse().parameters(), (1.0,0.5,0.25,90.,90.,90.) )
def get_frames_from_mysql(self,params): T = Timer("frames") CART = manager(params) db = CART.connection() cursor = db.cursor() cursor.execute("SELECT * FROM %s_frame;"%params.mysql.runtag) ALL = cursor.fetchall() from cctbx.crystal_orientation import crystal_orientation orientations = [crystal_orientation( (a[8],a[9],a[10],a[11],a[12],a[13],a[14],a[15],a[16]),False) for a in ALL] return dict( frame_id = flex.int( [a[0] for a in ALL] ), wavelength = flex.double( [a[1] for a in ALL] ), beam_x = flex.double( [a[2] for a in ALL] ), beam_y = flex.double( [a[3] for a in ALL] ), distance = flex.double( [a[4] for a in ALL] ), orientation = orientations, rotation100_rad = flex.double([a[17] for a in ALL] ), rotation010_rad = flex.double([a[18] for a in ALL] ), rotation001_rad = flex.double([a[19] for a in ALL] ), half_mosaicity_deg = flex.double([a[20] for a in ALL] ), wave_HE_ang = flex.double([a[21] for a in ALL] ), wave_LE_ang = flex.double([a[22] for a in ALL] ), domain_size_ang = flex.double([a[23] for a in ALL] ), unique_file_name = [a[24] for a in ALL], )
def __init__(self,detector=None,camera=None,structure=None,simulation=None): ext.xfel1.__init__(self) self.detector=detector self.camera=camera self.structure=structure self.sim=simulation self.uc = self.structure.p1_cell() self.bmat = matrix.sqr(self.uc.orthogonalization_matrix()).inverse().transpose() self.Ori = crystal_orientation.crystal_orientation( self.bmat, crystal_orientation.basis_type.reciprocal ) energy_in_eV = eV_per_inv_meter / (self.camera.lambda0) # lambda in meters # top hat function for dispersion self.full_pass_eV = energy_in_eV * matrix.col([1.-(self.sim.bandpass/2.), 1.+(self.sim.bandpass/2.)]) self.full_pass_lambda = eV_per_inv_meter * matrix.col((1./self.full_pass_eV[0], 1./self.full_pass_eV[1])) intensities = self.structure.p1_intensities() self.set_indices(intensities.indices()) self.set_intensities(intensities.data())
def read_frames_updated_detail(self): from cctbx.crystal_orientation import crystal_orientation from xfel.cxi.util import is_odd_numbered frames = {'frame_id': flex.int(), 'wavelength': flex.double(), 'cc': flex.double(), 'G': flex.double(), 'BFACTOR': flex.double(), 'RS': flex.double(), 'odd_numbered': flex.bool(), 'thetax': flex.double(), 'thetay': flex.double(), 'orientation': [], 'unit_cell': [], 'unique_file_name': []} stream = open(self.params.output.prefix + '_frame.db', 'r') for row in stream: items = row.split() CO = crystal_orientation([float(t) for t in items[8:17]], True) unique_file_name = eval(items[19]) frames['frame_id'].append(int(items[0])) frames['wavelength'].append(float(items[1])) frames['cc'].append(float(items[20])) frames['G'].append(float(items[5])) frames['BFACTOR'].append(float(items[6])) frames['RS'].append(float(items[7])) frames['thetax'].append(float(items[17])) frames['thetay'].append(float(items[18])) frames['odd_numbered'].append(is_odd_numbered(unique_file_name)) frames['orientation'].append(CO) frames['unit_cell'].append(CO.unit_cell()) frames['unique_file_name'].append(unique_file_name) stream.close() return frames
def calc_partiality_anisotropy_set(self, my_uc, rotx, roty, miller_indices, ry, rz, r0, re, nu, bragg_angle_set, alpha_angle_set, wavelength, crystal_init_orientation, spot_pred_x_mm_set, spot_pred_y_mm_set, detector_distance_mm, partiality_model, flag_beam_divergence): #use III.4 in Winkler et al 1979 (A35; P901) for set of miller indices O = sqr(my_uc.orthogonalization_matrix()).transpose() R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose() CO = crystal_orientation(O*R, basis_type.direct) CO_rotate = CO.rotate_thru((1,0,0), rotx ).rotate_thru((0,1,0), roty) A_star = sqr(CO_rotate.reciprocal_matrix()) S0 = -1*col((0,0,1./wavelength)) #caculate rs rs_set = r0 + (re * flex.tan(bragg_angle_set)) if flag_beam_divergence: rs_set += ((ry * flex.cos(alpha_angle_set))**2 + (rz * flex.sin(alpha_angle_set))**2)**(1/2) #calculate rh x = A_star.elems * miller_indices.as_vec3_double() sd_array = x + S0.elems rh_set = sd_array.norms() - (1/wavelength) #calculate partiality if partiality_model == "Lorentzian": partiality_set = ((rs_set**2)/((2*(rh_set**2))+(rs_set**2))) elif partiality_model == "Voigt": partiality_set = self.voigt(rh_set, rs_set, nu) elif partiality_model == "Lognormal": partiality_set = self.lognpdf(rh_set, rs_set, nu) #calculate delta_xy d_ratio = -detector_distance_mm/sd_array.parts()[2] calc_xy_array = flex.vec3_double(sd_array.parts()[0]*d_ratio, \ sd_array.parts()[1]*d_ratio, flex.double([0]*len(d_ratio))) pred_xy_array = flex.vec3_double(spot_pred_x_mm_set, spot_pred_y_mm_set, flex.double([0]*len(d_ratio))) delta_xy_set = (pred_xy_array - calc_xy_array).norms() return partiality_set, delta_xy_set, rs_set, rh_set
def get_crystal_orientation(ortho_matrix, rot_matrix): #From orthogonalization matrix and rotation matrix, #generate and return crystal orientation O = sqr(ortho_matrix).transpose() R = sqr(rot_matrix).transpose() X = O*R co = crystal_orientation(X, basis_type.direct) return co
def test_reduction(): from libtbx.test_utils import approx_equal uc = unit_cell((10,20,30,90,90,90)) reference = uc.parameters() assert approx_equal (rwgk_niggli(uc).parameters(), reference) CO = crystal_orientation.crystal_orientation(uc.fractionalization_matrix(),True) assert approx_equal ( CO.unit_cell().parameters(), reference) assert approx_equal ( rwgk_niggli(CO).unit_cell().parameters(), reference) return True
def exercise_functions(): orthorhombic = (1,0,0,0,0.5,0.,0.,0.,0.25) R = crystal_orientation(orthorhombic,True) A = R.rotate_thru((1,0,0,),math.pi/2.) assert approx_equal( A.direct_matrix(), (1,0, 0, 0,0,2,0,-4,0) ) B = R.rotate_thru((0,1,0,),math.pi/2.) assert approx_equal( B.direct_matrix(), (0,0,-1, 0,2,0,4, 0,0) ) C = R.rotate_thru((0,0,1,),math.pi/2.) assert approx_equal( C.direct_matrix(), (0,1, 0,-2,0,0,0, 0,4) )
def set_B(self, B): # also set the unit cell co = crystal_orientation(B,True) self._uc = co.unit_cell() self._B = matrix.sqr(self._uc.fractionalization_matrix()).transpose() # reset scan-varying data, if the static B has changed self.reset_scan_points()
def get_labelit_orient(self): UNI_I = matrix.sqr((0,1,0,1,0,0,0,0,-1)).inverse() a, b, c = tuple(self.a_axis), tuple(self.b_axis), tuple(self.c_axis) matXDS = matrix.sqr(a+b+c) print "matXDS=", matXDS[4] matRossmann = (UNI_I * matXDS.transpose()).transpose() orient = crystal_orientation.crystal_orientation(matRossmann, False) # reciprocal flag rotation_ax = self.get_endstation().rot_axi orient = orient.rotate_thru(rotation_ax, -self.starting_angle*math.pi/180.) return orient
def set_orientation(self, orientation, length_unit=1.E-10): #provide orientation as either an A matrix (Rossmann) or B matrix (Busing & Levy) # data type can be either scitbx.matrix.sqr or scitbx::mat3 # in either the reciprocal or direct space setting # or as a cctbx.crystal_orientation.crystal_orientation # if space group is not triclinic the orientation matrix should be close to # symmetrized, but exact symmetrization is done by averaging within the constructor. # length unit defaults to 1.E-10 meters = 1 Angstrom if "direct_matrix" in dir(orientation): self.orientation = orientation # data is already a cctbx orientation else: from cctbx.crystal_orientation import crystal_orientation which_setting = [crystal_orientation(orientation,True), crystal_orientation(orientation,False)] #kludgy test for space setting: unit cell volume is never < 100 Angstroms^3 conversion_to_A3 = (length_unit*length_unit*length_unit)/1.E-30 select = [a.unit_cell().volume()*conversion_to_A3 > 100. for a in which_setting] self.orientation = which_setting[select.index(True)]
def __init__(self,params): import cPickle as pickle from dxtbx.model.beam import beam_factory from dxtbx.model.detector import detector_factory from dxtbx.model.crystal import crystal_model from cctbx.crystal_orientation import crystal_orientation,basis_type from dxtbx.model.experiment.experiment_list import Experiment, ExperimentList from scitbx import matrix self.experiments = ExperimentList() self.unique_file_names = [] self.params = params data = pickle.load(open(self.params.output.prefix+"_frame.pickle","rb")) frames_text = data.split("\n") for item in frames_text: tokens = item.split(' ') wavelength = float(tokens[order_dict["wavelength"]]) beam = beam_factory.simple(wavelength = wavelength) detector = detector_factory.simple( sensor = detector_factory.sensor("PAD"), # XXX shouldn't hard code for XFEL distance = float(tokens[order_dict["distance"]]), beam_centre = [float(tokens[order_dict["beam_x"]]), float(tokens[order_dict["beam_y"]])], fast_direction = "+x", slow_direction = "+y", pixel_size = [self.params.pixel_size,self.params.pixel_size], image_size = [1795,1795], # XXX obviously need to figure this out ) reciprocal_matrix = matrix.sqr([float(tokens[order_dict[k]]) for k in [ 'res_ori_1','res_ori_2','res_ori_3','res_ori_4','res_ori_5','res_ori_6','res_ori_7','res_ori_8','res_ori_9']]) ORI = crystal_orientation(reciprocal_matrix, basis_type.reciprocal) direct = matrix.sqr(ORI.direct_matrix()) crystal = crystal_model( real_space_a = matrix.row(direct[0:3]), real_space_b = matrix.row(direct[3:6]), real_space_c = matrix.row(direct[6:9]), space_group_symbol = "P63", # XXX obviously another gap in the database paradigm mosaicity = float(tokens[order_dict["half_mosaicity_deg"]]), ) crystal.domain_size = float(tokens[order_dict["domain_size_ang"]]) #if isoform is not None: # newB = matrix.sqr(isoform.fractionalization_matrix()).transpose() # crystal.set_B(newB) self.experiments.append(Experiment(beam=beam, detector=None, #dummy for now crystal=crystal)) self.unique_file_names.append(tokens[order_dict["unique_file_name"]]) self.show_summary()
def crystfel_to_cctbx_coord_system(abasis, bbasis, cbasis): """CrystFEL is RHS with z down the beam, and y to the ceiling. CCTBX.Postrefine is RHS with z to the source, and y to the ceiling. """ from scitbx.matrix import sqr from cctbx import crystal_orientation a_mat = sqr((abasis[0], bbasis[0], cbasis[0], abasis[1], bbasis[1], cbasis[1], abasis[2], bbasis[2], cbasis[2])) coord_transformation = sqr(( -1, 0, 0, 0, 1, 0, 0, 0, -1)) new_coords = a_mat.__mul__(coord_transformation) ori = crystal_orientation.crystal_orientation(new_coords, crystal_orientation.basis_type.reciprocal) logging.debug("\naStar: {}\nbStar: {}\ncStar: {}".format(abasis, bbasis, cbasis)) logging.debug(str(ori)) return ori
def read_frames(self): from xfel.cxi.util import is_odd_numbered db = self.connection() cursor = db.cursor() cursor.execute("""SELECT frame_id_1_base,wavelength,c_c,slope,offset,res_ori_1,res_ori_2,res_ori_3, res_ori_4,res_ori_5,res_ori_6,res_ori_7,res_ori_8,res_ori_9, unique_file_name FROM `%s_frame`"""%self.params.mysql.runtag) ALL = cursor.fetchall() from cctbx.crystal_orientation import crystal_orientation orientations = [crystal_orientation( (a[5],a[6],a[7],a[8],a[9],a[10],a[11],a[12],a[13]),False) for a in ALL] return dict( frame_id = flex.int( [a[0]-1 for a in ALL] ), wavelength = flex.double( [a[1] for a in ALL] ), cc = flex.double( [a[2] for a in ALL] ), slope = flex.double( [a[3] for a in ALL] ), offset = flex.double( [a[4] for a in ALL] ), odd_numbered = flex.bool( [is_odd_numbered(a[14]) for a in ALL] ), orientation = orientations, unit_cell = [CO.unit_cell() for CO in orientations], unique_file_name = [a[14] for a in ALL] )
def read_frames_legacy_detail(self): from cctbx.crystal_orientation import crystal_orientation from xfel.cxi.util import is_odd_numbered # XXX issues with spaces in the file name, and leading and # trailing single quotes (stripped below). frames = {'frame_id': flex.int(), 'wavelength': flex.double(), 'cc': flex.double(), 'slope': flex.double(), 'offset': flex.double(), 'odd_numbered': flex.bool(), 'domain_size_ang': flex.double(), 'half_mosaicity_deg': flex.double(), 'orientation': [], 'unit_cell': [], 'unique_file_name': []} stream = open(self.params.output.prefix + '_frame.db', 'r') for row in stream: items = row.split() CO = crystal_orientation([float(t) for t in items[8:17]], False) unique_file_name = eval(items[24]) frames['frame_id'].append(int(items[0])) frames['wavelength'].append(float(items[1])) frames['cc'].append(float(items[5])) frames['slope'].append(float(items[6])) frames['offset'].append(float(items[7])) frames['domain_size_ang'].append(float(items[23])) frames['half_mosaicity_deg'].append(float(items[20])) frames['odd_numbered'].append(is_odd_numbered(unique_file_name)) frames['orientation'].append(CO) frames['unit_cell'].append(CO.unit_cell()) frames['unique_file_name'].append(unique_file_name) stream.close() return frames
def populate_orientation(self): assert self.xtal.get_A() is not None, "no crystal orientation matrix" self.frame['current_orientation'] = [ crystal_orientation(self.xtal.get_A(), True) ]
if key == "correction_vectors": # drop correction_vectors as they aren't as easy to split up continue elif key in [ "current_cb_op_to_primitive", "effective_tiling", "pointgroup", "identified_isoform" ]: dl[key] = data[key] dr[key] = data[key] dm[key] = data[key] dn[key] = data[key] continue elif key == "current_orientation": from cctbx import crystal_orientation dl[key] = [ crystal_orientation.crystal_orientation(c) for c in data[key] ] dr[key] = [ crystal_orientation.crystal_orientation(c) for c in data[key] ] dm[key] = [ crystal_orientation.crystal_orientation(c) for c in data[key] ] dn[key] = [ crystal_orientation.crystal_orientation(c) for c in data[key] ] continue
def refined_settings_from_refined_triclinic(experiments, reflections, params): """Generate a RefinedSettingsList from a triclinic model. Args: experiments: The experiments refined with a triclinic model reflections: A reflection table containing observed centroids params: The working PHIL parameters. Returns: RefinedSettingsList: A list of the refined settings. The highest symmetry setting will be first item in the list, and the triclinic setting will be last. """ if params.nproc is libtbx.Auto: params.nproc = number_of_processors() if params.refinement.reflections.outlier.algorithm in ("auto", libtbx.Auto): if experiments[0].goniometer is None: params.refinement.reflections.outlier.algorithm = "sauter_poon" else: # different default to dials.refine # tukey is faster and more appropriate at the indexing step params.refinement.reflections.outlier.algorithm = "tukey" assert len(experiments.crystals()) == 1 crystal = experiments.crystals()[0] used_reflections = copy.deepcopy(reflections) UC = crystal.get_unit_cell() refined_settings = RefinedSettingsList() for item in iotbx_converter( UC, params.lepage_max_delta, best_monoclinic_beta=params.best_monoclinic_beta): refined_settings.append(BravaisSetting(item)) triclinic = refined_settings.triclinic() # assert no transformation between indexing and bravais list assert str(triclinic["cb_op_inp_best"]) == "a,b,c" Nset = len(refined_settings) for j in range(Nset): refined_settings[j].setting_number = Nset - j for subgroup in refined_settings: bravais_lattice = str( bravais_types.bravais_lattice( group=subgroup["best_subsym"].space_group())) space_group = lowest_symmetry_space_group_for_bravais_lattice( bravais_lattice) orient = crystal_orientation(crystal.get_A(), True).change_basis( scitbx.matrix.sqr(subgroup["cb_op_inp_best"].c().as_double_array() [0:9]).transpose()) constrain_orient = orient.constrain(subgroup["system"]) subgroup["bravais"] = bravais_lattice subgroup.unrefined_crystal = dxtbx_crystal_from_orientation( constrain_orient, space_group) with concurrent.futures.ProcessPoolExecutor( max_workers=params.nproc) as pool: for i, result in enumerate( pool.map( refine_subgroup, ((params, subgroup, used_reflections, experiments) for subgroup in refined_settings), )): refined_settings[i] = result identify_likely_solutions(refined_settings) return refined_settings
def refined_settings_factory_from_refined_triclinic(params, experiments, reflections, i_setting=None, lepage_max_delta=5.0, nproc=1): assert len(experiments.crystals()) == 1 crystal = experiments.crystals()[0] used_reflections = copy.deepcopy(reflections) UC = crystal.get_unit_cell() Lfat = RefinedSettingsList() for item in iotbx_converter(UC, lepage_max_delta): Lfat.append(BravaisSetting(item)) triclinic = Lfat.triclinic() # assert no transformation between indexing and bravais list assert str(triclinic["cb_op_inp_best"]) == "a,b,c" Nset = len(Lfat) for j in range(Nset): Lfat[j].setting_number = Nset - j for j in range(Nset): cb_op = Lfat[j]["cb_op_inp_best"].c().as_double_array()[0:9] orient = crystal_orientation(crystal.get_A(), True) orient_best = orient.change_basis(scitbx.matrix.sqr(cb_op).transpose()) constrain_orient = orient_best.constrain(Lfat[j]["system"]) bravais = Lfat[j]["bravais"] cb_op_best_ref = Lfat[j][ "best_subsym"].change_of_basis_op_to_reference_setting() space_group = sgtbx.space_group_info( number=bravais_lattice_to_lowest_symmetry_spacegroup_number[ bravais]).group() space_group = space_group.change_basis(cb_op_best_ref.inverse()) bravais = str(bravais_types.bravais_lattice(group=space_group)) Lfat[j]["bravais"] = bravais Lfat[j].unrefined_crystal = dials_crystal_from_orientation( constrain_orient, space_group) args = [] for subgroup in Lfat: args.append((params, subgroup, used_reflections, experiments)) results = easy_mp.parallel_map( func=refine_subgroup, iterable=args, processes=nproc, method="multiprocessing", preserve_order=True, asynchronous=True, preserve_exception_message=True, ) for i, result in enumerate(results): Lfat[i] = result identify_likely_solutions(Lfat) return Lfat
def set_B(self, B): # also set the unit cell co = crystal_orientation(B, True) self._uc = co.unit_cell() self._B = matrix.sqr(self._uc.fractionalization_matrix()).transpose()
cell = results['cell'] axis = results['axis'] wavelength = results['wavelength'] start = results['starting_frame'] - 1 phi_start = results['phi_start'] phi_width = results['phi_width'] # FIXME really need to rotate the reference frame to put the # direct beam vector along (0,0,1) resolution = 1.8 orientation = A + B + C co = crystal_orientation(orientation, basis_type.direct) mm = co.unit_cell().max_miller_indices(resolution) ra = rotation_angles(resolution, co.reciprocal_matrix(), wavelength, axis) for record in open('SPOT.XDS', 'r').readlines(): lst = record.split() hkl = tuple(map(int, lst[-3:])) if hkl == (0, 0, 0): continue image = nint(float(lst[2])) if ra(hkl): phi1, phi2 = ra.get_intersection_angles() i = nint(start + (rtod * phi1 - phi_start) / phi_width) j = nint(start + (rtod * phi2 - phi_start) / phi_width) if abs(i - image) <= abs(j - image):
def set_B(self, B): # also set the unit cell co = crystal_orientation(B,True) self._uc = co.unit_cell() self._B = matrix.sqr(self._uc.fractionalization_matrix()).transpose()
def get_predictions_accounting_for_centering(self,experiments,reflections,cb_op_to_primitive,**kwargs): # interface requires this function to set current_orientation # in the actual setting used for Miller index calculation detector = experiments[0].detector crystal = experiments[0].crystal if self.horizons_phil.integration.model == "user_supplied": lower_limit_domain_size = math.pow( crystal.get_unit_cell().volume(), 1./3.)*self.horizons_phil.integration.mosaic.domain_size_lower_limit # default 10-unit cell block size minimum reasonable domain actual_used_domain_size = kwargs.get("domain_size_ang",lower_limit_domain_size) self.block_counter+=1 rot_mat = matrix.sqr(cb_op_to_primitive.c().r().as_double()).transpose() from cctbx.crystal_orientation import crystal_orientation, basis_type centered_orientation = crystal_orientation(crystal.get_A(),basis_type.reciprocal) self.current_orientation = centered_orientation self.current_cb_op_to_primitive = cb_op_to_primitive primitive_orientation = centered_orientation.change_basis(rot_mat) self.inputai.setOrientation(primitive_orientation) from cxi_user import pre_get_predictions if self.block_counter < 2: KLUDGE = self.horizons_phil.integration.mosaic.kludge1 # bugfix 1 of 2 for protocol 6, equation 2 self.inputai.setMosaicity(KLUDGE*self.inputai.getMosaicity()) oldbase = self.inputai.getBase() #print oldbase.xbeam, oldbase.ybeam newbeam = detector[0].get_beam_centre(experiments[0].beam.get_s0()) newdistance = -detector[0].get_beam_centre_lab(experiments[0].beam.get_s0())[2] from labelit.dptbx import Parameters base = Parameters(xbeam = newbeam[0], ybeam = newbeam[1], #oldbase.xbeam, ybeam = oldbase.ybeam, distance = newdistance, twotheta = 0.0) self.inputai.setBase(base) self.inputai.setWavelength(experiments[0].beam.get_wavelength()) self.bp3_wrapper = pre_get_predictions(self.inputai, self.horizons_phil, raw_image = self.imagefiles.images[self.image_number], imageindex = self.frame_numbers[self.image_number], spotfinder = self.spotfinder, limiting_resolution = self.limiting_resolution, domain_size_ang = actual_used_domain_size, ) BPpredicted = self.bp3_wrapper.ucbp3.selected_predictions_labelit_format() BPhkllist = self.bp3_wrapper.ucbp3.selected_hkls() self.actual = actual_used_domain_size primitive_hkllist = BPhkllist #not sure if matrix needs to be transposed first for outputting HKL's???: self.hkllist = cb_op_to_primitive.inverse().apply(primitive_hkllist) if self.horizons_phil.integration.spot_prediction == "dials": from dials.algorithms.spot_prediction import StillsReflectionPredictor predictor = StillsReflectionPredictor(experiments[0]) Rcalc = flex.reflection_table.empty_standard(len(self.hkllist)) Rcalc['miller_index'] = self.hkllist predictor.for_reflection_table(Rcalc, crystal.get_A()) self.predicted = Rcalc['xyzcal.mm'] self.dials_spot_prediction = Rcalc self.dials_model = experiments elif self.horizons_phil.integration.spot_prediction == "ucbp3": self.predicted = BPpredicted self.inputai.setOrientation(centered_orientation) if self.inputai.active_areas != None: self.predicted,self.hkllist = self.inputai.active_areas( self.predicted,self.hkllist,self.pixel_size) if self.block_counter < 2: down = self.inputai.getMosaicity()/KLUDGE print "Readjusting mosaicity back down to ",down self.inputai.setMosaicity(down) return
def organize_input(self, observations_pickle, iparams, avg_mode, pickle_filename=None): """Given the pickle file, extract and prepare observations object and the alpha angle (meridional to equatorial). """ #get general parameters if iparams.isoform_name is not None: if "identified_isoform" not in observations_pickle: return None, "No identified isoform" if observations_pickle[ "identified_isoform"] != iparams.isoform_name: return None, "Identified isoform(%s) is not the requested isoform (%s)" % ( observations_pickle["identified_isoform"], iparams.isoform_name) if iparams.flag_weak_anomalous: if avg_mode == 'final': target_anomalous_flag = iparams.target_anomalous_flag else: target_anomalous_flag = False else: target_anomalous_flag = iparams.target_anomalous_flag img_filename_only = '' if pickle_filename: img_filename_only = os.path.basename(pickle_filename) txt_exception = ' {0:40} ==> '.format(img_filename_only) #for dials integration pickles - also look for experimentxxx.json if "miller_index" in observations_pickle: from dxtbx.model.experiment_list import ExperimentListFactory exp_json_file = os.path.join( os.path.dirname(pickle_filename), img_filename_only.split('_')[0] + '_refined_experiments.json') if os.path.isfile(exp_json_file): experiments = ExperimentListFactory.from_json_file( exp_json_file) dials_crystal = experiments[0].crystal detector = experiments[0].detector beam = experiments[0].beam crystal_symmetry = crystal.symmetry( unit_cell=dials_crystal.get_unit_cell().parameters(), space_group_symbol=iparams.target_space_group) miller_set_all = miller.set( crystal_symmetry=crystal_symmetry, indices=observations_pickle['miller_index'], anomalous_flag=target_anomalous_flag) observations = miller_set_all.array( data=observations_pickle['intensity.sum.value'], sigmas=flex.sqrt( observations_pickle['intensity.sum.variance']) ).set_observation_type_xray_intensity() detector_distance_mm = detector[0].get_distance() alpha_angle_obs = flex.double([0] * len(observations.data())) wavelength = beam.get_wavelength() spot_pred_x_mm = observations_pickle['s1'] #a disguise of s1 spot_pred_y_mm = flex.double([0] * len(observations.data())) #calculate the crystal orientation O = sqr(dials_crystal.get_unit_cell().orthogonalization_matrix( )).transpose() R = sqr(dials_crystal.get_U()).transpose() from cctbx.crystal_orientation import crystal_orientation, basis_type crystal_init_orientation = crystal_orientation( O * R, basis_type.direct) else: txt_exception += exp_json_file + ' not found' print txt_exception return None, txt_exception else: #for cctbx.xfel proceed as usual observations = observations_pickle["observations"][0] detector_distance_mm = observations_pickle['distance'] mm_predictions = iparams.pixel_size_mm * ( observations_pickle['mapped_predictions'][0]) xbeam = observations_pickle["xbeam"] ybeam = observations_pickle["ybeam"] alpha_angle_obs = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \ for pred in mm_predictions]) spot_pred_x_mm = flex.double( [pred[0] - xbeam for pred in mm_predictions]) spot_pred_y_mm = flex.double( [pred[1] - ybeam for pred in mm_predictions]) #Polarization correction wavelength = observations_pickle["wavelength"] crystal_init_orientation = observations_pickle[ "current_orientation"][0] #continue reading... if iparams.flag_LP_correction and "observations" in observations_pickle: fx = 1 - iparams.polarization_horizontal_fraction fy = 1 - fx if fx > 1.0 or fx < 0: print 'Horizontal polarization fraction is not correct. The value must be >= 0 and <= 1' print 'No polarization correction. Continue with post-refinement' else: phi_angle_obs = flex.double([math.atan2(pred[1]-ybeam, pred[0]-xbeam) \ for pred in mm_predictions]) bragg_angle_obs = observations.two_theta(wavelength).data() P = ((fx*((flex.sin(phi_angle_obs)**2)+((flex.cos(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2)))+\ (fy*((flex.cos(phi_angle_obs)**2)+((flex.sin(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2)))) I_prime = observations.data() / P sigI_prime = observations.sigmas() / P observations = observations.customized_copy( data=flex.double(I_prime), sigmas=flex.double(sigI_prime)) #set observations with target space group - !!! required for correct #merging due to map_to_asu command. if iparams.target_crystal_system is not None: target_crystal_system = iparams.target_crystal_system else: target_crystal_system = observations.crystal_symmetry( ).space_group().crystal_system() lph = lbfgs_partiality_handler() if iparams.flag_override_unit_cell: uc_constrained_inp = lph.prep_input( iparams.target_unit_cell.parameters(), target_crystal_system) else: uc_constrained_inp = lph.prep_input( observations.unit_cell().parameters(), target_crystal_system) uc_constrained = list( lph.prep_output(uc_constrained_inp, target_crystal_system)) try: #apply constrain using the crystal system miller_set = symmetry(unit_cell=uc_constrained, space_group_symbol=iparams.target_space_group ).build_miller_set( anomalous_flag=target_anomalous_flag, d_min=iparams.merge.d_min) observations = observations.customized_copy( anomalous_flag=target_anomalous_flag, crystal_symmetry=miller_set.crystal_symmetry()) except Exception: a, b, c, alpha, beta, gamma = uc_constrained txt_exception += 'Mismatch spacegroup (%6.2f,%6.2f,%6.2f,%6.2f,%6.2f,%6.2f)' % ( a, b, c, alpha, beta, gamma) print txt_exception return None, txt_exception #reset systematic absence sys_absent_negate_flags = flex.bool([ sys_absent_flag[1] == False for sys_absent_flag in observations.sys_absent_flags() ]) observations = observations.select(sys_absent_negate_flags) alpha_angle_obs = alpha_angle_obs.select(sys_absent_negate_flags) spot_pred_x_mm = spot_pred_x_mm.select(sys_absent_negate_flags) spot_pred_y_mm = spot_pred_y_mm.select(sys_absent_negate_flags) #remove observations from rejection list if iparams.rejections: if pickle_filename in iparams.rejections: miller_indices_ori_rejected = iparams.rejections[ pickle_filename] i_sel_flag = flex.bool([True] * len(observations.data())) cnrej = 0 for miller_index_ori_rejected in miller_indices_ori_rejected: for i_index_ori, miller_index_ori in enumerate( observations.indices()): if miller_index_ori_rejected == miller_index_ori: i_sel_flag[i_index_ori] = False cnrej += 1 observations = observations.customized_copy( indices=observations.indices().select(i_sel_flag), data=observations.data().select(i_sel_flag), sigmas=observations.sigmas().select(i_sel_flag)) alpha_angle_obs = alpha_angle_obs.select(i_sel_flag) spot_pred_x_mm = spot_pred_x_mm.select(i_sel_flag) spot_pred_y_mm = spot_pred_y_mm.select(i_sel_flag) #filter resolution i_sel_res = observations.resolution_filter_selection( d_max=iparams.merge.d_max, d_min=iparams.merge.d_min) observations = observations.select(i_sel_res) alpha_angle_obs = alpha_angle_obs.select(i_sel_res) spot_pred_x_mm = spot_pred_x_mm.select(i_sel_res) spot_pred_y_mm = spot_pred_y_mm.select(i_sel_res) #Filter weak i_sel = (observations.data() / observations.sigmas()) > iparams.merge.sigma_min observations = observations.select(i_sel) alpha_angle_obs = alpha_angle_obs.select(i_sel) spot_pred_x_mm = spot_pred_x_mm.select(i_sel) spot_pred_y_mm = spot_pred_y_mm.select(i_sel) #filter icering (if on) if iparams.icering.flag_on: miller_indices = flex.miller_index() I_set = flex.double() sigI_set = flex.double() alpha_angle_obs_set = flex.double() spot_pred_x_mm_set = flex.double() spot_pred_y_mm_set = flex.double() for miller_index, d, I, sigI, alpha, spot_x, spot_y in zip( observations.indices(), observations.d_spacings().data(), observations.data(), observations.sigmas(), alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm): if d > iparams.icering.d_upper or d < iparams.icering.d_lower: miller_indices.append(miller_index) I_set.append(I) sigI_set.append(sigI) alpha_angle_obs_set.append(alpha) spot_pred_x_mm_set.append(spot_x) spot_pred_y_mm_set.append(spot_y) observations = observations.customized_copy(indices=miller_indices, data=I_set, sigmas=sigI_set) alpha_angle_obs = alpha_angle_obs_set[:] spot_pred_x_mm = spot_pred_x_mm_set[:] spot_pred_y_mm = spot_pred_y_mm_set[:] #replacing sigI (if set) if iparams.flag_replace_sigI: observations = observations.customized_copy( sigmas=flex.sqrt(observations.data())) inputs = observations, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation return inputs, 'OK'
def get_predictions_accounting_for_centering(self, experiments, reflections, cb_op_to_primitive, **kwargs): # interface requires this function to set current_orientation # in the actual setting used for Miller index calculation detector = experiments[0].detector crystal = experiments[0].crystal if self.horizons_phil.integration.model == "user_supplied": lower_limit_domain_size = math.pow( crystal.get_unit_cell().volume(), 1. / 3. ) * self.horizons_phil.integration.mosaic.domain_size_lower_limit # default 10-unit cell block size minimum reasonable domain actual_used_domain_size = kwargs.get("domain_size_ang", lower_limit_domain_size) self.block_counter += 1 rot_mat = matrix.sqr( cb_op_to_primitive.c().r().as_double()).transpose() from cctbx.crystal_orientation import crystal_orientation, basis_type centered_orientation = crystal_orientation(crystal.get_A(), basis_type.reciprocal) self.current_orientation = centered_orientation self.current_cb_op_to_primitive = cb_op_to_primitive primitive_orientation = centered_orientation.change_basis(rot_mat) self.inputai.setOrientation(primitive_orientation) from cxi_user import pre_get_predictions if self.block_counter < 2: KLUDGE = self.horizons_phil.integration.mosaic.kludge1 # bugfix 1 of 2 for protocol 6, equation 2 self.inputai.setMosaicity(KLUDGE * self.inputai.getMosaicity()) oldbase = self.inputai.getBase() #print oldbase.xbeam, oldbase.ybeam newbeam = detector[0].get_beam_centre(experiments[0].beam.get_s0()) newdistance = -detector[0].get_beam_centre_lab( experiments[0].beam.get_s0())[2] from labelit.dptbx import Parameters base = Parameters( xbeam=newbeam[0], ybeam=newbeam[1], #oldbase.xbeam, ybeam = oldbase.ybeam, distance=newdistance, twotheta=0.0) self.inputai.setBase(base) self.inputai.setWavelength(experiments[0].beam.get_wavelength()) self.bp3_wrapper = pre_get_predictions( self.inputai, self.horizons_phil, raw_image=self.imagefiles.images[self.image_number], imageindex=self.frame_numbers[self.image_number], spotfinder=self.spotfinder, limiting_resolution=self.limiting_resolution, domain_size_ang=actual_used_domain_size, ) BPpredicted = self.bp3_wrapper.ucbp3.selected_predictions_labelit_format( ) BPhkllist = self.bp3_wrapper.ucbp3.selected_hkls() self.actual = actual_used_domain_size primitive_hkllist = BPhkllist #not sure if matrix needs to be transposed first for outputting HKL's???: self.hkllist = cb_op_to_primitive.inverse().apply( primitive_hkllist) if self.horizons_phil.integration.spot_prediction == "dials": from dials.algorithms.spot_prediction import StillsReflectionPredictor predictor = StillsReflectionPredictor(experiments[0]) Rcalc = flex.reflection_table.empty_standard(len(self.hkllist)) Rcalc['miller_index'] = self.hkllist predictor.for_reflection_table(Rcalc, crystal.get_A()) self.predicted = Rcalc['xyzcal.mm'] self.dials_spot_prediction = Rcalc self.dials_model = experiments elif self.horizons_phil.integration.spot_prediction == "ucbp3": self.predicted = BPpredicted self.inputai.setOrientation(centered_orientation) if self.inputai.active_areas != None: self.predicted, self.hkllist = self.inputai.active_areas( self.predicted, self.hkllist, self.pixel_size) if self.block_counter < 2: down = self.inputai.getMosaicity() / KLUDGE print "Readjusting mosaicity back down to ", down self.inputai.setMosaicity(down) return
def postrefine_by_frame(self, frame_no, pickle_filename, iparams, miller_array_ref, pres_in, avg_mode): #1. Prepare data observations_pickle = read_frame(pickle_filename) pickle_filepaths = pickle_filename.split('/') img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1] txt_exception = ' {0:40} ==> '.format(img_filename_only) if observations_pickle is None: txt_exception += 'empty or bad input file\n' return None, txt_exception inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) if inputs is not None: observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs else: txt_exception += txt_organize_input + '\n' return None, txt_exception #2. Select data for post-refinement (only select indices that are common with the reference set observations_non_polar, index_basis_name = self.get_observations_non_polar( observations_original, pickle_filename, iparams) matches = miller.match_multi_indices( miller_indices_unique=miller_array_ref.indices(), miller_indices=observations_non_polar.indices()) pair_0 = flex.size_t([pair[0] for pair in matches.pairs()]) pair_1 = flex.size_t([pair[1] for pair in matches.pairs()]) references_sel = miller_array_ref.select(pair_0) observations_original_sel = observations_original.select(pair_1) observations_non_polar_sel = observations_non_polar.select(pair_1) alpha_angle_set = alpha_angle.select(pair_1) spot_pred_x_mm_set = spot_pred_x_mm.select(pair_1) spot_pred_y_mm_set = spot_pred_y_mm.select(pair_1) #4. Do least-squares refinement lsqrh = leastsqr_handler() try: refined_params, stats, n_refl_postrefined = lsqrh.optimize( references_sel.data(), observations_original_sel, wavelength, crystal_init_orientation, alpha_angle_set, spot_pred_x_mm_set, spot_pred_y_mm_set, iparams, pres_in, observations_non_polar_sel, detector_distance_mm) except Exception: txt_exception += 'optimization failed.\n' return None, txt_exception #caculate partiality for output (with target_anomalous check) G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \ a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation = inputs observations_non_polar, index_basis_name = self.get_observations_non_polar( observations_original, pickle_filename, iparams) from cctbx.uctbx import unit_cell uc_fin = unit_cell( (a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin)) if pres_in is not None: crystal_init_orientation = pres_in.crystal_orientation two_theta = observations_original.two_theta( wavelength=wavelength).data() ph = partiality_handler() partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set( uc_fin, rotx_fin, roty_fin, observations_original.indices(), ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, two_theta, alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) #calculate the new crystal orientation O = sqr(uc_fin.orthogonalization_matrix()).transpose() R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose() from cctbx.crystal_orientation import crystal_orientation, basis_type CO = crystal_orientation(O * R, basis_type.direct) crystal_fin_orientation = CO.rotate_thru( (1, 0, 0), rotx_fin).rotate_thru((0, 1, 0), roty_fin) #remove reflections with partiality below threshold i_sel = partiality_fin > iparams.merge.partiality_min partiality_fin_sel = partiality_fin.select(i_sel) rs_fin_sel = rs_fin.select(i_sel) rh_fin_sel = rh_fin.select(i_sel) observations_non_polar_sel = observations_non_polar.customized_copy(\ indices=observations_non_polar.indices().select(i_sel), data=observations_non_polar.data().select(i_sel), sigmas=observations_non_polar.sigmas().select(i_sel)) observations_original_sel = observations_original.customized_copy(\ indices=observations_original.indices().select(i_sel), data=observations_original.data().select(i_sel), sigmas=observations_original.sigmas().select(i_sel)) pres = postref_results() pres.set_params(observations=observations_non_polar_sel, observations_original=observations_original_sel, refined_params=refined_params, stats=stats, partiality=partiality_fin_sel, rs_set=rs_fin_sel, rh_set=rh_fin_sel, frame_no=frame_no, pickle_filename=pickle_filename, wavelength=wavelength, crystal_orientation=crystal_fin_orientation, detector_distance_mm=detector_distance_mm) r_change = ((pres.R_final - pres.R_init) / pres.R_init) * 100 r_xy_change = ( (pres.R_xy_final - pres.R_xy_init) / pres.R_xy_init) * 100 cc_change = ((pres.CC_final - pres.CC_init) / pres.CC_init) * 100 txt_postref = '{0:40} => RES:{1:5.2f} NREFL:{2:5d} R:{3:6.1f}% RXY:{4:5.1f}% CC:{5:5.1f}% G:{6:6.4f} B:{7:5.1f} CELL:{8:6.1f}{9:6.1f} {10:6.1f} {11:5.1f} {12:5.1f} {13:5.1f}'.format( img_filename_only + ' (' + index_basis_name + ')', observations_original_sel.d_min(), len(observations_original_sel.data()), r_change, r_xy_change, cc_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin) print txt_postref txt_postref += '\n' return pres, txt_postref
def exercise_change_basis(): assert approx_equal(O1.unit_cell().parameters(), (47.659, 47.6861, 49.6444, 62.9615, 73.8222, 73.5269), 1E-3) reindex = (0.0, -1.0, 0.0, -1.0, 0.0, 0.0, 0.0, 0.0, -1.0 ) # swap a & b and take inverse O2 = O1.change_basis(reindex) assert approx_equal(O2.unit_cell().parameters(), (47.6861, 47.659, 49.6444, 73.8222, 62.9615, 73.5269), 1E-3) rhombohedral_test = crystal_orientation( (0.002737747939994224, -0.0049133768326561278, 0.0023634556852316566, 0.0062204242383498082, 0.006107332242442573, 0.0047036576949967112, -0.0057640198753891566, -0.0025891042237382953, 0.0023674924674260264), basis_type.reciprocal) rhombohedral_reference = crystal_orientation( (-0.0076361080646872997, 0.0049061665572297979, 0.0023688116121433865, -0.00011109895272056645, -0.0061110173438898583, 0.0047062769738302939, 0.0031790372319626674, 0.0025876279220667518, 0.0023669727051432361), basis_type.reciprocal) # Find a similarity transform that maps the two cells onto each other c_inv_r_best = rhombohedral_test.best_similarity_transformation( other=rhombohedral_reference, fractional_length_tolerance=1.00, unimodular_generator_range=1) c_inv_r_int = tuple([int(round(ij, 0)) for ij in c_inv_r_best]) assert c_inv_r_int == (-1, 0, 0, 1, -1, 0, 0, 0, 1) c_inv = sgtbx.rt_mx(sgtbx.rot_mx(c_inv_r_int)) cb_op = sgtbx.change_of_basis_op(c_inv) rhombohedral_reindex = rhombohedral_test.change_basis(cb_op) assert rhombohedral_reindex.difference_Z_score( rhombohedral_reference) < 0.40 assert rhombohedral_reindex.direct_mean_square_difference( rhombohedral_reference) < 0.1 #an alternative test from ana that should fail (gives high msd~0.22; cell axes don't match): ana_reference = crystal_orientation( (0.0023650364919947241, 0.012819317075171401, 0.003042762222847376, 0.0081242553464681254, 0.0050052660206998077, -0.01472465697193685, -0.01373896574061278, 0.0083781530252581681, -0.0035301340829149005), basis_type.reciprocal) ana_current = crystal_orientation( (-0.014470153848927263, 0.0095185368147633793, 0.00087746490483763798, -0.0049989006573928716, -0.0079714727432991222, 0.014778692772530192, 0.0051268914129933571, 0.010264066188909109, 0.0044244589492769002), basis_type.reciprocal) c_inv_r_best = ana_current.best_similarity_transformation( other=ana_reference, fractional_length_tolerance=200.0, unimodular_generator_range=1) c_inv_r_int = tuple([int(round(ij, 0)) for ij in c_inv_r_best]) c_inv = sgtbx.rt_mx(sgtbx.rot_mx(c_inv_r_int)) cb_op = sgtbx.change_of_basis_op(c_inv) ana_reindex = ana_reference.change_basis(cb_op.inverse()) assert 200.0 > ana_reindex.difference_Z_score(ana_current) > 20. u = uctbx.unit_cell((10., 10., 10., 90., 90., 90.)) CO = crystal_orientation(u.fractionalization_matrix(), True) assert approx_equal( CO.unit_cell().parameters(), CO.change_basis((1, 0, 0, 0, 1, 0, 0, 0, 1)).unit_cell().parameters()) u = uctbx.unit_cell((2, 3, 5)) CO = crystal_orientation(u.fractionalization_matrix(), True) assert approx_equal( CO.change_basis((0, 1, 0, 0, 0, 1, 1, 0, 0)).unit_cell().parameters(), (5, 2, 3, 90, 90, 90)) cb_op = sgtbx.change_of_basis_op("y,z,x") assert approx_equal( CO.change_basis(cb_op).unit_cell().parameters(), (5, 2, 3, 90, 90, 90)) import scitbx.math from scitbx import matrix fmx = matrix.sqr( uctbx.unit_cell((10, 13, 17, 85, 95, 105)).fractionalization_matrix()) crm = matrix.sqr( scitbx.math.r3_rotation_axis_and_angle_as_matrix(axis=(-3, 5, -7), angle=37, deg=True)) co = crystal_orientation(crm * fmx.transpose(), True) assert approx_equal(co.crystal_rotation_matrix(), crm)
identity = (1, 0, 0, 0, 1, 0, 0, 0, 1) I = crystal_orientation(identity, False) #direct space orthorhombic = (1, 0, 0, 0, 0.5, 0., 0., 0., 0.25) R = crystal_orientation(orthorhombic, True) #reciprocal space assert R.direct_matrix() == (1.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 4.0) assert R.reciprocal_matrix() == orthorhombic inverse = (-1, 0, 0, 0, -1, 0, 0, 0, -1) negative = crystal_orientation(inverse, False) assert I == negative.make_positive() assert R.unit_cell().parameters() == (1.0, 2.0, 4.0, 90., 90., 90.) assert approx_equal(R.unit_cell_inverse().parameters(), (1.0, 0.5, 0.25, 90., 90., 90.)) O1 = crystal_orientation( (-0.015553395334476732, -0.0028287158782335244, 0.018868416534039902, -0.0016512962184570643, -0.020998220575299865, 0.0012056332661160732, 0.015789188025134133, -0.011166135863736777, 0.013045365404272641), True) def exercise_change_basis(): assert approx_equal(O1.unit_cell().parameters(), (47.659, 47.6861, 49.6444, 62.9615, 73.8222, 73.5269), 1E-3) reindex = (0.0, -1.0, 0.0, -1.0, 0.0, 0.0, 0.0, 0.0, -1.0 ) # swap a & b and take inverse O2 = O1.change_basis(reindex) assert approx_equal(O2.unit_cell().parameters(), (47.6861, 47.659, 49.6444, 73.8222, 62.9615, 73.5269), 1E-3) rhombohedral_test = crystal_orientation(
def is_similar_to(self, other, angle_tolerance=0.01, mosaicity_tolerance=0.8, uc_rel_length_tolerance=0.01, uc_abs_angle_tolerance=1.): ''' Test similarity of this to another crystal model :param other: the crystal model to test against :type other: crystal_model :param angle_tolerance: maximum tolerated orientation difference in degrees :type angle_tolerance: float :param mosaicity_tolerance: minimum tolerated fraction of the larger mosaicity for the smaller mosaicity to retain similarity :type mosaicity_tolerance:float :param uc_rel_length_tolerance: relative length tolerance to pass to unit_cell.is_similar_to :type uc_rel_length_tolerance: float :param uc_abs_angle_tolerance: absolute angle tolerance to pass to unit_cell.is_similar_to :type uc_abs_angle_tolerance: float :returns: Whether the other crystal model is similar to this :rtype: bool ''' # space group test if self.get_space_group() != other.get_space_group(): return False # mosaicity test m_a, m_b = self.get_mosaicity(deg=True), other.get_mosaicity(deg=True) min_m, max_m = min(m_a, m_b), max(m_a, m_b) if min_m <= 0.0: if max_m > 0.0: return False elif min_m / max_m < mosaicity_tolerance: return False # static orientation test U_a, U_b = self.get_U(), other.get_U() assert U_a.is_r3_rotation_matrix() assert U_b.is_r3_rotation_matrix() R_ab = U_b * U_a.transpose() uq = R_ab.r3_rotation_matrix_as_unit_quaternion() angle = uq.unit_quaternion_as_axis_and_angle(deg=True)[0] if abs(angle) > angle_tolerance: return False # static unit cell test uc_a, uc_b = self.get_unit_cell(), other.get_unit_cell() if not uc_a.is_similar_to(uc_b, relative_length_tolerance=uc_rel_length_tolerance, absolute_angle_tolerance=uc_abs_angle_tolerance): return False # scan varying tests if self.num_scan_points > 0: if other.num_scan_points != self.num_scan_points: return False for i in range(self.num_scan_points): U_a, U_b = self.get_U_at_scan_point(i), other.get_U_at_scan_point(i) assert U_a.is_r3_rotation_matrix() assert U_b.is_r3_rotation_matrix() R_ab = U_b * U_a.transpose() uq = R_ab.r3_rotation_matrix_as_unit_quaternion() angle = uq.unit_quaternion_as_axis_and_angle(deg=True)[0] if abs(angle) > angle_tolerance: return False B_a, B_b = self.get_B_at_scan_point(i), other.get_B_at_scan_point(i) uc_a = crystal_orientation(B_a,True).unit_cell() uc_b = crystal_orientation(B_b,True).unit_cell() if not uc_a.is_similar_to(uc_b, relative_length_tolerance=uc_rel_length_tolerance, absolute_angle_tolerance=uc_abs_angle_tolerance): return False return True
def calc_partiality_anisotropy_set( self, my_uc, rotx, roty, miller_indices, ry, rz, r0, re, nu, bragg_angle_set, alpha_angle_set, wavelength, crystal_init_orientation, spot_pred_x_mm_set, spot_pred_y_mm_set, detector_distance_mm, partiality_model, flag_beam_divergence, ): # use III.4 in Winkler et al 1979 (A35; P901) for set of miller indices O = sqr(my_uc.orthogonalization_matrix()).transpose() R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose() CO = crystal_orientation(O * R, basis_type.direct) CO_rotate = CO.rotate_thru((1, 0, 0), rotx).rotate_thru((0, 1, 0), roty) A_star = sqr(CO_rotate.reciprocal_matrix()) S0 = -1 * col((0, 0, 1.0 / wavelength)) # caculate rs rs_set = r0 + (re * flex.tan(bragg_angle_set)) if flag_beam_divergence: rs_set += ((ry * flex.cos(alpha_angle_set))**2 + (rz * flex.sin(alpha_angle_set))**2)**(1 / 2) # calculate rh x = A_star.elems * miller_indices.as_vec3_double() sd_array = x + S0.elems rh_set = sd_array.norms() - (1 / wavelength) # calculate partiality if partiality_model == "Lorentzian": partiality_set = (rs_set**2) / ((2 * (rh_set**2)) + (rs_set**2)) elif partiality_model == "Voigt": partiality_set = self.voigt(rh_set, rs_set, nu) elif partiality_model == "Lognormal": partiality_set = self.lognpdf(rh_set, rs_set, nu) # calculate delta_xy if sum(spot_pred_y_mm_set) == 0: # hack for dials integration - spot_pred_x_mm_set is s1 * to be fixed * delta_xy_set = (spot_pred_x_mm_set - sd_array).norms() else: d_ratio = -detector_distance_mm / sd_array.parts()[2] calc_xy_array = flex.vec3_double( sd_array.parts()[0] * d_ratio, sd_array.parts()[1] * d_ratio, flex.double([0] * len(d_ratio)), ) pred_xy_array = flex.vec3_double(spot_pred_x_mm_set, spot_pred_y_mm_set, flex.double([0] * len(d_ratio))) delta_xy_set = (pred_xy_array - calc_xy_array).norms() return partiality_set, delta_xy_set, rs_set, rh_set
def test_compare_example(): experiments = ExperimentListFactory.from_json(json_is, check_format=False) for experiment in experiments: header = { 'DIM': '2', 'DENZO_X_BEAM': '97.185', 'RANK': '0', 'PREFIX': 'step5_000009', 'BEAM_CENTER_Y': '97.13', 'BEAM_CENTER_X': '97.13', 'WAVELENGTH': '1.30432', 'OSC_START': '0', 'ADC_OFFSET': '10', 'BYTE_ORDER': 'little_endian', 'DIRECT_SPACE_ABC': '2.7790989649304656,3.721227037283121,0.616256870237976,-4.231398741367156,1.730877297917864,1.0247061633019547,2.9848502901631253,-4.645083818143041,28.595825588147285', 'OSC_RANGE': '0', 'DIALS_ORIGIN': '-97.185,97.185,-50', 'MOSFLM_CENTER_X': '97.13', 'MOSFLM_CENTER_Y': '97.13', 'CLOSE_DISTANCE': '50', 'BEAMLINE': 'fake', 'TWOTHETA': '0', 'ADXV_CENTER_Y': '96.965', 'ADXV_CENTER_X': '97.185', 'HEADER_BYTES': '1024', 'DETECTOR_SN': '000', 'DISTANCE': '50', 'PHI': '0', 'SIZE1': '1765', 'SIZE2': '1765', 'XDS_ORGX': '884', 'XDS_ORGY': '884', 'DENZO_Y_BEAM': '97.185', 'TIME': '1', 'TYPE': 'unsigned_short', 'PIXEL_SIZE': '0.11' } # the header of the simulated image, containing ground truth orientation rsabc = sqr([float(v) for v in header['DIRECT_SPACE_ABC'].split(',') ]) * permute.inverse() rsa = rsabc[0:3] rsb = rsabc[3:6] rsc = rsabc[6:9] header_cryst = Crystal(rsa, rsb, rsc, 'P1').change_basis(CB_OP_C_P.inverse()) header_cryst.set_space_group(experiment.crystal.get_space_group()) print('Header crystal') print(header_cryst) expt_crystal = experiment.crystal print('Integrated crystal') print(expt_crystal) header_ori = crystal_orientation.crystal_orientation( header_cryst.get_A(), crystal_orientation.basis_type.reciprocal) expt_ori = crystal_orientation.crystal_orientation( expt_crystal.get_A(), crystal_orientation.basis_type.reciprocal) print('Converted to cctbx') header_ori.show() expt_ori.show() # assert the equivalence between dxtbx crystal object and the cctbx crystal_orientation object assert approx_equal(header_cryst.get_U(), header_ori.get_U_as_sqr()) assert approx_equal(header_cryst.get_A(), header_ori.reciprocal_matrix()) assert approx_equal( header_cryst.get_B(), sqr(header_ori.unit_cell().fractionalization_matrix()).transpose()) assert approx_equal(expt_crystal.get_U(), expt_ori.get_U_as_sqr()) assert approx_equal(expt_crystal.get_A(), expt_ori.reciprocal_matrix()) assert approx_equal( expt_crystal.get_B(), sqr(expt_ori.unit_cell().fractionalization_matrix()).transpose()) cb_op_align = sqr( expt_ori.best_similarity_transformation(header_ori, 50, 1)) print('XYZ angles', cb_op_align.r3_rotation_matrix_as_x_y_z_angles(deg=True)) aligned_ori = expt_ori.change_basis(cb_op_align) U_integrated = aligned_ori.get_U_as_sqr() U_ground_truth = header_ori.get_U_as_sqr() missetting_rot = U_integrated * U_ground_truth.inverse() print("determinant", missetting_rot.determinant()) assert approx_equal(missetting_rot.determinant(), 1.0) assert missetting_rot.is_r3_rotation_matrix() angle, axis = missetting_rot.r3_rotation_matrix_as_unit_quaternion( ).unit_quaternion_as_axis_and_angle(deg=True) print("Angular offset is %13.10f deg." % (angle)) assert approx_equal(angle, 0.2609472065)
def postrefine_by_frame(self, frame_no, pres_in, iparams, miller_array_ref, avg_mode): #Prepare data if pres_in is None: return None, 'Found empty pickle file' observations_pickle = pickle.load(open(pres_in.pickle_filename,"rb")) wavelength = observations_pickle["wavelength"] crystal_init_orientation = observations_pickle["current_orientation"][0] pickle_filename = pres_in.pickle_filename pickle_filepaths = pickle_filename.split('/') img_filename_only = pickle_filepaths[len(pickle_filepaths)-1] txt_exception = ' {0:40} ==> '.format(img_filename_only) inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) if inputs is not None: observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \ detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs else: txt_exception += txt_organize_input + '\n' return None, txt_exception #Select data for post-refinement (only select indices that are common with the reference set observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams) matches = miller.match_multi_indices( miller_indices_unique=miller_array_ref.indices(), miller_indices=observations_non_polar.indices()) I_ref_match = flex.double([miller_array_ref.data()[pair[0]] for pair in matches.pairs()]) miller_indices_ref_match = flex.miller_index((miller_array_ref.indices()[pair[0]] for pair in matches.pairs())) I_obs_match = flex.double([observations_non_polar.data()[pair[1]] for pair in matches.pairs()]) sigI_obs_match = flex.double([observations_non_polar.sigmas()[pair[1]] for pair in matches.pairs()]) miller_indices_original_obs_match = flex.miller_index((observations_original.indices()[pair[1]] \ for pair in matches.pairs())) miller_indices_non_polar_obs_match = flex.miller_index((observations_non_polar.indices()[pair[1]] \ for pair in matches.pairs())) alpha_angle_set = flex.double([alpha_angle[pair[1]] for pair in matches.pairs()]) spot_pred_x_mm_set = flex.double([spot_pred_x_mm[pair[1]] for pair in matches.pairs()]) spot_pred_y_mm_set = flex.double([spot_pred_y_mm[pair[1]] for pair in matches.pairs()]) references_sel = miller_array_ref.customized_copy(data=I_ref_match, indices=miller_indices_ref_match) observations_original_sel = observations_original.customized_copy(data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_original_obs_match) observations_non_polar_sel = observations_non_polar.customized_copy(data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_non_polar_obs_match) #Do least-squares refinement lsqrh = leastsqr_handler() try: refined_params, stats, n_refl_postrefined = lsqrh.optimize(I_ref_match, observations_original_sel, wavelength, crystal_init_orientation, alpha_angle_set, spot_pred_x_mm_set, spot_pred_y_mm_set, iparams, pres_in, observations_non_polar_sel, detector_distance_mm) except Exception: txt_exception += 'optimization failed.\n' return None, txt_exception #caculate partiality for output (with target_anomalous check) G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, \ a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = refined_params inputs, txt_organize_input = self.organize_input(observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename) observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, \ detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs observations_non_polar, index_basis_name = self.get_observations_non_polar(observations_original, pickle_filename, iparams) from cctbx.uctbx import unit_cell uc_fin = unit_cell((a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin)) crystal_init_orientation = pres_in.crystal_orientation two_theta = observations_original.two_theta(wavelength=wavelength).data() ph = partiality_handler() partiality_fin, dummy, rs_fin, rh_fin = ph.calc_partiality_anisotropy_set(uc_fin, rotx_fin, roty_fin, observations_original.indices(), ry_fin, rz_fin, r0_fin, re_fin, voigt_nu_fin, two_theta, alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence) #calculate the new crystal orientation O = sqr(uc_fin.orthogonalization_matrix()).transpose() R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose() from cctbx.crystal_orientation import crystal_orientation, basis_type CO = crystal_orientation(O*R, basis_type.direct) crystal_fin_orientation = CO.rotate_thru((1,0,0), rotx_fin ).rotate_thru((0,1,0), roty_fin) #remove reflections with partiality below threshold i_sel = partiality_fin > iparams.merge.partiality_min partiality_fin_sel = partiality_fin.select(i_sel) rs_fin_sel = rs_fin.select(i_sel) rh_fin_sel = rh_fin.select(i_sel) observations_non_polar_sel = observations_non_polar.select(i_sel) observations_original_sel = observations_original.select(i_sel) mapped_predictions = mapped_predictions.select(i_sel) pres = postref_results() pres.set_params(observations = observations_non_polar_sel, observations_original = observations_original_sel, refined_params=refined_params, stats=stats, partiality=partiality_fin_sel, rs_set=rs_fin_sel, rh_set=rh_fin_sel, frame_no=frame_no, pickle_filename=pickle_filename, wavelength=wavelength, crystal_orientation=crystal_init_orientation, detector_distance_mm=detector_distance_mm, identified_isoform=identified_isoform, mapped_predictions=mapped_predictions, xbeam=xbeam, ybeam=ybeam) r_change, r_xy_change, cc_change, cc_iso_change = (0,0,0,0) try: r_change = ((pres.R_final - pres.R_init)/pres.R_init)*100 r_xy_change = ((pres.R_xy_final - pres.R_xy_init)/pres.R_xy_init)*100 cc_change = ((pres.CC_final - pres.CC_init)/pres.CC_init)*100 cc_iso_change = ((pres.CC_iso_final - pres.CC_iso_init)/pres.CC_iso_init)*100 except Exception: pass txt_postref= ' {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} R:{3:8.2f}% RXY:{4:8.2f}% CC:{5:6.2f}% CCISO:{6:6.2f}% G:{7:10.3e} B:{8:7.1f} CELL:{9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f}'.format(img_filename_only+' ('+index_basis_name+')', observations_original_sel.d_min(), len(observations_original_sel.data()), r_change, r_xy_change, cc_change, cc_iso_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin) print txt_postref txt_postref += '\n' return pres, txt_postref
dm = {} dn = {} for key in data: if key == "correction_vectors": # drop correction_vectors as they aren't as easy to split up continue elif key in ["current_cb_op_to_primitive", "effective_tiling", "pointgroup", "identified_isoform"]: dl[key] = data[key] dr[key] = data[key] dm[key] = data[key] dn[key] = data[key] continue elif key == "current_orientation": from cctbx import crystal_orientation dl[key] = [crystal_orientation.crystal_orientation(c) for c in data[key]] dr[key] = [crystal_orientation.crystal_orientation(c) for c in data[key]] dm[key] = [crystal_orientation.crystal_orientation(c) for c in data[key]] dn[key] = [crystal_orientation.crystal_orientation(c) for c in data[key]] continue try: assert len(data[key]) == len(sel_l) islist = True except TypeError, e: islist = False if islist: val_l = [] val_r = [] val_m = [] val_n = []
rot_mat = rotation_vector.axis_and_angle_as_r3_rotation_matrix( omegas[omegaidx] + dangles[omegaidx][n] * 0.1) HP = (rot_mat * matrix.sqr(Amat) ) * matrix.col(hkl + difference) + beam_vector #print "%10.7f %10.7f %10.7f"%HP.elems assert approx_equal(H1, HP, eps=1E-5) #print if __name__ == '__main__': wavelength = 1.2 resolution = 3.0 Amat = (0.0038039968697463817, 0.004498689311366309, 0.0044043429203887785, -0.00477859183801569, 0.006594300357213904, -0.002402759536958918, -0.01012056453488894, -0.0014226943325514182, 0.002789954423701981) orient = crystal_orientation(Amat, True) calc = partial_spot_position_partial_H( limiting_resolution=resolution, orientation=orient.reciprocal_matrix(), wavelength=wavelength, axial_direction=(0., 1., 0.)) rotation_scattering(calc, matrix.col((0., 1., 0.)), Amat, wavelength) test_finite_differences(calc, matrix.col((0., 1., 0.)), Amat, wavelength) print "OK"
) # from C to P print(str(CB_OP_C_P)) icount = 0 from scitbx.array_family import flex angles = flex.double() for stuff in get_items(): #print stuff icount += 1 print("Iteration", icount) # work up the crystal model from integration direct_A = stuff["integrated_crystal_model"].get_A_inverse_as_sqr() permute = sqr((0, 0, 1, 0, 1, 0, -1, 0, 0)) sim_compatible = direct_A * permute # permute columns when post multiplying from cctbx import crystal_orientation integrated_Ori = crystal_orientation.crystal_orientation( sim_compatible, crystal_orientation.basis_type.direct) #integrated_Ori.show(legend="integrated") # work up the crystal model from postrefinement direct_A = stuff["postref"].inverse() permute = sqr((0, 0, 1, 0, 1, 0, -1, 0, 0)) sim_compatible = direct_A * permute # permute columns when post multiplying from cctbx import crystal_orientation postref_Ori = crystal_orientation.crystal_orientation( sim_compatible, crystal_orientation.basis_type.direct) # work up the ground truth from header header_Ori = crystal_orientation.crystal_orientation( stuff["ABC"], crystal_orientation.basis_type.direct) #header_Ori.show(legend="header_Ori")
def run(args): distance = 125 centre = (97.075, 97.075) pix_size = (0.11, 0.11) image_size = (1765, 1765) wavelength = args.w or 1.0 # 1. Make a dummy detector detector = detector_factory.simple('SENSOR_UNKNOWN', # Sensor distance, centre, '+x','-y', # fast/slow direction pix_size, image_size) # 2. Get the miller array! mill_array = process_mtz(args.mtzfile[0]).as_intensity_array() ortho = sqr(mill_array.crystal_symmetry().unit_cell().reciprocal() \ .orthogonalization_matrix()) # 3.Create some image_pickle dictionairies that contain 'full' intensities, # but are otherwise complete. im = 0 while im < args.n: im += 1 A = sqr(flex.random_double_r3_rotation_matrix()) * ortho orientation = crystal_orientation(A, basis_type.reciprocal) pix_coords, miller_set = get_pix_coords(wavelength, A, mill_array, detector) if len(miller_set) > 10: # at least 10 reflections miller_set = cctbx.miller.set(mill_array.crystal_symmetry(), miller_set, anomalous_flag=False) obs = mill_array.common_set(miller_set) temp_dict = {'observations': [obs], 'mapped_predictions': [pix_coords], 'pointgroup': None, 'current_orientation': [orientation], 'xbeam': centre[0], 'ybeam': centre[1], 'wavelength': wavelength} old_node = ImageNode(dicti=temp_dict, scale=False) # Remove all reflection that are not at least p partial partial_sel = (old_node.partialities > p_threshold) temp_dict['full_observations'] = [obs.select(partial_sel)] temp_dict['observations'] = [obs.select(partial_sel) * old_node.partialities.select(partial_sel)] temp_dict['mapped_predictions'] = \ [temp_dict['mapped_predictions'][0].select(partial_sel)] if logging.Logger.root.level <= logging.DEBUG: # debug! before = temp_dict['full_observations'][0] after = temp_dict['observations'][0] / old_node.partialities assert sum(abs(before.data() - after.data())) < eps if args.r: partials = list(temp_dict['observations'][0].data()) jiggled_partials = flex.double([random.gauss(obs, args.r * obs) for obs in partials]) temp_dict['observations'][0] = temp_dict['observations'][0] \ .customized_copy(data=jiggled_partials) pkl_name = "simulated_data_{0:04d}.pickle".format(im) with(open(pkl_name, 'wb')) as pkl: cPickle.dump(temp_dict, pkl) ''' Only works with no noise:
def run_sim2smv(img_prefix=None, simparams=None,pdb_lines=None,crystal=None,spectra=None,rotation=None,rank=None,fsave=None,sfall_cluster=None,quick=False): smv_fileout = fsave direct_algo_res_limit = simparams.direct_algo_res_limit wavlen, flux, real_wavelength_A = next(spectra) # list of lambdas, list of fluxes, average wavelength real_flux = flex.sum(flux) assert real_wavelength_A > 0 # print(rank, " ## real_wavelength_A/real_flux = ", real_wavelength_A, real_flux*1.0/simparams.flux) if quick: wavlen = flex.double([real_wavelength_A]) flux = flex.double([real_flux]) # GF = gen_fmodel(resolution=simparams.direct_algo_res_limit,pdb_text=pdb_lines,algorithm=simparams.fmodel_algorithm,wavelength=real_wavelength_A) # GF.set_k_sol(simparams.k_sol) # GF.make_P1_primitive() sfall_main = sfall_cluster["main"] #GF.get_amplitudes() # use crystal structure to initialize Fhkl array # sfall_main.show_summary(prefix = "Amplitudes used ") N = crystal.number_of_cells(sfall_main.unit_cell()) #print("## number of N = ", N) SIM = nanoBragg(detpixels_slowfast=(simparams.detector_size_ny,simparams.detector_size_nx),pixel_size_mm=simparams.pixel_size_mm,\ Ncells_abc=(N,N,N),wavelength_A=real_wavelength_A,verbose=0) # workaround for problem with wavelength array, specify it separately in constructor. # SIM.adc_offset_adu = 0 # Do not offset by 40 SIM.adc_offset_adu = 10 # Do not offset by 40 SIM.seed = 0 # SIM.randomize_orientation() SIM.mosaic_spread_deg = simparams.mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = simparams.mosaic_domains # 77 seconds. SIM.distance_mm = simparams.distance_mm ## setup the mosaicity UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi/180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append( site.axis_and_angle_as_r3_rotation_matrix(m,deg=False) ) SIM.set_mosaic_blocks(UMAT_nm) ###################### SIM.beamcenter_convention=convention.ADXV SIM.beam_center_mm=(simparams.beam_center_x_mm, simparams.beam_center_y_mm) # 95.975 96.855 ###################### # get same noise each time this test is run SIM.seed = 0 SIM.oversample=simparams.oversample SIM.wavelength_A = real_wavelength_A SIM.polarization=simparams.polarization # this will become F000, marking the beam center SIM.default_F=simparams.default_F #SIM.missets_deg= (10,20,30) SIM.Fhkl=sfall_main Amatrix_rot = (rotation * sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose() SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter # print("## inside run_sim2smv, Amat_rot = ", Amatrix_rot) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM Ori = crystal_orientation.crystal_orientation(Amat, crystal_orientation.basis_type.reciprocal) SIM.xtal_shape=shapetype.Gauss # both crystal & RLP are Gaussian SIM.progress_meter=False # SIM.show_params() # flux is always in photons/s SIM.flux=real_flux SIM.exposure_s=simparams.exposure_s # assumes round beam SIM.beamsize_mm=simparams.beamsize_mm #cannot make this 3 microns; spots are too intense temp=SIM.Ncells_abc SIM.Ncells_abc=temp # print("## domains_per_crystal = ", crystal.domains_per_crystal) SIM.raw_pixels *= crystal.domains_per_crystal # must calculate the correct scale! # print("## Initial raw_pixels = ", flex.sum(SIM.raw_pixels)) for x in range(len(flux)): # CH = channel_pixels(wavlen[x],flux[x],N,UMAT_nm,Amatrix_rot,sfall_cluster[x],rank) # print("## in loop wavlen/flux/real_wavelength_A = ", wavlen[x], flux[x]/real_flux, real_wavelength_A) CH = channel_pixels(simparams=simparams,single_wavelength_A=wavlen[x],single_flux=flux[x],N=N,UMAT_nm=UMAT_nm, \ Amatrix_rot=Amatrix_rot,sfall_channel=sfall_cluster[x],rank=rank) SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal CHDBG_singleton.extract(channel_no=x, data=CH.raw_pixels) CH.free_all() # print("## sum raw_pixels after ", x, "is", flex.sum(SIM.raw_pixels)) # rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor bg = flex.vec2_double([(0,2.57),(0.0365,2.58),(0.07,2.8),(0.12,5),(0.162,8),(0.2,6.75),(0.18,7.32),(0.216,6.75),(0.236,6.5),(0.28,4.5),(0.3,4.3),(0.345,4.36),(0.436,3.77),(0.5,3.17)]) SIM.Fbg_vs_stol = bg SIM.amorphous_sample_thick_mm = simparams.water_sample_thick_mm SIM.amorphous_density_gcm3 = simparams.water_density_gcm3 SIM.amorphous_molecular_weight_Da = simparams.water_molecular_weight_Da SIM.flux=real_flux SIM.beamsize_mm=simparams.beamsize_mm # square (not user specified) SIM.exposure_s=simparams.exposure_s # multiplies flux x exposure SIM.add_background() # rough approximation to air bg = flex.vec2_double([(0,14.1),(0.045,13.5),(0.174,8.35),(0.35,4.78),(0.5,4.22)]) SIM.Fbg_vs_stol = bg SIM.amorphous_sample_thick_mm = simparams.air_sample_thick_mm # between beamstop and collimator SIM.amorphous_density_gcm3 = simparams.air_density_gcm3 SIM.amorphous_sample_molecular_weight_Da = simparams.air_molecular_weight_Da # nitrogen = N2 SIM.add_background() #apply beamstop mask here # settings for CCD SIM.detector_psf_kernel_radius_pixels=simparams.detector_psf_kernel_radius_pixels SIM.detector_psf_type=shapetype.Unknown # for CSPAD SIM.detector_psf_fwhm_mm=simparams.detector_psf_fwhm_mm SIM.apply_psf() SIM.add_noise() extra = "PREFIX=%s;\nRANK=%d;\n"%(img_prefix,rank) SIM.to_smv_format_py(fileout=smv_fileout,intfile_scale=1,rotmat=True,extra=extra,gz=True) SIM.free_all()
def test_compare_example(): experiments = ExperimentListFactory.from_json(json_is, check_format=False) for experiment in experiments: header = { "DIM": "2", "DENZO_X_BEAM": "97.185", "RANK": "0", "PREFIX": "step5_000009", "BEAM_CENTER_Y": "97.13", "BEAM_CENTER_X": "97.13", "WAVELENGTH": "1.30432", "OSC_START": "0", "ADC_OFFSET": "10", "BYTE_ORDER": "little_endian", "DIRECT_SPACE_ABC": "2.7790989649304656,3.721227037283121,0.616256870237976,-4.231398741367156,1.730877297917864,1.0247061633019547,2.9848502901631253,-4.645083818143041,28.595825588147285", "OSC_RANGE": "0", "DIALS_ORIGIN": "-97.185,97.185,-50", "MOSFLM_CENTER_X": "97.13", "MOSFLM_CENTER_Y": "97.13", "CLOSE_DISTANCE": "50", "BEAMLINE": "fake", "TWOTHETA": "0", "ADXV_CENTER_Y": "96.965", "ADXV_CENTER_X": "97.185", "HEADER_BYTES": "1024", "DETECTOR_SN": "000", "DISTANCE": "50", "PHI": "0", "SIZE1": "1765", "SIZE2": "1765", "XDS_ORGX": "884", "XDS_ORGY": "884", "DENZO_Y_BEAM": "97.185", "TIME": "1", "TYPE": "unsigned_short", "PIXEL_SIZE": "0.11", } # the header of the simulated image, containing ground truth orientation rsabc = (sqr([float(v) for v in header["DIRECT_SPACE_ABC"].split(",")]) * permute.inverse()) rsa = rsabc[0:3] rsb = rsabc[3:6] rsc = rsabc[6:9] header_cryst = Crystal(rsa, rsb, rsc, "P1").change_basis(CB_OP_C_P.inverse()) header_cryst.set_space_group(experiment.crystal.get_space_group()) print("Header crystal") print(header_cryst) expt_crystal = experiment.crystal print("Integrated crystal") print(expt_crystal) header_ori = crystal_orientation.crystal_orientation( header_cryst.get_A(), crystal_orientation.basis_type.reciprocal) expt_ori = crystal_orientation.crystal_orientation( expt_crystal.get_A(), crystal_orientation.basis_type.reciprocal) print("Converted to cctbx") header_ori.show() expt_ori.show() # assert the equivalence between dxtbx crystal object and the cctbx crystal_orientation object assert approx_equal(header_cryst.get_U(), header_ori.get_U_as_sqr()) assert approx_equal(header_cryst.get_A(), header_ori.reciprocal_matrix()) assert approx_equal( header_cryst.get_B(), sqr(header_ori.unit_cell().fractionalization_matrix()).transpose(), ) assert approx_equal(expt_crystal.get_U(), expt_ori.get_U_as_sqr()) assert approx_equal(expt_crystal.get_A(), expt_ori.reciprocal_matrix()) assert approx_equal( expt_crystal.get_B(), sqr(expt_ori.unit_cell().fractionalization_matrix()).transpose(), ) cb_op_align = sqr( expt_ori.best_similarity_transformation(header_ori, 50, 1)) print("XYZ angles", cb_op_align.r3_rotation_matrix_as_x_y_z_angles(deg=True)) aligned_ori = expt_ori.change_basis(cb_op_align) U_integrated = aligned_ori.get_U_as_sqr() U_ground_truth = header_ori.get_U_as_sqr() missetting_rot = U_integrated * U_ground_truth.inverse() print("determinant", missetting_rot.determinant()) assert approx_equal(missetting_rot.determinant(), 1.0) assert missetting_rot.is_r3_rotation_matrix() angle, axis = missetting_rot.r3_rotation_matrix_as_unit_quaternion( ).unit_quaternion_as_axis_and_angle(deg=True) print("Angular offset is %13.10f deg." % (angle)) assert approx_equal(angle, 0.2609472065)
def run_sim2smv(prefix, crystal, spectra, rotation, quick=False): direct_algo_res_limit = 2.0 wavlen, flux, wavelength_A = next( spectra) # list of lambdas, list of fluxes, average wavelength if quick: wavlen = flex.double([wavelength_A]) flux = flex.double([flex.sum(flux)]) print("Quick sim, lambda=%f, flux=%f" % (wavelength_A, flux[0])) #sfall = fcalc_from_pdb(resolution=direct_algo_res_limit,pdb_text=pdb_lines,algorithm="direct",wavelength=SIM.wavelength_A) sfall = fmodel_from_pdb(resolution=direct_algo_res_limit, pdb_text=pdb_lines, algorithm="fft", wavelength=wavelength_A) # use crystal structure to initialize Fhkl array sfall.show_summary(prefix="Amplitudes used ") N = crystal.number_of_cells(sfall.unit_cell()) #SIM = nanoBragg(detpixels_slowfast=(2000,2000),pixel_size_mm=0.11,Ncells_abc=(5,5,5),verbose=0) SIM = nanoBragg( detpixels_slowfast=(2000, 2000), pixel_size_mm=0.11, Ncells_abc=(N, N, N), # workaround for problem with wavelength array, specify it separately in constructor. wavelength_A=wavelength_A, verbose=0) SIM.adc_offset_adu = 0 # Do not offset by 40 SIM.adc_offset_adu = 10 # Do not offset by 40 import sys if len(sys.argv) > 2: SIM.seed = -int(sys.argv[2]) print("GOTHERE seed=", SIM.seed) if len(sys.argv) > 1: if sys.argv[1] == "random": SIM.randomize_orientation() SIM.mosaic_domains = 25 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image # 3000000 images would be 100000 hours on a 60-core machine (dials), or 11.4 years # using 2 nodes, 5.7 years. Do this at SLAC? NERSC? combination of all? # SLAC downtimes: Tues Dec 5 (24 hrs), Mon Dec 11 (72 hrs), Mon Dec 18 light use, 24 days SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev SIM.distance_mm = 141.7 UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) SIM.set_mosaic_blocks(UMAT_nm) #SIM.detector_thick_mm = 0.5 # = 0 for Rayonix #SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD #SIM.detector_attenuation_length_mm = default is silicon # get same noise each time this test is run SIM.seed = 1 SIM.oversample = 1 SIM.wavelength_A = wavelength_A SIM.polarization = 1 # this will become F000, marking the beam center SIM.default_F = 0 #SIM.missets_deg= (10,20,30) print("mosaic_seed=", SIM.mosaic_seed) print("seed=", SIM.seed) print("calib_seed=", SIM.calib_seed) print("missets_deg =", SIM.missets_deg) SIM.Fhkl = sfall print("Determinant", rotation.determinant()) Amatrix_rot = ( rotation * sqr(sfall.unit_cell().orthogonalization_matrix())).transpose() print("RAND_ORI", prefix, end=' ') print(" ".join([i for i in Amatrix_rot])) SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter print("unit_cell_Adeg=", SIM.unit_cell_Adeg) print("unit_cell_tuple=", SIM.unit_cell_tuple) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM from cctbx import crystal_orientation Ori = crystal_orientation.crystal_orientation( Amat, crystal_orientation.basis_type.reciprocal) print("Python unit cell from SIM state", Ori.unit_cell()) # fastest option, least realistic #SIM.xtal_shape=shapetype.Tophat # RLP = hard sphere #SIM.xtal_shape=shapetype.Square # gives fringes SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian #SIM.xtal_shape=shapetype.Round # Crystal is a hard sphere # only really useful for long runs SIM.progress_meter = False # prints out value of one pixel only. will not render full image! #SIM.printout_pixel_fastslow=(500,500) #SIM.printout=True SIM.show_params() # flux is always in photons/s SIM.flux = 1e12 SIM.exposure_s = 1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense temp = SIM.Ncells_abc print("Ncells_abc=", SIM.Ncells_abc) SIM.Ncells_abc = temp print("Ncells_abc=", SIM.Ncells_abc) print("xtal_size_mm=", SIM.xtal_size_mm) print("unit_cell_Adeg=", SIM.unit_cell_Adeg) print("unit_cell_tuple=", SIM.unit_cell_tuple) print("missets_deg=", SIM.missets_deg) print("Amatrix=", SIM.Amatrix) print("beam_center_mm=", SIM.beam_center_mm) print("XDS_ORGXY=", SIM.XDS_ORGXY) print("detector_pivot=", SIM.detector_pivot) print("xtal_shape=", SIM.xtal_shape) print("beamcenter_convention=", SIM.beamcenter_convention) print("fdet_vector=", SIM.fdet_vector) print("sdet_vector=", SIM.sdet_vector) print("odet_vector=", SIM.odet_vector) print("beam_vector=", SIM.beam_vector) print("polar_vector=", SIM.polar_vector) print("spindle_axis=", SIM.spindle_axis) print("twotheta_axis=", SIM.twotheta_axis) print("distance_meters=", SIM.distance_meters) print("distance_mm=", SIM.distance_mm) print("close_distance_mm=", SIM.close_distance_mm) print("detector_twotheta_deg=", SIM.detector_twotheta_deg) print("detsize_fastslow_mm=", SIM.detsize_fastslow_mm) print("detpixels_fastslow=", SIM.detpixels_fastslow) print("detector_rot_deg=", SIM.detector_rot_deg) print("curved_detector=", SIM.curved_detector) print("pixel_size_mm=", SIM.pixel_size_mm) print("point_pixel=", SIM.point_pixel) print("polarization=", SIM.polarization) print("nopolar=", SIM.nopolar) print("oversample=", SIM.oversample) print("region_of_interest=", SIM.region_of_interest) print("wavelength_A=", SIM.wavelength_A) print("energy_eV=", SIM.energy_eV) print("fluence=", SIM.fluence) print("flux=", SIM.flux) print("exposure_s=", SIM.exposure_s) print("beamsize_mm=", SIM.beamsize_mm) print("dispersion_pct=", SIM.dispersion_pct) print("dispsteps=", SIM.dispsteps) print("divergence_hv_mrad=", SIM.divergence_hv_mrad) print("divsteps_hv=", SIM.divsteps_hv) print("divstep_hv_mrad=", SIM.divstep_hv_mrad) print("round_div=", SIM.round_div) print("phi_deg=", SIM.phi_deg) print("osc_deg=", SIM.osc_deg) print("phisteps=", SIM.phisteps) print("phistep_deg=", SIM.phistep_deg) print("detector_thick_mm=", SIM.detector_thick_mm) print("detector_thicksteps=", SIM.detector_thicksteps) print("detector_thickstep_mm=", SIM.detector_thickstep_mm) print("***mosaic_spread_deg=", SIM.mosaic_spread_deg) print("***mosaic_domains=", SIM.mosaic_domains) print("indices=", SIM.indices) print("amplitudes=", SIM.amplitudes) print("Fhkl_tuple=", SIM.Fhkl_tuple) print("default_F=", SIM.default_F) print("interpolate=", SIM.interpolate) print("integral_form=", SIM.integral_form) from libtbx.development.timers import Profiler P = Profiler("nanoBragg") # now actually burn up some CPU #SIM.add_nanoBragg_spots() del P # simulated crystal is only 125 unit cells (25 nm wide) # amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume) print(crystal.domains_per_crystal) SIM.raw_pixels *= crystal.domains_per_crystal # must calculate the correct scale! for x in range(len(flux)): print("+++++++++++++++++++++++++++++++++++++++ Wavelength", x) CH = channel_pixels(wavlen[x], flux[x], N, UMAT_nm, Amatrix_rot, sfall) SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal CH.free_all() if quick: SIM.to_smv_format(fileout=prefix + "_intimage_001.img") # rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor bg = flex.vec2_double([(0, 2.57), (0.0365, 2.58), (0.07, 2.8), (0.12, 5), (0.162, 8), (0.2, 6.75), (0.18, 7.32), (0.216, 6.75), (0.236, 6.5), (0.28, 4.5), (0.3, 4.3), (0.345, 4.36), (0.436, 3.77), (0.5, 3.17)]) SIM.Fbg_vs_stol = bg SIM.amorphous_sample_thick_mm = 0.1 SIM.amorphous_density_gcm3 = 1 SIM.amorphous_molecular_weight_Da = 18 SIM.flux = 1e12 SIM.beamsize_mm = 0.003 # square (not user specified) SIM.exposure_s = 1.0 # multiplies flux x exposure SIM.add_background() if quick: SIM.to_smv_format(fileout=prefix + "_intimage_002.img") # rough approximation to air bg = flex.vec2_double([(0, 14.1), (0.045, 13.5), (0.174, 8.35), (0.35, 4.78), (0.5, 4.22)]) SIM.Fbg_vs_stol = bg #SIM.amorphous_sample_thick_mm = 35 # between beamstop and collimator SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator SIM.amorphous_density_gcm3 = 1.2e-3 SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2 print("amorphous_sample_size_mm=", SIM.amorphous_sample_size_mm) print("amorphous_sample_thick_mm=", SIM.amorphous_sample_thick_mm) print("amorphous_density_gcm3=", SIM.amorphous_density_gcm3) print("amorphous_molecular_weight_Da=", SIM.amorphous_molecular_weight_Da) SIM.add_background() #apply beamstop mask here # set this to 0 or -1 to trigger automatic radius. could be very slow with bright images # settings for CCD SIM.detector_psf_kernel_radius_pixels = 5 #SIM.detector_psf_fwhm_mm=0.08; #SIM.detector_psf_type=shapetype.Fiber # rayonix=Fiber, CSPAD=None (or small Gaussian) SIM.detector_psf_type = shapetype.Unknown # for CSPAD SIM.detector_psf_fwhm_mm = 0 #SIM.apply_psf() print("One pixel-->", SIM.raw_pixels[500000]) # at this point we scale the raw pixels so that the output array is on an scale from 0 to 50000. # that is the default behavior (intfile_scale<=0), otherwise it applies intfile_scale as a multiplier on an abs scale. if quick: SIM.to_smv_format(fileout=prefix + "_intimage_003.img") print("quantum_gain=", SIM.quantum_gain) #defaults to 1. converts photons to ADU print("adc_offset_adu=", SIM.adc_offset_adu) print("detector_calibration_noise_pct=", SIM.detector_calibration_noise_pct) print("flicker_noise_pct=", SIM.flicker_noise_pct) print("readout_noise_adu=", SIM.readout_noise_adu ) # gaussian random number to add to every pixel (0 for PAD) # apply Poissonion correction, then scale to ADU, then adc_offset. # should be 10 for most Rayonix, Pilatus should be 0, CSPAD should be 0. print("detector_psf_type=", SIM.detector_psf_type) print("detector_psf_fwhm_mm=", SIM.detector_psf_fwhm_mm) print("detector_psf_kernel_radius_pixels=", SIM.detector_psf_kernel_radius_pixels) SIM.add_noise() #converts phtons to ADU. print("raw_pixels=", SIM.raw_pixels) SIM.to_smv_format(fileout=prefix + ".img", intfile_scale=1) # try to write as CBF if False: import dxtbx from dxtbx.format.FormatCBFMiniPilatus import FormatCBFMiniPilatus img = dxtbx.load(prefix + ".img") print(img) FormatCBFMiniPilatus.as_file(detector=img.get_detector(), beam=img.get_beam(), gonio=img.get_goniometer(), scan=img.get_scan(), data=img.get_raw_data(), path=prefix + ".cbf") SIM.free_all()
def tst_all(): F = foo() assert F == (1, 2, 3, 4) size = 1516 D = detector(slowpixels=1516, fastpixels=1516, pixel_size=0.00011) D.set_region_of_interest(0, int(0.6 * size), int(0.4 * size), int(0.6 * size)) D.set_oversampling(1) C = camera() C.distance = 0.18166 C.Ybeam = 0.08338 C.Zbeam = 0.08338 C.lambda0 = 6.2E-10 C.dispersion = 0.002 C.dispsteps = 4 C.hdivrange = 0 C.vdivrange = 0 C.hdivstep = 1 C.vdivstep = 1 C.source_distance = 10. C.fluence = 1.E24 Amat = sqr((127.6895065259495, 0.6512077339887, -0.4403031342553, -1.1449112128916, 225.3922539826207, 1.8393136632579, 1.0680694468752, -2.4923062985132, 306.0953037195841)) PSII = amplitudes_from_pdb(8., "fft", True) from cctbx import crystal_orientation X = crystal() X.orientation = crystal_orientation.crystal_orientation( Amat, crystal_orientation.basis_type.direct) X.miller = PSII.indices() X.amplitudes = PSII.data() X.Na = 6 X.Nb = 6 X.Nc = 6 SIM = fast_bragg_simulation() SIM.set_detector(D) SIM.set_camera(C) SIM.set_crystal(X) SIM.sweep_over_detector() data = D.raw scale_factor = 55000. / flex.max(data) #print "scale_factor",scale_factor fileout = "intimage_001.img" SIM.to_smv_format(fileout=fileout, intfile_scale=scale_factor, saturation=40000) import os assert os.path.isfile(fileout) os.remove(fileout) #simulation is complete, now we'll autoindex the image fragment and verify # that the indexed cell is similar to the input cell. if (not libtbx.env.has_module("annlib")): print "Skipping some tests: annlib not available." return # 1. Analysis of the image to identify the Bragg peak centers. # This step uses an inefficient algorithm and implementation and # is most time consuming; but the code is only for testing, not production from rstbx.diffraction.fastbragg.tst_utils_clustering import specific_libann_cluster M = specific_libann_cluster(scale_factor * data, intensity_cutoff=25, distance_cutoff=17) # M is a dictionary of peak intensities indexed by pixel coordinates # 2. Now autoindex the pattern from rstbx.diffraction.fastbragg.tst_utils_clustering import index_wrapper SIM.C = C SIM.D = D ai, ref_uc = index_wrapper(M.keys(), SIM, PSII) tst_uc = ai.getOrientation().unit_cell() #print ref_uc # (127.692, 225.403, 306.106, 90, 90, 90) #print tst_uc # (106.432, 223.983, 303.102, 90.3185, 91.5998, 90.5231) # 3. Final assertion. In the given orientation, # the unit cell A vector is into the beam and is not well sampled, # so tolerances have to be fairly relaxed, 5%. # Labelit does better with the target_cell restraint, but this improved # algorithm is not used here for the test assert ref_uc.is_similar_to(tst_uc, relative_length_tolerance=0.20, absolute_angle_tolerance=2.0)
def run(args): distance = 125 centre = (97.075, 97.075) pix_size = (0.11, 0.11) image_size = (1765, 1765) wavelength = args.w or 1.0 # 1. Make a dummy detector detector = detector_factory.simple( 'SENSOR_UNKNOWN', # Sensor distance, centre, '+x', '-y', # fast/slow direction pix_size, image_size) # 2. Get the miller array! mill_array = process_mtz(args.mtzfile[0]).as_intensity_array() ortho = sqr(mill_array.crystal_symmetry().unit_cell().reciprocal() \ .orthogonalization_matrix()) # 3.Create some image_pickle dictionairies that contain 'full' intensities, # but are otherwise complete. im = 0 while im < args.n: im += 1 A = sqr(flex.random_double_r3_rotation_matrix()) * ortho orientation = crystal_orientation(A, basis_type.reciprocal) pix_coords, miller_set = get_pix_coords(wavelength, A, mill_array, detector) if len(miller_set) > 10: # at least 10 reflections miller_set = cctbx.miller.set(mill_array.crystal_symmetry(), miller_set, anomalous_flag=False) obs = mill_array.common_set(miller_set) temp_dict = { 'observations': [obs], 'mapped_predictions': [pix_coords], 'pointgroup': None, 'current_orientation': [orientation], 'xbeam': centre[0], 'ybeam': centre[1], 'wavelength': wavelength } old_node = ImageNode(dicti=temp_dict, scale=False) # Remove all reflection that are not at least p partial partial_sel = (old_node.partialities > p_threshold) temp_dict['full_observations'] = [obs.select(partial_sel)] temp_dict['observations'] = [ obs.select(partial_sel) * old_node.partialities.select(partial_sel) ] temp_dict['mapped_predictions'] = \ [temp_dict['mapped_predictions'][0].select(partial_sel)] if logging.Logger.root.level <= logging.DEBUG: # debug! before = temp_dict['full_observations'][0] after = temp_dict['observations'][0] / old_node.partialities assert sum(abs(before.data() - after.data())) < eps if args.r: partials = list(temp_dict['observations'][0].data()) jiggled_partials = flex.double( [random.gauss(obs, args.r * obs) for obs in partials]) temp_dict['observations'][0] = temp_dict['observations'][0] \ .customized_copy(data=jiggled_partials) pkl_name = "simulated_data_{0:04d}.pickle".format(im) with (open(pkl_name, 'wb')) as pkl: cPickle.dump(temp_dict, pkl) ''' Only works with no noise:
def image_case_factory(item_no): mosflm = sqr( (1, 0, 0, 0, 1, 0, 0, 0, 1)) # mosflm lab vectors in their own frame labelit = sqr( (0, 0, -1, -1, 0, 0, 0, 1, 0)) # mosflm basis vectors in labelit frame MLx = (0, 0, -1) MLy = (-1, 0, 0) MLz = (0, 1, 0) # "virtual" rotation axis being used by cctbx.xfel for CXI LM = mosflm * labelit.inverse( ) # converts labelit frame coords to mosflm frame SWAPXY = sqr((0, 1, 0, 1, 0, 0, 0, 0, 1)) # in labelit frame SWAPZ = sqr((1, 0, 0, 0, 1, 0, 0, 0, -1)) # in labelit frame R90 = sqr((0, -1, 0, 1, 0, 0, 0, 0, 1)) # in labelit frame, rotation 90 on beam axis CONTAINER_SZ = 1000 WAVELENGTH = 1.32 # given by James Holton container_no = item // CONTAINER_SZ filename = "/net/viper/raid1/sauter/fake/result/basic00/" filename = os.path.join(filename, "r%02d" % container_no, TRIAL, "integration", "int-data_%05d.pickle" % item_no) trial_results = pickle.load(open(filename, "rb")) current_hexagonal_ori = trial_results["current_orientation"][0] current_cb_op_to_primitive = trial_results["current_cb_op_to_primitive"][0] current_triclinic_ori = current_hexagonal_ori.change_basis( current_cb_op_to_primitive) filename = "/net/viper/raid1/sauter/fake/holton/mosflm_matrix" filename = os.path.join(filename, "%02d" % container_no, "%05d.mat" % item_no) lines = open(filename).readlines() A0 = lines[0].strip().split() A1 = lines[1].strip().split() A2 = lines[2].strip().split() A = sqr( (float(A0[0]), float(A0[1]), float(A0[2]), float(A1[0]), float(A1[1]), float(A1[2]), float(A2[0]), float(A2[1]), float(A2[2]))) A = A / WAVELENGTH Holton_hexagonal_ori = crystal_orientation(SWAPZ * SWAPXY * R90 * LM * A, True) Holton_triclinic_ori = Holton_hexagonal_ori.change_basis( current_cb_op_to_primitive) c_inv_r_best = Holton_triclinic_ori.best_similarity_transformation( other=current_triclinic_ori, fractional_length_tolerance=50., unimodular_generator_range=1) c_inv_r_int = tuple([int(round(ij, 0)) for ij in c_inv_r_best]) from cctbx import sgtbx c_inv = sgtbx.rt_mx(sgtbx.rot_mx(c_inv_r_int)) cb_op = sgtbx.change_of_basis_op(c_inv) comparison_triclinic = Holton_triclinic_ori.change_basis(cb_op) comparison_hexagonal = comparison_triclinic.change_basis( current_cb_op_to_primitive.inverse()) print( sqr(current_hexagonal_ori.direct_matrix()) - sqr(comparison_hexagonal.direct_matrix())) print("item %d" % item_no) SC = ScoringContainer() SC.model = current_hexagonal_ori SC.reference = comparison_hexagonal return SC
def postrefine_by_frame(self, frame_no, pickle_filename, iparams, miller_array_ref, pres_in, avg_mode): # 1. Prepare data observations_pickle = pickle.load(open(pickle_filename, "rb")) crystal_init_orientation = observations_pickle["current_orientation"][0] wavelength = observations_pickle["wavelength"] pickle_filepaths = pickle_filename.split("/") img_filename_only = pickle_filepaths[len(pickle_filepaths) - 1] txt_exception = " {0:40} ==> ".format(img_filename_only) inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename ) if inputs is not None: observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm = inputs else: txt_exception += txt_organize_input + "\n" return None, txt_exception # 2. Determine polarity - always do this even if flag_polar = False # the function will take care of it. polar_hkl, cc_iso_raw_asu, cc_iso_raw_rev = self.determine_polar( observations_original, iparams, pickle_filename, pres=pres_in ) # 3. Select data for post-refinement (only select indices that are common with the reference set observations_non_polar = self.get_observations_non_polar(observations_original, polar_hkl) matches = miller.match_multi_indices( miller_indices_unique=miller_array_ref.indices(), miller_indices=observations_non_polar.indices() ) I_ref_match = flex.double([miller_array_ref.data()[pair[0]] for pair in matches.pairs()]) miller_indices_ref_match = flex.miller_index((miller_array_ref.indices()[pair[0]] for pair in matches.pairs())) I_obs_match = flex.double([observations_non_polar.data()[pair[1]] for pair in matches.pairs()]) sigI_obs_match = flex.double([observations_non_polar.sigmas()[pair[1]] for pair in matches.pairs()]) miller_indices_original_obs_match = flex.miller_index( (observations_original.indices()[pair[1]] for pair in matches.pairs()) ) miller_indices_non_polar_obs_match = flex.miller_index( (observations_non_polar.indices()[pair[1]] for pair in matches.pairs()) ) alpha_angle_set = flex.double([alpha_angle[pair[1]] for pair in matches.pairs()]) spot_pred_x_mm_set = flex.double([spot_pred_x_mm[pair[1]] for pair in matches.pairs()]) spot_pred_y_mm_set = flex.double([spot_pred_y_mm[pair[1]] for pair in matches.pairs()]) references_sel = miller_array_ref.customized_copy(data=I_ref_match, indices=miller_indices_ref_match) observations_original_sel = observations_original.customized_copy( data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_original_obs_match ) observations_non_polar_sel = observations_non_polar.customized_copy( data=I_obs_match, sigmas=sigI_obs_match, indices=miller_indices_non_polar_obs_match ) # 4. Do least-squares refinement lsqrh = leastsqr_handler() try: refined_params, stats, n_refl_postrefined = lsqrh.optimize( I_ref_match, observations_original_sel, wavelength, crystal_init_orientation, alpha_angle_set, spot_pred_x_mm_set, spot_pred_y_mm_set, iparams, pres_in, observations_non_polar_sel, detector_distance_mm, ) except Exception: txt_exception += "optimization failed.\n" return None, txt_exception # caculate partiality for output (with target_anomalous check) G_fin, B_fin, rotx_fin, roty_fin, ry_fin, rz_fin, r0_fin, re_fin, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin = ( refined_params ) inputs, txt_organize_input = self.organize_input( observations_pickle, iparams, avg_mode, pickle_filename=pickle_filename ) observations_original, alpha_angle, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm = inputs observations_non_polar = self.get_observations_non_polar(observations_original, polar_hkl) from cctbx.uctbx import unit_cell uc_fin = unit_cell((a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin)) if pres_in is not None: crystal_init_orientation = pres_in.crystal_orientation two_theta = observations_original.two_theta(wavelength=wavelength).data() from mod_leastsqr import calc_partiality_anisotropy_set partiality_fin, dummy, rs_fin, rh_fin = calc_partiality_anisotropy_set( uc_fin, rotx_fin, roty_fin, observations_original.indices(), ry_fin, rz_fin, r0_fin, re_fin, two_theta, alpha_angle, wavelength, crystal_init_orientation, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, iparams.partiality_model, iparams.flag_beam_divergence, ) # calculate the new crystal orientation O = sqr(uc_fin.orthogonalization_matrix()).transpose() R = sqr(crystal_init_orientation.crystal_rotation_matrix()).transpose() from cctbx.crystal_orientation import crystal_orientation, basis_type CO = crystal_orientation(O * R, basis_type.direct) crystal_fin_orientation = CO.rotate_thru((1, 0, 0), rotx_fin).rotate_thru((0, 1, 0), roty_fin) # remove reflections with partiality below threshold i_sel = partiality_fin > iparams.merge.partiality_min partiality_fin_sel = partiality_fin.select(i_sel) rs_fin_sel = rs_fin.select(i_sel) rh_fin_sel = rh_fin.select(i_sel) observations_non_polar_sel = observations_non_polar.customized_copy( indices=observations_non_polar.indices().select(i_sel), data=observations_non_polar.data().select(i_sel), sigmas=observations_non_polar.sigmas().select(i_sel), ) observations_original_sel = observations_original.customized_copy( indices=observations_original.indices().select(i_sel), data=observations_original.data().select(i_sel), sigmas=observations_original.sigmas().select(i_sel), ) pres = postref_results() pres.set_params( observations=observations_non_polar_sel, observations_original=observations_original_sel, refined_params=refined_params, stats=stats, partiality=partiality_fin_sel, rs_set=rs_fin_sel, rh_set=rh_fin_sel, frame_no=frame_no, pickle_filename=pickle_filename, wavelength=wavelength, crystal_orientation=crystal_fin_orientation, detector_distance_mm=detector_distance_mm, ) r_change, r_xy_change, cc_change, cc_iso_change = (0, 0, 0, 0) try: r_change = ((pres.R_final - pres.R_init) / pres.R_init) * 100 r_xy_change = ((pres.R_xy_final - pres.R_xy_init) / pres.R_xy_init) * 100 cc_change = ((pres.CC_final - pres.CC_init) / pres.CC_init) * 100 cc_iso_change = ((pres.CC_iso_final - pres.CC_iso_init) / pres.CC_iso_init) * 100 except Exception: pass txt_postref = " {0:40} ==> RES:{1:5.2f} NREFL:{2:5d} R:{3:8.2f}% RXY:{4:8.2f}% CC:{5:6.2f}% CCISO:{6:6.2f}% G:{7:10.3e} B:{8:7.1f} CELL:{9:6.2f} {10:6.2f} {11:6.2f} {12:6.2f} {13:6.2f} {14:6.2f}".format( img_filename_only + " (" + polar_hkl + ")", observations_original_sel.d_min(), len(observations_original_sel.data()), r_change, r_xy_change, cc_change, cc_iso_change, pres.G, pres.B, a_fin, b_fin, c_fin, alpha_fin, beta_fin, gamma_fin, ) print txt_postref txt_postref += "\n" return pres, txt_postref
def populate_orientation(self): assert self.xtal.get_A_at_scan_point(self.scan_no) is not None, "no crystal orientation matrix" self.frame['current_orientation'] = [crystal_orientation(self.xtal.get_A_at_scan_point(self.scan_no).elems, True)]
def get_unit_cell_at_scan_point(self, t): B = self.get_B_at_scan_point(t) co = crystal_orientation(B) return co.unit_cell()
GF = gen_fmodel(resolution=3.0,pdb_text=pdb_lines,algorithm="fft",wavelength=1.7) CB_OP_C_P = GF.xray_structure.change_of_basis_op_to_primitive_setting() # from C to P print(str(CB_OP_C_P)) icount=0 from scitbx.array_family import flex angles=flex.double() for stuff in get_items(): #print stuff icount+=1 print("Iteration",icount) from cctbx import crystal_orientation # work up the ground truth from header header_Ori = crystal_orientation.crystal_orientation(stuff["coarse"], crystal_orientation.basis_type.direct) header_Ori.show(legend="header_Ori") C2_ground_truth = header_Ori.change_basis(CB_OP_C_P.inverse()) C2_ground_truth.show(legend="coarse_ground_truth") # work up the ground truth from mosaic ensemble mosaic_Ori = crystal_orientation.crystal_orientation(stuff["fine"], crystal_orientation.basis_type.direct) mosaic_Ori.show(legend="mosaic_Ori") fine_ground_truth = mosaic_Ori.change_basis(CB_OP_C_P.inverse()) fine_ground_truth.show(legend="fine_ground_truth") # now calculate the angle as mean a_to_a,b_to_b,c_to_c aoff = C2_ground_truth.a.angle(fine_ground_truth.a,deg=True) boff = C2_ground_truth.b.angle(fine_ground_truth.b,deg=True)
def populate_orientation(self): assert self.xtal.get_A() is not None, "no crystal orientation matrix" self.frame['current_orientation'] = [crystal_orientation(self.xtal.get_A().elems, True)]
def get_unit_cell_at_scan_point(self, t): B = self.get_B_at_scan_point(t) co = crystal_orientation(B) return co.unit_cell()
def exercise_basic(): identity = (1,0,0,0,1,0,0,0,1) I = crystal_orientation(identity,False) #direct space orthorhombic = (1,0,0,0,0.5,0.,0.,0.,0.25) R = crystal_orientation(orthorhombic,True) #reciprocal space assert R.direct_matrix() == (1.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 4.0) assert R.reciprocal_matrix() == orthorhombic inverse = (-1,0,0,0,-1,0,0,0,-1) negative = crystal_orientation(inverse,False) assert I == negative.make_positive() assert R.unit_cell().parameters() == (1.0,2.0,4.0,90.,90.,90.) assert approx_equal( R.unit_cell_inverse().parameters(), (1.0,0.5,0.25,90.,90.,90.) ) O1 = crystal_orientation((-0.015553395334476732, -0.0028287158782335244, 0.018868416534039902, -0.0016512962184570643, -0.020998220575299865, 0.0012056332661160732, 0.015789188025134133, -0.011166135863736777, 0.013045365404272641),True) def exercise_change_basis(): assert approx_equal(O1.unit_cell().parameters(), (47.659, 47.6861, 49.6444, 62.9615, 73.8222, 73.5269),1E-3) reindex = (0.0, -1.0, 0.0, -1.0, 0.0, 0.0, 0.0, 0.0, -1.0) # swap a & b and take inverse O2 = O1.change_basis(reindex) assert approx_equal(O2.unit_cell().parameters(), (47.6861, 47.659, 49.6444, 73.8222, 62.9615, 73.5269),1E-3) rhombohedral_test = crystal_orientation(( 0.002737747939994224, -0.0049133768326561278, 0.0023634556852316566, 0.0062204242383498082, 0.006107332242442573, 0.0047036576949967112, -0.0057640198753891566, -0.0025891042237382953, 0.0023674924674260264),basis_type.reciprocal) rhombohedral_reference = crystal_orientation(( -0.0076361080646872997, 0.0049061665572297979, 0.0023688116121433865,
def tst_all(): F = foo() assert F == (1,2,3,4) size=1516 D = detector(slowpixels=1516,fastpixels=1516,pixel_size=0.00011) D.set_region_of_interest(0,int(0.6*size),int(0.4*size),int(0.6*size)) D.set_oversampling(1) C = camera() C.distance = 0.18166 C.Ybeam = 0.08338 C.Zbeam = 0.08338 C.lambda0 = 6.2E-10 C.dispersion = 0.002 C.dispsteps = 4 C.hdivrange = 0 C.vdivrange = 0 C.hdivstep = 1 C.vdivstep = 1 C.source_distance = 10. C.fluence = 1.E24 Amat = sqr((127.6895065259495, 0.6512077339887,-0.4403031342553, -1.1449112128916,225.3922539826207,1.8393136632579, 1.0680694468752,-2.4923062985132,306.0953037195841)) PSII = amplitudes_from_pdb(8.,"fft",True) from cctbx import crystal_orientation X = crystal() X.orientation = crystal_orientation.crystal_orientation( Amat,crystal_orientation.basis_type.direct) X.miller = PSII.indices() X.amplitudes = PSII.data() X.Na = 6; X.Nb = 6; X.Nc = 6 SIM = fast_bragg_simulation() SIM.set_detector(D) SIM.set_camera(C) SIM.set_crystal(X) SIM.sweep_over_detector() data = D.raw scale_factor = 55000./flex.max(data) #print "scale_factor",scale_factor fileout="intimage_001.img" SIM.to_smv_format(fileout=fileout, intfile_scale = scale_factor, saturation=40000) import os assert os.path.isfile(fileout) os.remove(fileout) #simulation is complete, now we'll autoindex the image fragment and verify # that the indexed cell is similar to the input cell. if (not libtbx.env.has_module("annlib")): print "Skipping some tests: annlib not available." return # 1. Analysis of the image to identify the Bragg peak centers. # This step uses an inefficient algorithm and implementation and # is most time consuming; but the code is only for testing, not production from rstbx.diffraction.fastbragg.tst_utils_clustering import specific_libann_cluster M=specific_libann_cluster(scale_factor*data,intensity_cutoff = 25,distance_cutoff=17) # M is a dictionary of peak intensities indexed by pixel coordinates # 2. Now autoindex the pattern from rstbx.diffraction.fastbragg.tst_utils_clustering import index_wrapper SIM.C = C SIM.D = D ai,ref_uc = index_wrapper(M.keys(), SIM, PSII) tst_uc = ai.getOrientation().unit_cell() #print ref_uc # (127.692, 225.403, 306.106, 90, 90, 90) #print tst_uc # (106.432, 223.983, 303.102, 90.3185, 91.5998, 90.5231) # 3. Final assertion. In the given orientation, # the unit cell A vector is into the beam and is not well sampled, # so tolerances have to be fairly relaxed, 5%. # Labelit does better with the target_cell restraint, but this improved # algorithm is not used here for the test assert ref_uc.is_similar_to(tst_uc, relative_length_tolerance=0.20, absolute_angle_tolerance= 2.0)
def exercise_change_basis(): assert approx_equal(O1.unit_cell().parameters(), (47.659, 47.6861, 49.6444, 62.9615, 73.8222, 73.5269),1E-3) reindex = (0.0, -1.0, 0.0, -1.0, 0.0, 0.0, 0.0, 0.0, -1.0) # swap a & b and take inverse O2 = O1.change_basis(reindex) assert approx_equal(O2.unit_cell().parameters(), (47.6861, 47.659, 49.6444, 73.8222, 62.9615, 73.5269),1E-3) rhombohedral_test = crystal_orientation(( 0.002737747939994224, -0.0049133768326561278, 0.0023634556852316566, 0.0062204242383498082, 0.006107332242442573, 0.0047036576949967112, -0.0057640198753891566, -0.0025891042237382953, 0.0023674924674260264),basis_type.reciprocal) rhombohedral_reference = crystal_orientation(( -0.0076361080646872997, 0.0049061665572297979, 0.0023688116121433865, -0.00011109895272056645, -0.0061110173438898583, 0.0047062769738302939, 0.0031790372319626674, 0.0025876279220667518, 0.0023669727051432361),basis_type.reciprocal) # Find a similarity transform that maps the two cells onto each other c_inv_r_best = rhombohedral_test.best_similarity_transformation( other = rhombohedral_reference, fractional_length_tolerance = 1.00, unimodular_generator_range=1) c_inv_r_int = tuple([int(round(ij,0)) for ij in c_inv_r_best]) assert c_inv_r_int == (-1, 0, 0, 1, -1, 0, 0, 0, 1) c_inv = sgtbx.rt_mx(sgtbx.rot_mx(c_inv_r_int)) cb_op = sgtbx.change_of_basis_op(c_inv) rhombohedral_reindex = rhombohedral_test.change_basis(cb_op) assert rhombohedral_reindex.difference_Z_score(rhombohedral_reference) < 0.40 assert rhombohedral_reindex.direct_mean_square_difference(rhombohedral_reference) < 0.1 #an alternative test from ana that should fail (gives high msd~0.22; cell axes don't match): ana_reference = crystal_orientation(( 0.0023650364919947241, 0.012819317075171401, 0.003042762222847376, 0.0081242553464681254, 0.0050052660206998077, -0.01472465697193685, -0.01373896574061278, 0.0083781530252581681, -0.0035301340829149005),basis_type.reciprocal) ana_current = crystal_orientation(( -0.014470153848927263, 0.0095185368147633793, 0.00087746490483763798, -0.0049989006573928716, -0.0079714727432991222, 0.014778692772530192, 0.0051268914129933571, 0.010264066188909109, 0.0044244589492769002),basis_type.reciprocal) c_inv_r_best = ana_current.best_similarity_transformation( other = ana_reference, fractional_length_tolerance = 200.0, unimodular_generator_range=1) c_inv_r_int = tuple([int(round(ij,0)) for ij in c_inv_r_best]) c_inv = sgtbx.rt_mx(sgtbx.rot_mx(c_inv_r_int)) cb_op = sgtbx.change_of_basis_op(c_inv) ana_reindex = ana_reference.change_basis(cb_op.inverse()) assert 200.0 > ana_reindex.difference_Z_score(ana_current) > 20. u = uctbx.unit_cell((10., 10., 10., 90., 90., 90.)) CO = crystal_orientation(u.fractionalization_matrix(),True) assert approx_equal( CO.unit_cell().parameters(), CO.change_basis((1,0,0, 0,1,0, 0,0,1)).unit_cell().parameters()) u = uctbx.unit_cell((2,3,5)) CO = crystal_orientation(u.fractionalization_matrix(),True) assert approx_equal( CO.change_basis((0,1,0, 0,0,1, 1,0,0)).unit_cell().parameters(), (5,2,3,90,90,90)) cb_op = sgtbx.change_of_basis_op("y,z,x") assert approx_equal( CO.change_basis(cb_op).unit_cell().parameters(), (5,2,3,90,90,90)) import scitbx.math from scitbx import matrix fmx = matrix.sqr( uctbx.unit_cell((10, 13, 17, 85, 95, 105)).fractionalization_matrix()) crm = matrix.sqr(scitbx.math.r3_rotation_axis_and_angle_as_matrix( axis=(-3,5,-7), angle=37, deg=True)) co = crystal_orientation(crm * fmx.transpose(), True) assert approx_equal(co.crystal_rotation_matrix(), crm)
def run_sim2smv(img_prefix=None, simparams=None, pdb_lines=None, crystal=None, spectra=None, rotation=None, rank=None, fsave=None, sfall_cluster=None, quick=False): smv_fileout = fsave direct_algo_res_limit = simparams.direct_algo_res_limit wavlen, flux, real_wavelength_A = next( spectra) # list of lambdas, list of fluxes, average wavelength real_flux = flex.sum(flux) assert real_wavelength_A > 0 # print(rank, " ## real_wavelength_A/real_flux = ", real_wavelength_A, real_flux*1.0/simparams.flux) if quick: wavlen = flex.double([real_wavelength_A]) flux = flex.double([real_flux]) # GF = gen_fmodel(resolution=simparams.direct_algo_res_limit,pdb_text=pdb_lines,algorithm=simparams.fmodel_algorithm,wavelength=real_wavelength_A) # GF.set_k_sol(simparams.k_sol) # GF.make_P1_primitive() sfall_main = sfall_cluster["main"] #GF.get_amplitudes() # use crystal structure to initialize Fhkl array # sfall_main.show_summary(prefix = "Amplitudes used ") N = crystal.number_of_cells(sfall_main.unit_cell()) #print("## number of N = ", N) SIM = nanoBragg(detpixels_slowfast=(simparams.detector_size_ny,simparams.detector_size_nx),pixel_size_mm=simparams.pixel_size_mm,\ Ncells_abc=(N,N,N),wavelength_A=real_wavelength_A,verbose=0) # workaround for problem with wavelength array, specify it separately in constructor. # SIM.adc_offset_adu = 0 # Do not offset by 40 SIM.adc_offset_adu = 10 # Do not offset by 40 SIM.seed = 0 # SIM.randomize_orientation() SIM.mosaic_spread_deg = simparams.mosaic_spread_deg # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = simparams.mosaic_domains # 77 seconds. SIM.distance_mm = simparams.distance_mm ## setup the mosaicity UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) SIM.set_mosaic_blocks(UMAT_nm) ###################### SIM.beamcenter_convention = convention.ADXV SIM.beam_center_mm = (simparams.beam_center_x_mm, simparams.beam_center_y_mm) # 95.975 96.855 ###################### # get same noise each time this test is run SIM.seed = 0 SIM.oversample = simparams.oversample SIM.wavelength_A = real_wavelength_A SIM.polarization = simparams.polarization # this will become F000, marking the beam center SIM.default_F = simparams.default_F #SIM.missets_deg= (10,20,30) SIM.Fhkl = sfall_main Amatrix_rot = ( rotation * sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose() SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter # print("## inside run_sim2smv, Amat_rot = ", Amatrix_rot) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM Ori = crystal_orientation.crystal_orientation( Amat, crystal_orientation.basis_type.reciprocal) print(fsave, "Amatrix_rot", Amatrix_rot) print(fsave, "Amat", Amat) print(fsave, "Ori", Ori) print(fsave, "rotation", rotation) print(fsave, "sqr(sfall_main.unit_cell().orthogonalization_matrix())", sqr(sfall_main.unit_cell().orthogonalization_matrix())) SIM.free_all()
def run_sim2smv(ROI, prefix, crystal, spectra, rotation, rank, tophat_spectrum=True, quick=False): smv_fileout = prefix + ".img" direct_algo_res_limit = 1.7 wavlen, flux, wavelength_A = next( spectra) # list of lambdas, list of fluxes, average wavelength if tophat_spectrum: sum_flux = flex.sum(flux) #from IPython import embed; embed() ave_flux = sum_flux / 60. # 60 energy channels for ix in range(len(wavlen)): energy = 12398.425 / wavlen[ix] if energy >= 7090 and energy <= 7150: flux[ix] = ave_flux else: flux[ix] = 0. if quick: wavlen = flex.double([wavelength_A]) flux = flex.double([flex.sum(flux)]) print("Quick sim, lambda=%f, flux=%f" % (wavelength_A, flux[0])) #from matplotlib import pyplot as plt #plt.plot(flux,"r-") #plt.title(smv_fileout) #plt.show() GF = gen_fmodel(resolution=direct_algo_res_limit, pdb_text=pdb_lines, algorithm="fft", wavelength=wavelength_A) GF.set_k_sol(0.435) GF.make_P1_primitive() sfall_main = GF.get_amplitudes() # use crystal structure to initialize Fhkl array sfall_main.show_summary(prefix="Amplitudes used ") N = crystal.number_of_cells(sfall_main.unit_cell()) #SIM = nanoBragg(detpixels_slowfast=(2000,2000),pixel_size_mm=0.11,Ncells_abc=(5,5,5),verbose=0) SIM = nanoBragg( detpixels_slowfast=(3000, 3000), pixel_size_mm=0.11, Ncells_abc=(N, N, N), # workaround for problem with wavelength array, specify it separately in constructor. wavelength_A=wavelength_A, verbose=0) SIM.adc_offset_adu = 0 # Do not offset by 40 SIM.adc_offset_adu = 10 # Do not offset by 40 import sys if len(sys.argv) > 2: SIM.seed = -int(sys.argv[2]) print("GOTHERE seed=", SIM.seed) if len(sys.argv) > 1: if sys.argv[1] == "random": SIM.randomize_orientation() SIM.mosaic_domains = 50 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image # 3000000 images would be 100000 hours on a 60-core machine (dials), or 11.4 years # using 2 nodes, 5.7 years. Do this at SLAC? NERSC? combination of all? # SLAC downtimes: Tues Dec 5 (24 hrs), Mon Dec 11 (72 hrs), Mon Dec 18 light use, 24 days SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev SIM.distance_mm = 141.7 UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) SIM.set_mosaic_blocks(UMAT_nm) #SIM.detector_thick_mm = 0.5 # = 0 for Rayonix #SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD #SIM.detector_attenuation_length_mm = default is silicon # get same noise each time this test is run SIM.seed = 1 SIM.oversample = 1 SIM.wavelength_A = wavelength_A SIM.polarization = 1 # this will become F000, marking the beam center SIM.default_F = 0 #SIM.missets_deg= (10,20,30) print("mosaic_seed=", SIM.mosaic_seed) print("seed=", SIM.seed) print("calib_seed=", SIM.calib_seed) print("missets_deg =", SIM.missets_deg) SIM.Fhkl = sfall_main print("Determinant", rotation.determinant()) Amatrix_rot = ( rotation * sqr(sfall_main.unit_cell().orthogonalization_matrix())).transpose() print("RAND_ORI", prefix, end=' ') for i in Amatrix_rot: print(i, end=' ') print() SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter print("unit_cell_Adeg=", SIM.unit_cell_Adeg) print("unit_cell_tuple=", SIM.unit_cell_tuple) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM from cctbx import crystal_orientation Ori = crystal_orientation.crystal_orientation( Amat, crystal_orientation.basis_type.reciprocal) print("Python unit cell from SIM state", Ori.unit_cell()) # fastest option, least realistic #SIM.xtal_shape=shapetype.Tophat # RLP = hard sphere #SIM.xtal_shape=shapetype.Square # gives fringes SIM.xtal_shape = shapetype.Gauss # both crystal & RLP are Gaussian #SIM.xtal_shape=shapetype.Round # Crystal is a hard sphere # only really useful for long runs SIM.progress_meter = False # prints out value of one pixel only. will not render full image! #SIM.printout_pixel_fastslow=(500,500) #SIM.printout=True SIM.show_params() # flux is always in photons/s SIM.flux = 1e12 SIM.exposure_s = 1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense temp = SIM.Ncells_abc print("Ncells_abc=", SIM.Ncells_abc) SIM.Ncells_abc = temp print("Ncells_abc=", SIM.Ncells_abc) print("xtal_size_mm=", SIM.xtal_size_mm) print("unit_cell_Adeg=", SIM.unit_cell_Adeg) print("unit_cell_tuple=", SIM.unit_cell_tuple) print("missets_deg=", SIM.missets_deg) print("Amatrix=", SIM.Amatrix) print("beam_center_mm=", SIM.beam_center_mm) print("XDS_ORGXY=", SIM.XDS_ORGXY) print("detector_pivot=", SIM.detector_pivot) print("xtal_shape=", SIM.xtal_shape) print("beamcenter_convention=", SIM.beamcenter_convention) print("fdet_vector=", SIM.fdet_vector) print("sdet_vector=", SIM.sdet_vector) print("odet_vector=", SIM.odet_vector) print("beam_vector=", SIM.beam_vector) print("polar_vector=", SIM.polar_vector) print("spindle_axis=", SIM.spindle_axis) print("twotheta_axis=", SIM.twotheta_axis) print("distance_meters=", SIM.distance_meters) print("distance_mm=", SIM.distance_mm) print("close_distance_mm=", SIM.close_distance_mm) print("detector_twotheta_deg=", SIM.detector_twotheta_deg) print("detsize_fastslow_mm=", SIM.detsize_fastslow_mm) print("detpixels_fastslow=", SIM.detpixels_fastslow) print("detector_rot_deg=", SIM.detector_rot_deg) print("curved_detector=", SIM.curved_detector) print("pixel_size_mm=", SIM.pixel_size_mm) print("point_pixel=", SIM.point_pixel) print("polarization=", SIM.polarization) print("nopolar=", SIM.nopolar) print("oversample=", SIM.oversample) print("region_of_interest=", SIM.region_of_interest) print("wavelength_A=", SIM.wavelength_A) print("energy_eV=", SIM.energy_eV) print("fluence=", SIM.fluence) print("flux=", SIM.flux) print("exposure_s=", SIM.exposure_s) print("beamsize_mm=", SIM.beamsize_mm) print("dispersion_pct=", SIM.dispersion_pct) print("dispsteps=", SIM.dispsteps) print("divergence_hv_mrad=", SIM.divergence_hv_mrad) print("divsteps_hv=", SIM.divsteps_hv) print("divstep_hv_mrad=", SIM.divstep_hv_mrad) print("round_div=", SIM.round_div) print("phi_deg=", SIM.phi_deg) print("osc_deg=", SIM.osc_deg) print("phisteps=", SIM.phisteps) print("phistep_deg=", SIM.phistep_deg) print("detector_thick_mm=", SIM.detector_thick_mm) print("detector_thicksteps=", SIM.detector_thicksteps) print("detector_thickstep_mm=", SIM.detector_thickstep_mm) print("***mosaic_spread_deg=", SIM.mosaic_spread_deg) print("***mosaic_domains=", SIM.mosaic_domains) print("indices=", SIM.indices) print("amplitudes=", SIM.amplitudes) print("Fhkl_tuple=", SIM.Fhkl_tuple) print("default_F=", SIM.default_F) print("interpolate=", SIM.interpolate) print("integral_form=", SIM.integral_form) from libtbx.development.timers import Profiler P = Profiler("nanoBragg") # now actually burn up some CPU #SIM.add_nanoBragg_spots() del P # simulated crystal is only 125 unit cells (25 nm wide) # amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume) print(crystal.domains_per_crystal) SIM.raw_pixels *= crystal.domains_per_crystal # must calculate the correct scale! output = StringIO() # open("myfile","w") make_response_plot = response_plot(False, title=prefix) for x in range(0, 100, 2): #len(flux)): if flux[x] == 0.0: continue print("+++++++++++++++++++++++++++++++++++++++ Wavelength", x) CH = channel_pixels(ROI, wavlen[x], flux[x], N, UMAT_nm, Amatrix_rot, GF, output) incremental_signal = CH.raw_pixels * crystal.domains_per_crystal if x in [26, 40, 54, 68]: # subsample 7096, 7110, 7124, 7138 eV print("----------------------> subsample", x) make_response_plot.incr_subsample(x, ROI, incremental_signal) make_response_plot.increment(x, ROI, incremental_signal) SIM.raw_pixels += incremental_signal CH.free_all() message = output.getvalue().split() miller = (int(message[4]), int(message[5]), int(message[6])) intensity = float(message[9]) #SIM.to_smv_format(fileout=prefix + "_intimage_001.img") pixels = SIM.raw_pixels roi_pixels = pixels[ROI[1][0]:ROI[1][1], ROI[0][0]:ROI[0][1]] print("Reducing full shape of", pixels.focus(), "to ROI of", roi_pixels.focus()) SIM.free_all() make_response_plot.plot(roi_pixels, miller) return dict(roi_pixels=roi_pixels, miller=miller, intensity=intensity)
def __init__(self, params): import cPickle as pickle from dxtbx.model import BeamFactory from dxtbx.model import DetectorFactory from dxtbx.model.crystal import CrystalFactory from cctbx.crystal_orientation import crystal_orientation, basis_type from dxtbx.model import Experiment, ExperimentList from scitbx import matrix self.experiments = ExperimentList() self.unique_file_names = [] self.params = params data = pickle.load( open(self.params.output.prefix + "_frame.pickle", "rb")) frames_text = data.split("\n") for item in frames_text: tokens = item.split(' ') wavelength = float(tokens[order_dict["wavelength"]]) beam = BeamFactory.simple(wavelength=wavelength) detector = DetectorFactory.simple( sensor=DetectorFactory.sensor( "PAD"), # XXX shouldn't hard code for XFEL distance=float(tokens[order_dict["distance"]]), beam_centre=[ float(tokens[order_dict["beam_x"]]), float(tokens[order_dict["beam_y"]]) ], fast_direction="+x", slow_direction="+y", pixel_size=[self.params.pixel_size, self.params.pixel_size], image_size=[1795, 1795], # XXX obviously need to figure this out ) reciprocal_matrix = matrix.sqr([ float(tokens[order_dict[k]]) for k in [ 'res_ori_1', 'res_ori_2', 'res_ori_3', 'res_ori_4', 'res_ori_5', 'res_ori_6', 'res_ori_7', 'res_ori_8', 'res_ori_9' ] ]) ORI = crystal_orientation(reciprocal_matrix, basis_type.reciprocal) direct = matrix.sqr(ORI.direct_matrix()) transfer_dict = dict( __id__="crystal", ML_half_mosaicity_deg=float( tokens[order_dict["half_mosaicity_deg"]]), ML_domain_size_ang=float( tokens[order_dict["domain_size_ang"]]), real_space_a=matrix.row(direct[0:3]), real_space_b=matrix.row(direct[3:6]), real_space_c=matrix.row(direct[6:9]), space_group_hall_symbol=self.params.target_space_group.type(). hall_symbol(), ) crystal = CrystalFactory.from_dict(transfer_dict) """ old code reflects python-based crystal model crystal = Crystal( real_space_a = matrix.row(direct[0:3]), real_space_b = matrix.row(direct[3:6]), real_space_c = matrix.row(direct[6:9]), space_group_symbol = self.params.target_space_group.type().lookup_symbol(), mosaicity = float(tokens[order_dict["half_mosaicity_deg"]]), ) crystal.domain_size = float(tokens[order_dict["domain_size_ang"]]) """ #if isoform is not None: # newB = matrix.sqr(isoform.fractionalization_matrix()).transpose() # crystal.set_B(newB) self.experiments.append( Experiment( beam=beam, detector=None, #dummy for now crystal=crystal)) self.unique_file_names.append( tokens[order_dict["unique_file_name"]]) self.show_summary()
rot_mat = rotation_vector.axis_and_angle_as_r3_rotation_matrix( omegas[omegaidx] + dangles[omegaidx][n]*0.1) HP = (rot_mat * matrix.sqr(Amat))*matrix.col(hkl+difference) + beam_vector #print "%10.7f %10.7f %10.7f"%HP.elems assert approx_equal(H1,HP,eps=1E-5) #print if __name__ == '__main__': wavelength = 1.2 resolution = 3.0 Amat = (0.0038039968697463817, 0.004498689311366309, 0.0044043429203887785, -0.00477859183801569, 0.006594300357213904, -0.002402759536958918, -0.01012056453488894, -0.0014226943325514182, 0.002789954423701981) orient = crystal_orientation(Amat,True) calc = partial_spot_position_partial_H( limiting_resolution = resolution, orientation = orient.reciprocal_matrix(), wavelength = wavelength, axial_direction = (0.,1.,0.) ) rotation_scattering(calc,matrix.col((0.,1.,0.)),Amat,wavelength) test_finite_differences(calc,matrix.col((0.,1.,0.)),Amat,wavelength) print "OK"
def refined_settings_factory_from_refined_triclinic( params, experiments, reflections, i_setting=None, lepage_max_delta=5.0, nproc=1, refiner_verbosity=0): assert len(experiments.crystals()) == 1 crystal = experiments.crystals()[0] used_reflections = copy.deepcopy(reflections) UC = crystal.get_unit_cell() from rstbx.dps_core.lepage import iotbx_converter Lfat = refined_settings_list() for item in iotbx_converter(UC, lepage_max_delta): Lfat.append(bravais_setting(item)) supergroup = Lfat.supergroup() triclinic = Lfat.triclinic() triclinic_miller = used_reflections['miller_index'] # assert no transformation between indexing and bravais list assert str(triclinic['cb_op_inp_best'])=="a,b,c" Nset = len(Lfat) for j in xrange(Nset): Lfat[j].setting_number = Nset-j from cctbx.crystal_orientation import crystal_orientation from cctbx import sgtbx from scitbx import matrix for j in xrange(Nset): cb_op = Lfat[j]['cb_op_inp_best'].c().as_double_array()[0:9] orient = crystal_orientation(crystal.get_A(),True) orient_best = orient.change_basis(matrix.sqr(cb_op).transpose()) constrain_orient = orient_best.constrain(Lfat[j]['system']) bravais = Lfat[j]["bravais"] cb_op_best_ref = Lfat[j]['best_subsym'].change_of_basis_op_to_reference_setting() space_group = sgtbx.space_group_info( number=bravais_lattice_to_lowest_symmetry_spacegroup_number[bravais]).group() space_group = space_group.change_basis(cb_op_best_ref.inverse()) bravais = str(bravais_types.bravais_lattice(group=space_group)) Lfat[j]["bravais"] = bravais Lfat[j].unrefined_crystal = dials_crystal_from_orientation( constrain_orient, space_group) args = [] for subgroup in Lfat: args.append(( params, subgroup, used_reflections, experiments, refiner_verbosity)) results = easy_mp.parallel_map( func=refine_subgroup, iterable=args, processes=nproc, method="multiprocessing", preserve_order=True, asynchronous=True, preserve_exception_message=True) for i, result in enumerate(results): Lfat[i] = result return Lfat
def run(self, experiments, reflections): self.logger.log_step_time("POSTREFINEMENT") if (not self.params.postrefinement.enable) or (self.params.scaling.algorithm != "mark0"): # mark1 implies no scaling/post-refinement self.logger.log("No post-refinement was done") if self.mpi_helper.rank == 0: self.logger.main_log("No post-refinement was done") return experiments, reflections target_symm = symmetry(unit_cell = self.params.scaling.unit_cell, space_group_info = self.params.scaling.space_group) i_model = self.params.scaling.i_model miller_set = self.params.scaling.miller_set # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. assert len(i_model.indices()) == len(miller_set.indices()) assert (i_model.indices() == miller_set.indices()).count(False) == 0 new_experiments = ExperimentList() new_reflections = flex.reflection_table() experiments_rejected_by_reason = {} # reason:how_many_rejected for experiment in experiments: exp_reflections = reflections.select(reflections['exp_id'] == experiment.identifier) # Build a miller array for the experiment reflections with original miller indexes exp_miller_indices_original = miller.set(target_symm, exp_reflections['miller_index'], not self.params.merging.merge_anomalous) observations_original_index = miller.array(exp_miller_indices_original, exp_reflections['intensity.sum.value'], flex.double(flex.sqrt(exp_reflections['intensity.sum.variance']))) assert exp_reflections.size() == exp_miller_indices_original.size() assert observations_original_index.size() == exp_miller_indices_original.size() # Build a miller array for the experiment reflections with asu miller indexes exp_miller_indices_asu = miller.set(target_symm, exp_reflections['miller_index_asymmetric'], True) observations = miller.array(exp_miller_indices_asu, exp_reflections['intensity.sum.value'], flex.double(flex.sqrt(exp_reflections['intensity.sum.variance']))) matches = miller.match_multi_indices(miller_indices_unique = miller_set.indices(), miller_indices = observations.indices()) pair1 = flex.int([pair[1] for pair in matches.pairs()]) # refers to the observations pair0 = flex.int([pair[0] for pair in matches.pairs()]) # refers to the model assert exp_reflections.size() == exp_miller_indices_original.size() assert observations_original_index.size() == exp_miller_indices_original.size() # narrow things down to the set that matches, only observations_pair1_selected = observations.customized_copy(indices = flex.miller_index([observations.indices()[p] for p in pair1]), data = flex.double([observations.data()[p] for p in pair1]), sigmas = flex.double([observations.sigmas()[p] for p in pair1])) observations_original_index_pair1_selected = observations_original_index.customized_copy(indices = flex.miller_index([observations_original_index.indices()[p] for p in pair1]), data = flex.double([observations_original_index.data()[p] for p in pair1]), sigmas = flex.double([observations_original_index.sigmas()[p] for p in pair1])) I_observed = observations_pair1_selected.data() MILLER = observations_original_index_pair1_selected.indices() ORI = crystal_orientation(experiment.crystal.get_A(), basis_type.reciprocal) Astar = matrix.sqr(ORI.reciprocal_matrix()) Astar_from_experiment = matrix.sqr(experiment.crystal.get_A()) assert Astar == Astar_from_experiment WAVE = experiment.beam.get_wavelength() BEAM = matrix.col((0.0,0.0,-1./WAVE)) BFACTOR = 0. MOSAICITY_DEG = experiment.crystal.get_half_mosaicity_deg() DOMAIN_SIZE_A = experiment.crystal.get_domain_size_ang() # calculation of correlation here I_reference = flex.double([i_model.data()[pair[0]] for pair in matches.pairs()]) I_invalid = flex.bool([i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()]) use_weights = False # New facility for getting variance-weighted correlation if use_weights: # variance weighting I_weight = flex.double([1./(observations_pair1_selected.sigmas()[pair[1]])**2 for pair in matches.pairs()]) else: I_weight = flex.double(len(observations_pair1_selected.sigmas()), 1.) I_weight.set_selected(I_invalid, 0.) """Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement: include_negatives = True + and - reflections both used for Rh distribution for initial estimate of RS parameter + and - reflections both used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) include_negatives = False + and - reflections both used for Rh distribution for initial estimate of RS parameter + reflections only used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) """ # RB: By design, for MPI-Merge "include negatives" is implicitly True SWC = simple_weighted_correlation(I_weight, I_reference, I_observed) if self.params.output.log_level == 0: self.logger.log("Old correlation is: %f"%SWC.corr) if self.params.postrefinement.algorithm == "rs": Rhall = flex.double() for mill in MILLER: H = matrix.col(mill) Xhkl = Astar*H Rh = ( Xhkl + BEAM ).length() - (1./WAVE) Rhall.append(Rh) Rs = math.sqrt(flex.mean(Rhall*Rhall)) RS = 1./10000. # reciprocal effective domain size of 1 micron RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.]) parameterization_class = rs_parameterization refinery = rs_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC = I_reference, IOBSVEC = I_observed) elif self.params.postrefinement.algorithm == "eta_deff": eta_init = 2. * MOSAICITY_DEG * math.pi/180. D_eff_init = 2. * DOMAIN_SIZE_A current = flex.double([SWC.slope, BFACTOR, eta_init, 0., 0., D_eff_init]) parameterization_class = eta_deff_parameterization refinery = eta_deff_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC = I_reference, IOBSVEC = I_observed) func = refinery.fvec_callable(parameterization_class(current)) functional = flex.sum(func * func) if self.params.output.log_level == 0: self.logger.log("functional: %f"%functional) self.current = current; self.parameterization_class = parameterization_class self.refinery = refinery; self.observations_pair1_selected = observations_pair1_selected; self.observations_original_index_pair1_selected = observations_original_index_pair1_selected error_detected = False try: self.run_plain() result_observations_original_index, result_observations, result_matches = self.result_for_cxi_merge() assert result_observations_original_index.size() == result_observations.size() assert result_matches.pairs().size() == result_observations_original_index.size() except (AssertionError, ValueError, RuntimeError) as e: error_detected = True reason = repr(e) if not reason: reason = "Unknown error" if not reason in experiments_rejected_by_reason: experiments_rejected_by_reason[reason] = 1 else: experiments_rejected_by_reason[reason] += 1 if not error_detected: new_experiments.append(experiment) new_exp_reflections = flex.reflection_table() new_exp_reflections['miller_index_asymmetric'] = flex.miller_index(result_observations.indices()) new_exp_reflections['intensity.sum.value'] = flex.double(result_observations.data()) new_exp_reflections['intensity.sum.variance'] = flex.double(flex.pow(result_observations.sigmas(),2)) new_exp_reflections['exp_id'] = flex.std_string(len(new_exp_reflections), experiment.identifier) new_reflections.extend(new_exp_reflections) ''' # debugging elif reason.startswith("ValueError"): self.logger.log("Rejected b/c of value error exp id: %s; unit cell: %s"%(exp_id, str(experiment.crystal.get_unit_cell())) ) ''' # report rejected experiments, reflections experiments_rejected_by_postrefinement = len(experiments) - len(new_experiments) reflections_rejected_by_postrefinement = reflections.size() - new_reflections.size() self.logger.log("Experiments rejected by post-refinement: %d"%experiments_rejected_by_postrefinement) self.logger.log("Reflections rejected by post-refinement: %d"%reflections_rejected_by_postrefinement) all_reasons = [] for reason, count in six.iteritems(experiments_rejected_by_reason): self.logger.log("Experiments rejected due to %s: %d"%(reason,count)) all_reasons.append(reason) comm = self.mpi_helper.comm MPI = self.mpi_helper.MPI # Collect all rejection reasons from all ranks. Use allreduce to let each rank have all reasons. all_reasons = comm.allreduce(all_reasons, MPI.SUM) all_reasons = set(all_reasons) # Now that each rank has all reasons from all ranks, we can treat the reasons in a uniform way. total_experiments_rejected_by_reason = {} for reason in all_reasons: rejected_experiment_count = 0 if reason in experiments_rejected_by_reason: rejected_experiment_count = experiments_rejected_by_reason[reason] total_experiments_rejected_by_reason[reason] = comm.reduce(rejected_experiment_count, MPI.SUM, 0) total_accepted_experiment_count = comm.reduce(len(new_experiments), MPI.SUM, 0) # how many reflections have we rejected due to post-refinement? rejected_reflections = len(reflections) - len(new_reflections); total_rejected_reflections = self.mpi_helper.sum(rejected_reflections) if self.mpi_helper.rank == 0: for reason, count in six.iteritems(total_experiments_rejected_by_reason): self.logger.main_log("Total experiments rejected due to %s: %d"%(reason,count)) self.logger.main_log("Total experiments accepted: %d"%total_accepted_experiment_count) self.logger.main_log("Total reflections rejected due to post-refinement: %d"%total_rejected_reflections) self.logger.log_step_time("POSTREFINEMENT", True) return new_experiments, new_reflections
def is_similar_to( self, other, angle_tolerance=0.01, mosaicity_tolerance=0.8, uc_rel_length_tolerance=0.01, uc_abs_angle_tolerance=1.0, ): """ Test similarity of this to another crystal model :param other: the crystal model to test against :type other: crystal_model :param angle_tolerance: maximum tolerated orientation difference in degrees :type angle_tolerance: float :param mosaicity_tolerance: minimum tolerated fraction of the larger mosaicity for the smaller mosaicity to retain similarity :type mosaicity_tolerance:float :param uc_rel_length_tolerance: relative length tolerance to pass to unit_cell.is_similar_to :type uc_rel_length_tolerance: float :param uc_abs_angle_tolerance: absolute angle tolerance to pass to unit_cell.is_similar_to :type uc_abs_angle_tolerance: float :returns: Whether the other crystal model is similar to this :rtype: bool """ # space group test if self.get_space_group() != other.get_space_group(): return False # mosaicity test m_a, m_b = self.get_mosaicity(deg=True), other.get_mosaicity(deg=True) min_m, max_m = min(m_a, m_b), max(m_a, m_b) if min_m <= 0.0: if max_m > 0.0: return False elif min_m / max_m < mosaicity_tolerance: return False # static orientation test U_a, U_b = self.get_U(), other.get_U() assert U_a.is_r3_rotation_matrix() assert U_b.is_r3_rotation_matrix() R_ab = U_b * U_a.transpose() uq = R_ab.r3_rotation_matrix_as_unit_quaternion() angle = uq.unit_quaternion_as_axis_and_angle(deg=True)[0] if abs(angle) > angle_tolerance: return False # static unit cell test uc_a, uc_b = self.get_unit_cell(), other.get_unit_cell() if not uc_a.is_similar_to( uc_b, relative_length_tolerance=uc_rel_length_tolerance, absolute_angle_tolerance=uc_abs_angle_tolerance, ): return False # scan varying tests if self.num_scan_points > 0: if other.num_scan_points != self.num_scan_points: return False for i in range(self.num_scan_points): U_a, U_b = self.get_U_at_scan_point( i), other.get_U_at_scan_point(i) assert U_a.is_r3_rotation_matrix() assert U_b.is_r3_rotation_matrix() R_ab = U_b * U_a.transpose() uq = R_ab.r3_rotation_matrix_as_unit_quaternion() angle = uq.unit_quaternion_as_axis_and_angle(deg=True)[0] if abs(angle) > angle_tolerance: return False B_a, B_b = self.get_B_at_scan_point( i), other.get_B_at_scan_point(i) uc_a = crystal_orientation(B_a, True).unit_cell() uc_b = crystal_orientation(B_b, True).unit_cell() if not uc_a.is_similar_to( uc_b, relative_length_tolerance=uc_rel_length_tolerance, absolute_angle_tolerance=uc_abs_angle_tolerance, ): return False return True
def apply_symmetry(self, crystal_model): """Apply symmetry constraints to a crystal model. Returns the crystal model (with symmetry constraints applied) in the same setting as provided as input. The cb_op returned by the method is that necessary to transform that model to the user-provided target symmetry. Args: crystal_model (dxtbx.model.Crystal): The input crystal model to which to apply symmetry constraints. Returns: (dxtbx.model.Crystal, cctbx.sgtbx.change_of_basis_op): The crystal model with symmetry constraints applied, and the change_of_basis_op that transforms the returned model to the user-specified target symmetry. """ if not (self.target_symmetry_primitive and self.target_symmetry_primitive.space_group()): return crystal, sgtbx.change_of_basis_op() target_space_group = self.target_symmetry_primitive.space_group() A = crystal_model.get_A() max_delta = self._max_delta items = iotbx_converter(crystal_model.get_unit_cell(), max_delta=max_delta) target_sg_ref = target_space_group.info().reference_setting().group() best_angular_difference = 1e8 best_subgroup = None for item in items: if bravais_lattice(group=target_sg_ref) != item["bravais"]: continue if item["max_angular_difference"] < best_angular_difference: best_angular_difference = item["max_angular_difference"] best_subgroup = item if best_subgroup is None: return None, None cb_op_inp_best = best_subgroup["cb_op_inp_best"] best_subsym = best_subgroup["best_subsym"] ref_subsym = best_subgroup["ref_subsym"] cb_op_ref_best = ref_subsym.change_of_basis_op_to_best_cell() cb_op_best_ref = cb_op_ref_best.inverse() cb_op_inp_ref = cb_op_best_ref * cb_op_inp_best cb_op_ref_inp = cb_op_inp_ref.inverse() orient = crystal_orientation(A, True) orient_ref = orient.change_basis( scitbx.matrix.sqr( (cb_op_inp_ref).c().as_double_array()[0:9]).transpose()) constrain_orient = orient_ref.constrain(best_subgroup["system"]) direct_matrix = constrain_orient.direct_matrix() a = scitbx.matrix.col(direct_matrix[:3]) b = scitbx.matrix.col(direct_matrix[3:6]) c = scitbx.matrix.col(direct_matrix[6:9]) model = Crystal(a, b, c, space_group=target_sg_ref) assert target_sg_ref.is_compatible_unit_cell(model.get_unit_cell()) model = model.change_basis(cb_op_ref_inp) if self.cb_op_inp_best is not None: # Then the unit cell has been provided: this is the cb_op to map to the # user-provided input unit cell return model, self.cb_op_inp_best.inverse() * cb_op_inp_best if not self.cb_op_ref_inp.is_identity_op(): if self.target_symmetry_inp.space_group( ) == best_subsym.space_group(): # Handle where e.g. the user has requested I2 instead of the reference C2 return model, cb_op_inp_best # The user has specified a setting that is not the reference setting return model, self.cb_op_ref_inp * cb_op_inp_ref # Default to reference setting # This change of basis op will ensure that we get the best beta angle without # changing the centring (e.g. from C2 to I2) cb_op_ref_best = ref_subsym.change_of_basis_op_to_best_cell( best_monoclinic_beta=False) return model, cb_op_ref_best * cb_op_inp_ref
def run_sim2smv(prefix, crystal, spectra, rotation, rank, gpu_channels_singleton, params, quick=False, save_bragg=False, sfall_channels=None): smv_fileout = prefix + ".img" burst_buffer_expand_dir = os.path.expandvars(params.logger.outdir) burst_buffer_fileout = os.path.join(burst_buffer_expand_dir, smv_fileout) reference_fileout = os.path.join(".", smv_fileout) if not quick: if not write_safe(reference_fileout): print("File %s already exists, skipping in rank %d" % (reference_fileout, rank)) return direct_algo_res_limit = 1.7 wavlen, flux, wavelength_A = next( spectra) # list of lambdas, list of fluxes, average wavelength assert wavelength_A > 0 # use crystal structure to initialize Fhkl array N = crystal.number_of_cells(sfall_channels[0].unit_cell()) #SIM = nanoBragg(detpixels_slowfast=(2000,2000),pixel_size_mm=0.11,Ncells_abc=(5,5,5),verbose=0) SIM = nanoBragg( detpixels_slowfast=(3000, 3000), pixel_size_mm=0.11, Ncells_abc=(N, N, N), # workaround for problem with wavelength array, specify it separately in constructor. wavelength_A=wavelength_A, verbose=0) SIM.adc_offset_adu = 0 # Do not offset by 40 SIM.adc_offset_adu = 10 # Do not offset by 40 import sys if len(sys.argv) > 2: SIM.seed = -int(sys.argv[2]) if len(sys.argv) > 1: if sys.argv[1] == "random": SIM.randomize_orientation() SIM.mosaic_spread_deg = 0.05 # interpreted by UMAT_nm as a half-width stddev SIM.mosaic_domains = 25 # 77 seconds. With 100 energy points, 7700 seconds (2 hours) per image # 3000000 images would be 100000 hours on a 60-core machine (dials), or 11.4 years # using 2 nodes, 5.7 years. Do this at SLAC? NERSC? combination of all? # SLAC downtimes: Tues Dec 5 (24 hrs), Mon Dec 11 (72 hrs), Mon Dec 18 light use, 24 days # mosaic_domains setter must come after mosaic_spread_deg setter SIM.distance_mm = 141.7 UMAT_nm = flex.mat3_double() mersenne_twister = flex.mersenne_twister(seed=0) scitbx.random.set_random_seed(1234) rand_norm = scitbx.random.normal_distribution(mean=0, sigma=SIM.mosaic_spread_deg * math.pi / 180.) g = scitbx.random.variate(rand_norm) mosaic_rotation = g(SIM.mosaic_domains) for m in mosaic_rotation: site = col(mersenne_twister.random_double_point_on_sphere()) UMAT_nm.append(site.axis_and_angle_as_r3_rotation_matrix(m, deg=False)) SIM.set_mosaic_blocks(UMAT_nm) #SIM.detector_thick_mm = 0.5 # = 0 for Rayonix #SIM.detector_thicksteps = 1 # should default to 1 for Rayonix, but set to 5 for CSPAD #SIM.detector_attenuation_length_mm = default is silicon # get same noise each time this test is run SIM.seed = 1 SIM.oversample = 1 SIM.wavelength_A = wavelength_A SIM.polarization = 1 # this will become F000, marking the beam center SIM.default_F = 0 SIM.Fhkl = sfall_channels[0] # instead of sfall_main Amatrix_rot = (rotation * sqr( sfall_channels[0].unit_cell().orthogonalization_matrix())).transpose() SIM.Amatrix_RUB = Amatrix_rot #workaround for failing init_cell, use custom written Amatrix setter print("unit_cell_Adeg=", SIM.unit_cell_Adeg) print("unit_cell_tuple=", SIM.unit_cell_tuple) Amat = sqr(SIM.Amatrix).transpose() # recovered Amatrix from SIM from cctbx import crystal_orientation Ori = crystal_orientation.crystal_orientation( Amat, crystal_orientation.basis_type.reciprocal) # fastest option, least realistic SIM.xtal_shape = shapetype.Gauss_argchk # both crystal & RLP are Gaussian # only really useful for long runs SIM.progress_meter = False # prints out value of one pixel only. will not render full image! # flux is always in photons/s SIM.flux = 1e12 SIM.exposure_s = 1.0 # so total fluence is e12 # assumes round beam SIM.beamsize_mm = 0.003 #cannot make this 3 microns; spots are too intense temp = SIM.Ncells_abc SIM.Ncells_abc = temp # rough approximation to water: interpolation points for sin(theta/lambda) vs structure factor water_bg = flex.vec2_double([(0, 2.57), (0.0365, 2.58), (0.07, 2.8), (0.12, 5), (0.162, 8), (0.18, 7.32), (0.2, 6.75), (0.216, 6.75), (0.236, 6.5), (0.28, 4.5), (0.3, 4.3), (0.345, 4.36), (0.436, 3.77), (0.5, 3.17)]) assert [a[0] for a in water_bg] == sorted([a[0] for a in water_bg]) # rough approximation to air air_bg = flex.vec2_double([(0, 14.1), (0.045, 13.5), (0.174, 8.35), (0.35, 4.78), (0.5, 4.22)]) assert [a[0] for a in air_bg] == sorted([a[0] for a in air_bg]) # simulated crystal is only 125 unit cells (25 nm wide) # amplify spot signal to simulate physical crystal of 4000x larger: 100 um (64e9 x the volume) SIM.raw_pixels *= crystal.domains_per_crystal # must calculate the correct scale! use_exascale_api = os.environ.get("USE_EXASCALE_API", "True") cache_fhkl_on_gpu = os.environ.get("CACHE_FHKL_ON_GPU", "True") assert use_exascale_api in ["True", "False" ] and cache_fhkl_on_gpu in ["True", "False"] use_exascale_api = (use_exascale_api == "True") cache_fhkl_on_gpu = (cache_fhkl_on_gpu == "True") QQ = Profiler("nanoBragg Bragg spots rank %d" % (rank)) if use_exascale_api: #something new devices_per_node = int(os.environ["DEVICES_PER_NODE"]) SIM.device_Id = rank % devices_per_node assert gpu_channels_singleton.get_deviceID() == SIM.device_Id if cache_fhkl_on_gpu: #flag to switch on GPU energy channels if gpu_channels_singleton.get_nchannels() == 0: # if uninitialized P = Profiler("Initialize the channels singleton rank %d" % (rank)) for x in range(len(flux)): gpu_channels_singleton.structure_factors_to_GPU_direct( x, sfall_channels[x].indices(), sfall_channels[x].data()) del P import time print("datetime for channels singleton rank %d" % (rank), time.time()) # allocate GPU arrays SIM.allocate() # loop over energies for x in range(len(flux)): P = Profiler("USE_EXASCALE_API nanoBragg Python and C++ rank %d" % (rank)) print("USE_EXASCALE_API+++++++++++++++++++++++ Wavelength", x) # from channel_pixels function SIM.wavelength_A = wavlen[x] SIM.flux = flux[x] if cache_fhkl_on_gpu: # new interface, use singleton to store all-image energy channels SIM.add_energy_channel_from_gpu_amplitudes( x, gpu_channels_singleton) else: # old interface, host-to-device each energy channel, each image SIM.Fhkl = sfall_channels[x] SIM.add_energy_channel_cuda() del P SIM.scale_in_place( crystal.domains_per_crystal) # apply scale directly on GPU SIM.wavelength_A = wavelength_A # return to canonical energy for subsequent background if add_background_algorithm == "cuda": QQ = Profiler("nanoBragg background rank %d" % (rank)) SIM.Fbg_vs_stol = water_bg SIM.amorphous_sample_thick_mm = 0.1 SIM.amorphous_density_gcm3 = 1 SIM.amorphous_molecular_weight_Da = 18 SIM.flux = 1e12 SIM.beamsize_mm = 0.003 # square (not user specified) SIM.exposure_s = 1.0 # multiplies flux x exposure SIM.add_background() SIM.Fbg_vs_stol = air_bg SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator SIM.amorphous_density_gcm3 = 1.2e-3 SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2 SIM.add_background() # deallocate GPU arrays SIM.get_raw_pixels() # updates SIM.raw_pixels from GPU SIM.deallocate() SIM.Amatrix_RUB = Amatrix_rot # return to canonical orientation del QQ else: for x in range(len(flux)): if rank in [0, 7]: P = Profiler("nanoBragg Python and C++ rank %d" % (rank)) if rank in [0, 7]: print("+++++++++++++++++++++++++++++++++++++++ Wavelength", x) CH = channel_pixels(wavlen[x], flux[x], N, UMAT_nm, Amatrix_rot, rank, sfall_channels[x]) SIM.raw_pixels += CH.raw_pixels * crystal.domains_per_crystal CHDBG_singleton.extract(channel_no=x, data=CH.raw_pixels) CH.free_all() if rank in [0, 7]: del P if add_background_algorithm in ["jh", "sort_stable"]: QQ = Profiler("nanoBragg background rank %d" % (rank)) SIM.Fbg_vs_stol = water_bg SIM.amorphous_sample_thick_mm = 0.1 SIM.amorphous_density_gcm3 = 1 SIM.amorphous_molecular_weight_Da = 18 SIM.flux = 1e12 SIM.beamsize_mm = 0.003 # square (not user specified) SIM.exposure_s = 1.0 # multiplies flux x exposure SIM.add_background( sort_stable=(add_background_algorithm == "sort_stable")) SIM.Fbg_vs_stol = air_bg SIM.amorphous_sample_thick_mm = 10 # between beamstop and collimator SIM.amorphous_density_gcm3 = 1.2e-3 SIM.amorphous_sample_molecular_weight_Da = 28 # nitrogen = N2 SIM.add_background( sort_stable=(add_background_algorithm == "sort_stable")) del QQ SIM.detector_psf_kernel_radius_pixels = 5 SIM.detector_psf_type = shapetype.Unknown # for CSPAD SIM.detector_psf_fwhm_mm = 0 #SIM.apply_psf() QQ = Profiler("nanoBragg noise rank %d" % (rank)) #SIM.add_noise() #converts phtons to ADU. del QQ extra = "PREFIX=%s;\nRANK=%d;\n" % (prefix, rank) SIM.to_smv_format_py(fileout=burst_buffer_fileout, intfile_scale=1, rotmat=True, extra=extra, gz=True) SIM.free_all()