def compute_f_calc(fmodel, params): from cctbx import miller coeffs_partial_set = fmodel.f_obs().structure_factors_from_scatterers( xray_structure = fmodel.xray_structure).f_calc() if(hasattr(params,"dev") and params.dev.complete_set_up_to_d_min): coeffs = fmodel.xray_structure.structure_factors( d_min = fmodel.f_obs().d_min()).f_calc() frac_inc = 1.*coeffs_partial_set.data().size()/coeffs.data().size() n_miss = coeffs.data().size() - coeffs_partial_set.data().size() if(params.dev.aply_same_incompleteness_to_complete_set_at == "randomly"): sel = flex.random_bool(coeffs.data().size(), frac_inc) coeffs = coeffs.select(sel) elif(params.dev.aply_same_incompleteness_to_complete_set_at == "low"): coeffs = coeffs.sort() coeffs = miller.set( crystal_symmetry = coeffs, indices = coeffs.indices()[n_miss+1:], anomalous_flag = coeffs.anomalous_flag()).array( data = coeffs.data()[n_miss+1:]) elif(params.dev.aply_same_incompleteness_to_complete_set_at == "high"): coeffs = coeffs.sort(reverse=True) coeffs = miller.set( crystal_symmetry = coeffs, indices = coeffs.indices()[n_miss+1:], anomalous_flag = coeffs.anomalous_flag()).array( data = coeffs.data()[n_miss+1:]) else: coeffs = coeffs_partial_set return coeffs
def as_miller_arrays(self, crystal_symmetry=None, force_symmetry=False, merge_equivalents=True, base_array_info=None): from cctbx import miller from cctbx import crystal if (crystal_symmetry is None): crystal_symmetry = crystal.symmetry() if (base_array_info is None): base_array_info = miller.array_info(source_type="shelx_hklf") miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=self.indices()).auto_anomalous() miller_arrays = [] obs = (miller.array( miller_set=miller_set, data=self.data(), sigmas=self.sigmas()) .set_info(base_array_info.customized_copy(labels=["obs", "sigmas"]))) miller_arrays.append(obs) if (self.alphas() is not None): miller_arrays.append(miller.array( miller_set=miller_set, data=self.alphas()) .set_info(base_array_info.customized_copy(labels=["alphas"]))) return miller_arrays
def exercise(): ma = miller.array( miller.set(crystal.symmetry(unit_cell=(5,5,5, 90, 90, 90), space_group=sgtbx.space_group('P 2x')), indices=flex.miller_index( [(1,0,0), (0,1,0), (0,0,1), (-1,0,0), (0,-1,0), (0,0,-1), (1,1,0), (1,0,1), (0,1,1), (-1,-1,0), (-1,0,-1), (0,-1,-1), (1,-1,0), (1,0,-1), (0,1,-1), (-1,1,0), (-1,0,1), (0,-1,1), (1,1,1), (-1,1,1), (1,-1,1), (1,1,-1), (-1,-1,-1), (1,-1,-1), (-1,1,-1), (-1,-1,1)])), data=flex.complex_double(flex.random_double(26), flex.random_double(26))) f_at_h = dict(zip(ma.indices(), ma.data())) for op in ("-x, y+1/2, -z", "x+1/2, -y, z-1/2"): op = sgtbx.rt_mx(op) original, transformed = ma.common_sets( ma.change_basis(sgtbx.change_of_basis_op(op.inverse()))) for h, f in original: assert f == f_at_h[h] for h, op_f in transformed: assert approx_equal( op_f, f_at_h[h*op.r()]*exp(1j*2*pi*row(h).dot(col(op.t().as_double()))))
def tst_pgtools(): unit_cell = uctbx.unit_cell('40, 40, 60, 90.0, 90.0, 90.0') mi = flex.miller_index(((2,4,6), (2,4,8))) xs = crystal.symmetry(unit_cell, "P 1 2 1") ms = miller.set(xs, mi) # Go to the minimum cell, for safety cob_min_cell = ms.change_of_basis_op_to_minimum_cell() ms_new = ms.change_basis( cob_min_cell ) lattice_group = sgtbx.lattice_symmetry.group( ms_new.unit_cell(), max_delta=5.0) point_group_low = ms_new.space_group().build_derived_point_group() point_group_high = lattice_group.build_derived_point_group() pgtree = pt.point_group_graph(point_group_low,point_group_high) # find the possible routes from 'P 2' to 'P 4 2 2' atlas = pgtree.graph.find_all_paths( 'P 1 2 1', 'P 4 2 2') route_1 = ['P 1 2 1', 'P 4 2 2'] route_2 = ['P 1 2 1', 'P 2 2 2', 'P 4 2 2'] assert route_1 in atlas assert route_2 in atlas assert len(atlas)==2 # Now lets 'disqualify' point group 'P 2 2 2' pgtree.remove_point_group_and_its_super_groups_from_graph( str(sgtbx.space_group_info(16))) assert len(pgtree.graph.node_objects)==1 assert pgtree.graph.node_objects.has_key ( 'P 1 2 1' )
def as_miller_array(self, crystal_symmetry=None, force_symmetry=False, merge_equivalents=True, base_array_info=None): if (base_array_info is None): base_array_info = miller.array_info(source_type="scalepack_merge") crystal_symmetry_from_file = self.crystal_symmetry() if (self.anomalous): labels = ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"] else: labels = ["I", "SIGI"] return (miller.array( miller_set=miller.set( crystal_symmetry=crystal_symmetry_from_file.join_symmetry( other_symmetry=crystal_symmetry, force=force_symmetry), indices=self.miller_indices, anomalous_flag=self.anomalous), data=self.i_obs, sigmas=self.sigmas) .set_info(base_array_info.customized_copy( labels=labels, crystal_symmetry_from_file=crystal_symmetry_from_file)) .set_observation_type_xray_intensity())
def randomize_struture_factors(map_coeffs, number_of_kicks, phases_only=False): map_coeff_data = None for kick in xrange(number_of_kicks): rc, ar, pr = random.choice([(0.1, 0.10, 40), (0.2, 0.09, 40), (0.3, 0.08, 30), (0.4, 0.07, 30), (0.5, 0.06, 20), (0.6, 0.05, 20), (0.7, 0.04, 20), (0.8, 0.03, 10), (0.9, 0.02, 10), (1.0, 0.01, 10) ]) if(phases_only): ar = 0 sel = flex.random_bool(map_coeffs.size(), rc) mc = map_coeffs.randomize_amplitude_and_phase( amplitude_error=ar, phase_error_deg=pr, selection=sel) if(map_coeff_data is None): map_coeff_data = mc.data() else: map_coeff_data = map_coeff_data + mc.data() map_coeff_data = map_coeff_data/number_of_kicks return miller.set( crystal_symmetry = map_coeffs.crystal_symmetry(), indices = map_coeffs.indices(), anomalous_flag = False).array(data = map_coeff_data)
def df2m(df, cell, spgr, data=None, sigmas=None): """Constructs a miller.array from the columns specified by data/sigmas in the dataframe, if both are None, returns just the indices. needs cell and spgr to generate a symmetry object.""" anomalous_flag = False if isinstance(df, pd.DataFrame): try: sel = df[data].notnull() # select notnull data items for index except ValueError: index = df.index else: index = df.index[sel] else: index = df indices = flex.miller_index(index) if data: data = flex.double(df[data][sel]) if sigmas: sigmas = flex.double(df[sigmas][sel]) symm = make_symmetry(cell, spgr) ms = miller.set( crystal_symmetry=symm, indices=indices, anomalous_flag=anomalous_flag) return miller.array(ms, data=data, sigmas=sigmas)
def as_mtz(self,scale_factor,prefix): #This reads in an hkl map and returns a .mtz map #Read in hkl file and populate miller array inf = open(self.diffuse_file, 'r') indices = flex.miller_index() i_obs = flex.double() sig_i = flex.double() for line in inf.readlines(): assert len(line.split())==4 line = line.strip().split() #####ATTENTION:SCALE FACTOR############## i_obs_ = float(line[3])/scale_factor #is a uniform scale factor meant to re-size all diffuse intensities (normally too large for scalepack) sig_i_ = math.sqrt(i_obs_) #if(abs(i_obs_)>1.e-6): # perhaps you don't want zeros indices.append([int(line[0]),int(line[1]),int(line[2])]) i_obs.append(i_obs_) sig_i.append(sig_i_) inf.close() # get miller array object cs = self.symmetry ma = miller.array(miller_set=miller.set(cs, indices), data=i_obs, sigmas=sig_i) ma.set_observation_type_xray_intensity() mtz_dataset = ma.as_mtz_dataset(column_root_label="I") mtz_dataset.mtz_object().write(prefix + '.mtz')
def as_miller_set(self, anomalous_flag=None): if anomalous_flag is None: anomalous_flag = self.anomalous return miller.set(crystal_symmetry=self.symm, indices=self.indices, anomalous_flag=anomalous_flag)
def __init__(self): self.data_obs1 = flex.double(2,1.0) self.data_obs2 = flex.double(2,3.0) self.sigma_obs1 = flex.double(2,0.1) self.sigma_obs2 = flex.double(2,1) self.unit_cell = uctbx.unit_cell('20, 30, 40, 90.0, 90.0, 90.0') #mi = flex.miller_index(((1,2,3), (1,2,3))) self.mi = flex.miller_index(((1,2,3), (5,6,7))) self.xs = crystal.symmetry((20,30,40), "P 2 2 2") self.ms = miller.set(self.xs, self.mi) self.u = [1,2,3,4,5,6] self.p_scale = 0.40 #self.u = [0,0,0,0,0,0] #self.p_scale = 0.00 self.ls_i_wt = scaling.least_squares_on_i_wt( self.mi, self.data_obs1, self.sigma_obs1, self.data_obs2, self.sigma_obs2, self.p_scale, self.unit_cell, self.u) self.ls_i = scaling.least_squares_on_i( self.mi, self.data_obs1, self.sigma_obs1, self.data_obs2, self.sigma_obs2, self.p_scale, self.unit_cell, self.u) self.ls_f_wt = scaling.least_squares_on_f_wt( self.mi, self.data_obs1, self.sigma_obs1, self.data_obs2, self.sigma_obs2, self.p_scale, self.unit_cell, self.u) self.ls_f = scaling.least_squares_on_f( self.mi, self.data_obs1, self.sigma_obs1, self.data_obs2, self.sigma_obs2, self.p_scale, self.unit_cell, self.u) self.tst_ls_f_wt() self.tst_ls_i_wt() self.tst_ls_f() self.tst_ls_i() self.tst_hes_ls_i_wt() self.tst_hes_ls_f_wt() self.tst_hes_ls_i() self.tst_hes_ls_f()
def cluster_completeness(self, clno, anomalous_flag, calc_redundancy=True): if clno not in self.clusters: print "Cluster No. %d not found" % clno return cls = self.clusters[clno][3] msets = map(lambda x: self.miller_sets[self.files[x-1]], cls) num_idx = sum(map(lambda x: x.size(), msets)) all_idx = flex.miller_index() all_idx.reserve(num_idx) for mset in msets: all_idx.extend(mset.indices()) # Calc median cell cells = numpy.array(map(lambda x: x.unit_cell().parameters(), msets)) median_cell = map(lambda i: numpy.median(cells[:,i]), xrange(6)) symm = msets[0].customized_copy(unit_cell=median_cell) assert anomalous_flag is not None # XXX all must belong to the same Laue group and appropriately reindexed.. all_set = miller.set(indices=all_idx, crystal_symmetry=symm, anomalous_flag=anomalous_flag) all_set = all_set.resolution_filter(d_min=self.d_min) # dummy for redundancy calculation. dirty way.. if calc_redundancy: dummy_array = miller.array(miller_set=all_set, data=flex.int(all_set.size())) merge = dummy_array.merge_equivalents() cmpl = merge.array().completeness() redun = merge.redundancies().as_double().mean() return cmpl, redun else: cmpl = all_set.unique_under_symmetry().completeness() return cmpl
def merge_obs(indices, iobs, sel, symm, anomalous_flag, d_min, d_max): #indices = flex.miller_index(indices) #iobs = flex.double(iobs) if sel is not None and len(sel) > 0: sel = flex.bool(sel) indices = indices.select(sel) iobs = iobs.select(sel) miller_set = miller.set(crystal_symmetry=symm, indices=indices, anomalous_flag=anomalous_flag) array = miller.array(miller_set=miller_set, data=iobs, ).set_observation_type_xray_intensity() array = array.resolution_filter(d_min=d_min, d_max=d_max) ## New way #array = array.map_to_asu() # #merger = yamtbx_dataproc_crystfel_ext.merge_equivalents_crystfel() #merger.add_observations(array.indices(), array.data()) #merger.merge() return array.merge_equivalents(algorithm="crystfel") # if sigmas is None, merge_equivalents_real() is used which simply averages.
def exercise(prefix="tst_models_to_from_chains"): # inputs expected_n = 5 xrs = iotbx.pdb.input(source_info=None, lines=pdb_str).xray_structure_simple() input_file_name = "%s.pdb"%prefix of = open(input_file_name,"w") print >> of, pdb_str of.close() mi = flex.miller_index(((0,0,1), )) ms = miller.set(xrs.crystal_symmetry(), mi, anomalous_flag=False) complete_set = ms.complete_set(d_min=3) fc = complete_set.structure_factors_from_scatterers( xray_structure=xrs).f_calc() # models -> chains easy_run.call("phenix.models_as_chains %s"%input_file_name) pdb_inp = iotbx.pdb.input(file_name="chains_"+input_file_name) h = pdb_inp.construct_hierarchy() assert len(list(h.chains()))==expected_n assert len(list(h.models()))==1 xrs_c = pdb_inp.xray_structure_simple() fc_c = complete_set.structure_factors_from_scatterers( xray_structure=xrs_c).f_calc() # easy_run.call("phenix.chains_as_models chains_%s"%input_file_name) pdb_inp = iotbx.pdb.input(file_name="models_chains_"+input_file_name) h = pdb_inp.construct_hierarchy() assert len(list(h.chains()))==expected_n assert len(list(h.models()))==expected_n xrs_m = pdb_inp.xray_structure_simple() fc_m = complete_set.structure_factors_from_scatterers( xray_structure=xrs_m).f_calc() # assert approx_equal(0, r(fc, fc_c) ) assert approx_equal(0, r(fc, fc_m) ) assert approx_equal(0, r(fc_c, fc_m))
def extend_symmetry(self,scale_factor,prefix): #This reads in an hkl map and returns a .sca map #Read in hkl file and populate miller array inf = open(self.diffuse_file, 'r') indices = flex.miller_index() i_obs = flex.double() sig_i = flex.double() for line in inf.readlines(): assert len(line.split())==4 line = line.strip().split() #####ATTENTION:SCALE FACTOR############## i_obs_ = float(line[3])/scale_factor #is a uniform scale factor meant to re-size all diffuse intensities (normally too large for scalepack) sig_i_ = math.sqrt(i_obs_) #if(abs(i_obs_)>1.e-6): # perhaps you don't want zeros indices.append([int(line[0]),int(line[1]),int(line[2])]) i_obs.append(i_obs_) sig_i.append(sig_i_) inf.close() # get miller array object cs = self.symmetry ma = miller.array(miller_set=miller.set(cs, indices), data=i_obs, sigmas=sig_i) ma.set_observation_type_xray_intensity() ma_anom = ma.customized_copy(anomalous_flag=False) ma_p1 = ma_anom.expand_to_p1() merge.write(file_name= prefix + '.sca', miller_array=ma_p1) self.p1_map = prefix + '.sca'
def aniso_convert(uc,sg,map,res): os.system('hkl2vtk %s_friedel.hkl %s_friedel.vtk %s_raw.vtk' %(map,map,map)) os.system('vtk2lat %s_friedel.vtk %s_friedel.lat' %(map,map)) os.system('avgrlt %s_friedel.lat %s_friedel.rf' %(map,map)) os.system('subrflt %s_friedel.rf %s_friedel.lat anisotropic.lat' %(map,map)) os.system('lat2hkl anisotropic.lat anisotropic.hkl') #Read in hkl file and populate miller array from cctbx import crystal inf = open('anisotropic.hkl', "r") indices = flex.miller_index() i_obs = flex.double() #sig_i = flex.double() for line in inf.readlines(): assert len(line.split())==4 line = line.strip().split() i_obs_ = float(line[3])#/10000 #10000 is a uniform scale factor meant to re-size all diffuse intensities (normally too large for scalepack) #sig_i_ = math.sqrt(i_obs_) #if(abs(i_obs_)>1.e-6): # perhaps you don't want zeros if i_obs_ != -32768.0: indices.append([int(line[0]),int(line[1]),int(line[2])]) i_obs.append(i_obs_) #sig_i.append(sig_i_) inf.close() #Get miller array object cs = crystal.symmetry(unit_cell=(float(uc[0]), float(uc[1]), float(uc[2]), float(uc[3]), float(uc[4]), float(uc[5])), space_group=sg) miller_set=miller.set(cs, indices) ma = miller.array(miller_set=miller_set, data=i_obs, sigmas=None) mtz_dataset = ma.as_mtz_dataset(column_root_label="Amplitude") mtz_dataset.mtz_object().write('%s_anisotropic.mtz' %map)
def __init__(self, strategies, n_bins=8, degrees_per_bin=5): from cctbx import crystal, miller import copy sg = strategies[0].experiment.crystal.get_space_group() \ .build_derived_reflection_intensity_group(anomalous_flag=True) cs = crystal.symmetry( unit_cell=strategies[0].experiment.crystal.get_unit_cell(), space_group=sg) for i, strategy in enumerate(strategies): if i == 0: predicted = copy.deepcopy(strategy.predicted) else: predicted_ = copy.deepcopy(strategy.predicted) predicted_['dose'] += (flex.max(predicted['dose']) + 1) predicted.extend(predicted_) ms = miller.set(cs, indices=predicted['miller_index'], anomalous_flag=True) ma = miller.array(ms, data=flex.double(ms.size(),1), sigmas=flex.double(ms.size(), 1)) if 1: merging = ma.merge_equivalents() o = merging.array().customized_copy( data=merging.redundancies().data().as_double()).as_mtz_dataset('I').mtz_object() o.write('predicted.mtz') d_star_sq = ma.d_star_sq().data() binner = ma.setup_binner_d_star_sq_step( d_star_sq_step=(flex.max(d_star_sq)-flex.min(d_star_sq)+1e-8)/n_bins) dose = predicted['dose'] range_width = 1 range_min = flex.min(dose) - range_width range_max = flex.max(dose) n_steps = 2 + int((range_max - range_min) - range_width) binner_non_anom = ma.as_non_anomalous_array().use_binning( binner) self.n_complete = flex.size_t(binner_non_anom.counts_complete()[1:-1]) from xia2.Modules.PyChef2 import ChefStatistics chef_stats = ChefStatistics( ma.indices(), ma.data(), ma.sigmas(), ma.d_star_sq().data(), dose, self.n_complete, binner, ma.space_group(), ma.anomalous_flag(), n_steps) def fraction_new(completeness): # Completeness so far at end of image completeness_end = completeness[1:] # Completeness so far at start of image completeness_start = completeness[:-1] # Fraction of unique reflections observed for the first time on each image return completeness_end - completeness_start self.dose = dose self.ieither_completeness = chef_stats.ieither_completeness() self.iboth_completeness = chef_stats.iboth_completeness() self.frac_new_ref = fraction_new(self.ieither_completeness) / degrees_per_bin self.frac_new_pairs = fraction_new(self.iboth_completeness) / degrees_per_bin
def exercise_systematic(verbose): from cctbx import miller from cctbx import crystal from cctbx.array_family import flex cs = crystal.symmetry( unit_cell=(13,15,14,90,90,100), space_group_symbol="P112") ms = miller.set( crystal_symmetry=cs, indices=flex.miller_index([ (0,0,1),(0,0,-1), (0,1,1), (1,0,0), (-1,-1,-1)]), anomalous_flag=True).map_to_asu() cf = ms.centric_flags().data() assert cf.count(True) == 1 mt = flex.mersenne_twister(seed=0) ma = ms.array( data=mt.random_double(size=5)+0.1, sigmas=mt.random_double(size=5)+0.1) def recycle(expected_column_data): mtz_obj = ma.as_mtz_dataset(column_root_label="X").mtz_object() sio = StringIO() mtz_obj.show_column_data_human_readable(out=sio) from libtbx.test_utils import show_diff if (verbose): sys.stdout.write(sio.getvalue()) assert not show_diff(sio.getvalue(), expected_column_data) ma_2 = only_element(mtz_obj.as_miller_arrays()) assert_equal_data_and_sigmas(ma, ma_2) recycle("""\ Column data: ------------------------------------------------------------------------------- X(+) SIGX(+) X(-) SIGX(-) 0 0 1 0.517022 0.192339 0.820324 0.28626 0 1 1 0.100114 0.445561 None None 1 0 0 0.402333 0.496767 None None 1 1 1 None None 0.246756 0.638817 ------------------------------------------------------------------------------- """) from cctbx.xray import observation_types ma.set_observation_type(observation_types.reconstructed_amplitude()) recycle("""\ Column data: ------------------------------------------------------------------------------- X SIGX DANOX SIGDANOX ISYMX 0 0 1 0.668673 0.172438 -0.303302 0.344875 0 0 1 1 0.100114 0.445561 None None 1 1 0 0 0.402333 0.496767 None None 0 1 1 1 0.246756 0.638817 None None 2 ------------------------------------------------------------------------------- """)
def verify(sg_fcalc, sg_hl, sg_cns, p1_cns): sg_phase_integrals = miller.array( miller_set=miller.set( crystal_symmetry=sg_fcalc, indices=sg_cns.miller_indices, anomalous_flag=sg_fcalc.anomalous_flag()), data=sg_cns.hl).phase_integrals() for h, cns_pi, miller_pi in zip(sg_cns.miller_indices, sg_cns.pi.data, sg_phase_integrals.data()): if (abs(cns_pi - miller_pi) > 1.e-2): print "Error:", h, cns_pi, miller_pi if (0): return raise AssertionError # p1_phase_integrals = miller.array( miller_set=miller.set( crystal_symmetry=sg_fcalc.cell_equivalent_p1(), indices=p1_cns.miller_indices, anomalous_flag=sg_fcalc.anomalous_flag()), data=p1_cns.hl).phase_integrals() for h, cns_pi, miller_pi in zip(p1_cns.miller_indices, p1_cns.pi.data, p1_phase_integrals.data()): if (abs(cns_pi - miller_pi) > 1.e-2): print "Error:", h, cns_pi, miller_pi if (0): return raise AssertionError # space_group = sg_fcalc.space_group() asu = sg_fcalc.space_group_info().reciprocal_space_asu() lookup_dict = miller.make_lookup_dict(sg_fcalc.indices()) for p1_i, h in enumerate(p1_cns.miller_indices): h_asu = miller.asym_index(space_group, asu, h) h_eq = h_asu.one_column(sg_fcalc.anomalous_flag()) fcalc_asu = h_eq.complex_eq(p1_cns.fcalc.data[p1_i]) hl_asu = h_eq.hendrickson_lattman_eq(p1_cns.hl[p1_i]) sg_i = lookup_dict[h_eq.h()] assert abs(sg_fcalc.data()[sg_i] - fcalc_asu) < 1.e-2 if (not approx_equal(sg_hl[sg_i], hl_asu, eps=1.e-2)): print "Error:", sg_fcalc.space_group_info() print sg_fcalc.indices()[sg_i] print "i:", sg_hl[sg_i] print "o:", hl_asu if (0): return raise AssertionError
def dummy_miller_set(crystal_symmetry, log_n_reflections): indices = flex.miller_index(((1,2,3),)) for i in xrange(log_n_reflections): indices.append(indices) return miller.set( crystal_symmetry=crystal_symmetry, indices=indices, anomalous_flag=False)
def run(files): assert len(files) == 2 hkl1 = xds_ascii.XDS_ASCII(files[0], sys.stdout) hkl2 = xds_ascii.XDS_ASCII(files[1], sys.stdout) hkl1_points = numpy.column_stack((hkl1.xd, hkl1.yd, hkl1.zd)) tree1 = spatial.cKDTree(hkl1_points) n_ovl, n_nonovl = 0, 0 novl_indices, novl_i, novl_sigma = flex.miller_index(), flex.double(), flex.double() for i in xrange(len(hkl2.indices)): x, y, z = hkl2.xd[i], hkl2.yd[i], hkl2.zd[i] #if z > 180: # continue dists, idxs = tree1.query((x,y,z), k=3, p=1) overlaps = [] for dist, idx in zip(dists, idxs): idx = int(idx) xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx] if abs(z-zo) < 2.5 and (xo-x)**2+(yo-y)**2 < 15**2: # FIXME MAGIC NUMBER! overlaps.append((dist,idx)) if len(overlaps) == 0: novl_indices.append(hkl2.indices[i]) novl_i.append(hkl2.iobs[i]) novl_sigma.append(hkl2.sigma_iobs[i]) n_nonovl += 1 else: print hkl2.indices[i], x, y, z for dist, idx in overlaps: xo, yo, zo = hkl1.xd[idx], hkl1.yd[idx], hkl1.zd[idx] print hkl1.indices[idx], xo, yo, zo print dist, idx print print n_ref = len(hkl2.indices) print "%.2f%% overlap!" % (100.*(n_ref-n_nonovl)/n_ref) novl_array = miller.array(miller_set=miller.set(crystal_symmetry=hkl2.symm, indices=novl_indices), data=novl_i, sigmas=novl_sigma) stats = dataset_statistics(novl_array, anomalous=False, sigma_filtering="xds") stats.show(out=sys.stdout) novl_array = novl_array.customized_copy(anomalous_flag=False).map_to_asu() novl_array = novl_array.eliminate_sys_absent() novl_array = novl_array.select(novl_array.sigmas() >= 0) filtr = filter_intensities_by_sigma(novl_array, "xds") hklout = os.path.splitext(os.path.basename(files[1]))[0] + "_novl.mtz" filtr.array_merged.set_observation_type_xray_intensity().as_mtz_dataset(column_root_label="IMEAN").mtz_object().write(hklout)
def column_group( crystal_symmetry_from_file, crystal_symmetry, base_array_info, primary_column_type, labels, group, observation_type): assert group.data.size() == group.indices.size() sigmas = getattr(group, "sigmas", None) if (sigmas is not None): assert sigmas.size() == group.indices.size() if (group.anomalous_flag): miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=group.indices, anomalous_flag=True) if (miller_set.n_bijvoet_pairs() == 0): # account for non-sensical files generated via # ccp4i "import merged data" tab with default parameters miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=group.indices, anomalous_flag=False) else: miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=group.indices).auto_anomalous(min_fraction_bijvoet_pairs=2/3.) result = miller_set.array( data=group.data, sigmas=sigmas).set_info( base_array_info.customized_copy( labels=labels, crystal_symmetry_from_file=crystal_symmetry_from_file, type_hints_from_file=column_type_as_miller_array_type_hints.get( primary_column_type))) if (observation_type is not None): result.set_observation_type(observation_type) elif (not result.is_complex_array()): if (primary_column_type in "FG"): result.set_observation_type_xray_amplitude() elif (primary_column_type in "JK"): result.set_observation_type_xray_intensity() return result
def i_obs(self, anomalous_flag=None): assert "IOBS" in self.data assert "SIGMA" in self.data array_info = miller.array_info(source_type="xds_integrate")#, wavelength=) return miller.array(miller_set=miller.set(crystal_symmetry=self.crystal_symmetry(), indices=self.hkl, anomalous_flag=anomalous_flag), data=self.data["IOBS"], sigmas=self.data["SIGMA"]).set_info(array_info).set_observation_type_xray_intensity()
def miller_set (self, crystal_symmetry=None, force_symmetry=False) : crystal_symmetry_from_file = self.crystal_symmetry() return miller.set( crystal_symmetry=crystal_symmetry_from_file.join_symmetry( other_symmetry=crystal_symmetry, force=force_symmetry), indices=self.miller_indices, anomalous_flag=self.anomalous_flag)
def as_normalised_array(miller_array, asu_contents, wilson_plot=None): """Old python code replaced by faster C++ code.""" from cctbx import statistics from cctbx import eltbx if not wilson_plot: wilson_plot = statistics.wilson_plot(miller_array, asu_contents) # cache scattering factor info gaussians = {} for chemical_type in asu_contents: gaussians.setdefault(chemical_type, eltbx.xray_scattering.wk1995( chemical_type).fetch()) stol_sq = miller_array.sin_theta_over_lambda_sq() epsilons = miller_array.epsilons() e_sq_minus_1 = 0 n_e_greater_than_2 = 0 normalised_f_obs = flex.double() space_group = miller_array.space_group() tau = space_group.n_ltr() for i in xrange(0,miller_array.size()): s_sq = stol_sq.data()[i] f_sq = math.pow(miller_array.data()[i], 2) epsilon = epsilons.data()[i] sum_fj_sq = 0 for chemical_type, n_atoms in asu_contents.items(): n_atoms *= space_group.order_z() f0 = gaussians[chemical_type].at_stol_sq(s_sq) sum_fj_sq += f0 * f0 * n_atoms e_sq = f_sq\ /(wilson_plot.wilson_intensity_scale_factor*math.exp(-2*wilson_plot.wilson_b*s_sq) *epsilon *tau *sum_fj_sq) normalised_f_obs.append(math.sqrt(e_sq)) e_sq_minus_1 += abs(e_sq - 1) if (e_sq > 4.0): n_e_greater_than_2 += 1 r = Empty() r.array = miller.array( miller_set=miller.set( crystal_symmetry=miller_array.crystal_symmetry(), indices=miller_array.indices()).auto_anomalous(), data=normalised_f_obs, sigmas=miller_array.sigmas()) r.mean_e_sq_minus_1 = e_sq_minus_1/r.array.size() r.percent_e_sq_gt_2 = (100.0*n_e_greater_than_2)/r.array.size() return r
def exercise_sys_absent_intensity_distribution(): xs = crystal.symmetry((3,4,5), "F222") mi = flex.miller_index(((1,2,3), (1,1,1), (1,2,2), (0,0,4))) ms = miller.set(xs, mi) data = flex.double((-1,-2,3,4)) sigmas = flex.double((2,3,4,5)) f_obs = miller.array(ms, data=data, sigmas=sigmas).set_observation_type_xray_intensity() dist = statistics.sys_absent_intensity_distribution(f_obs) assert approx_equal(dist.x,(-0.5,0.75)) assert approx_equal(dist.y,(-1,3)) assert approx_equal(dist.indices,((1,2,3), (1,2,2)))
def arrays(self): ret = {} for key in self.data: arr = miller.array(miller_set=miller.set(crystal_symmetry=self.crystal_symmetry(), indices=self.hkl, anomalous_flag=False), data=self.data[key]) ret[key] = arr return ret
def unit_cell_to_symetry_object(img_dict): xsym = symmetry(unit_cell=img_dict['current_orientation'][0].unit_cell().niggli_cell(), space_group_symbol=point_group) miller_set = miller.set(crystal_symmetry=xsym, indices=flex.miller_index(img_dict['Millers']), anomalous_flag=True) miller_array = miller.array(miller_set, flex.double(img_dict['Is']), flex.double(img_dict['sigIs'])) miller_array.set_observation_type_xray_intensity() miller_array.set_info("Raw data obtained by integration using CrystFEL") return miller_array
def unmerged_miller_set(self, crystal_symmetry=None, force_symmetry=False): if (not force_symmetry or crystal_symmetry is None or crystal_symmetry.space_group_info() is None): space_group_info = self.space_group_info() else: space_group_info = None return miller.set( crystal_symmetry=self.crystal_symmetry().join_symmetry( other_symmetry=crystal_symmetry, force=force_symmetry), indices=self.original_indices, anomalous_flag=True)
def make_joined_set (miller_arrays) : if(len(miller_arrays)==0): return None cs0 = miller_arrays[0].crystal_symmetry() for ma in miller_arrays: if(not ma.crystal_symmetry().is_similar_symmetry(cs0)): return None from cctbx import miller master_set = miller.set( crystal_symmetry=miller_arrays[0].crystal_symmetry(), indices=miller_arrays[0].indices(), anomalous_flag=False) master_indices = miller_arrays[0].indices().deep_copy() for array in miller_arrays[1:] : current_indices = array.indices() missing_isel = miller.match_indices(master_indices, current_indices).singles(1) missing_indices = current_indices.select(missing_isel) master_indices.extend(missing_indices) master_set = miller.set( crystal_symmetry=miller_arrays[0].crystal_symmetry(), indices=master_indices, anomalous_flag=False) return master_set.map_to_asu().unique_under_symmetry()
def tst_sg_tools(): unit_cell = uctbx.unit_cell('40, 50, 60, 90.0, 90.0, 90.0') mi = flex.miller_index(((2,4,6), (2,4,8))) xs = crystal.symmetry(unit_cell, "P 1 1 21") ms = miller.set(xs, mi) tmp_choice = pt.space_group_graph_from_cell_and_sg(unit_cell, xs.space_group()) xs1=crystal.symmetry(uctbx.unit_cell('40.00 60.00 50.00 90.00 90.00 90.00'), 'P 1 21 1') xs2=crystal.symmetry(uctbx.unit_cell('40.00 50.00 60.00 90.00 90.00 90.00'), 'P 2 2 21') xs3=crystal.symmetry(uctbx.unit_cell('40.00 60.00 50.00 90.00 90.00 90.00'), 'P 21 21 2') xs4=crystal.symmetry(uctbx.unit_cell('50.00 60.00 40.00 90.00 90.00 90.00'), 'P 21 21 2') xs5=crystal.symmetry(uctbx.unit_cell('40.00 50.00 60.00 90.00 90.00 90.00'), 'P 21 21 21') p222_dict = { str( xs2.unit_cell().parameters()) + " " + str( xs2.space_group_info() ): None, str( xs3.unit_cell().parameters()) + " " + str( xs3.space_group_info() ):None, str( xs4.unit_cell().parameters()) + " " + str( xs4.space_group_info() ):None, str( xs5.unit_cell().parameters()) + " " + str( xs5.space_group_info() ):None } p112 = tmp_choice.pg_graph.graph.node_objects['P 1 1 2'].allowed_xtal_syms # check the cell parameters assert approx_equal(p112[0][0].unit_cell().parameters(), xs1.unit_cell().parameters()) # check the sg assert p112[0][0].space_group() == xs1.space_group() # check the change of basis operator assert approx_equal(xs.change_basis(p112[0][1]).unit_cell().parameters(), xs1.unit_cell().parameters()) p222 = tmp_choice.pg_graph.graph.node_objects['P 2 2 2'].allowed_xtal_syms for xs in p222: comp_string = (str(xs[0].unit_cell().parameters()) + " " + str(xs[0].space_group_info())) assert p222_dict.has_key(comp_string)
def run(self, experiments, reflections): self.logger.log_step_time("REINDEX") # Get list of twinning operators for this space group operators = twin_laws(self.params.scaling.i_model).operators if not operators: self.logger.log("No indexing ambiguity. Skipping this step.") return experiments, reflections self.logger.log("Resolving indexing ambiguity using operators h,k,l, %s"%", ".join( \ [op.operator.r().as_hkl() for op in operators])) operators = [ sgtbx.change_of_basis_op(op.operator.r().as_hkl()) for op in operators ] result = flex.reflection_table() scaler = experiment_scaler(self.params, self.mpi_helper, self.logger) model_intensities = self.params.scaling.i_model target_symm = symmetry( unit_cell=self.params.scaling.unit_cell, space_group_info=self.params.scaling.space_group) def get_correlation(cb_op=None): """ Helper function to get CC to the reference given an operator """ # Build a miller array for the experiment reflections exp_miller_indices = miller.set( target_symm, exp_reflections['miller_index_asymmetric'], True) exp_intensities = miller.array( exp_miller_indices, exp_reflections['intensity.sum.value'], flex.sqrt(exp_reflections['intensity.sum.variance'])) if cb_op: exp_intensities = exp_intensities.change_basis( cb_op).map_to_asu() # Extract an array of HKLs from the model to match the experiment HKLs matching_indices = miller.match_multi_indices( miller_indices_unique=model_intensities.indices(), miller_indices=exp_intensities.indices()) # Least squares scaling_result = scaler.fit_experiment_to_reference( model_intensities, exp_intensities, matching_indices) return scaling_result.correlation if scaling_result.correlation is not None else -1 # Test each experiment to see if an operator gives a better CC to the reference, and if it does, apply it for expt_id, experiment in enumerate(experiments): exp_reflections = reflections.select( reflections['exp_id'] == experiment.identifier) all_correlations = [] best_correlation = get_correlation() all_correlations.append(best_correlation) best_op = None for cb_op in operators: test_correlation = get_correlation(cb_op) all_correlations.append(test_correlation) if test_correlation > best_correlation: best_correlation = test_correlation best_op = cb_op if best_op: exp_miller_indices = miller.set( target_symm, exp_reflections['miller_index'], True).change_basis(best_op) exp_reflections[ 'miller_index_asymmetric'] = exp_miller_indices.map_to_asu( ).indices() exp_reflections['miller_index'] = exp_miller_indices.indices() experiment.crystal = MosaicCrystalSauter2014( experiment.crystal.change_basis(best_op) ) # need to use wrapper because of cctbx/dxtbx#5 result.extend(exp_reflections) self.logger.log( "Expt %d, reindexing op correlations: %s" % (expt_id, ", ".join(["%6.3f" % c for c in all_correlations]))) self.logger.log_step_time("REINDEX", True) self.logger.log("Memory usage: %d MB" % get_memory_usage()) return experiments, result
# Load the cctbx.xfel.merge output for estimation of initial params if args.mtzinput: # TODO: make this so that one can selec on label (e.g. fobs) F = any_reflection_file( args.merge).as_miller_arrays()[0].as_amplitude_array() else: F = utils.open_flex(args.merge) #.as_amplitude_array() Fmap = {h: amp for h, amp in zip(F.indices(), F.data())} merge_Hi = list(set(map_hkl_list(Fmap.keys(), symbol=args.symbol))) symm = symmetry(unit_cell=(a, b, c, al, be, ga), space_group_symbol=args.symbol) # diffBragg resolutions mset = miller.set(symm, flex.miller_index(u_all_Hi), anomalous_flag=True) reso = mset.d_spacings().data() reso = np.array(reso)[:, None] # merged resolutions msetm = miller.set(symm, flex.miller_index(merge_Hi), anomalous_flag=True) resom = msetm.d_spacings().data() resom = np.array(resom)[:, None] tree = cKDTree(resom) o, knearest = tree.query(reso, k=5) u_all_amp = [] for i_h, h in enumerate(u_all_Hi): if h in Fmap: in_array = True
def run(args): import libtbx.load_env usage = "dev.dials.estimate_resolution_limit indexed.expt indexed.refl [options]" parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, read_reflections=True, check_format=False, epilog=help_message, ) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if len(experiments) == 0: parser.print_help() return elif len(experiments) > 1: raise Sorry("More than one experiment present") experiment = experiments[0] assert len(reflections) == 1 reflections = reflections[0] intensities = reflections["intensity.sum.value"] variances = reflections["intensity.sum.variance"] if "intensity.prf.value" in reflections: intensities = reflections["intensity.prf.value"] variances = reflections["intensity.prf.variance"] sel = variances > 0 intensities = intensities.select(sel) variances = variances.select(sel) sigmas = flex.sqrt(variances) indices = reflections["miller_index"].select(sel) from cctbx import crystal, miller crystal_symmetry = crystal.symmetry( space_group=experiment.crystal.get_space_group(), unit_cell=experiment.crystal.get_unit_cell(), ) miller_set = miller.set(crystal_symmetry=crystal_symmetry, anomalous_flag=True, indices=indices) miller_array = miller.array( miller_set=miller_set, data=intensities, sigmas=sigmas).set_observation_type_xray_intensity() # miller_array.setup_binner(n_bins=50, reflections_per_bin=100) miller_array.setup_binner(auto_binning=True, n_bins=20) result = miller_array.i_over_sig_i(use_binning=True) result.show() from cctbx import uctbx d_star_sq_centre = result.binner.bin_centers(2) i_over_sig_i = flex.double( [d if d is not None else 0 for d in result.data[1:-1]]) sel = i_over_sig_i > 0 d_star_sq_centre = d_star_sq_centre.select(sel) i_over_sig_i = i_over_sig_i.select(sel) log_i_over_sig_i = flex.log(i_over_sig_i) weights = result.binner.counts()[1:-1].as_double().select(sel) fit = flex.linear_regression(d_star_sq_centre, log_i_over_sig_i, weights=weights) m = fit.slope() c = fit.y_intercept() import math y_cutoff = math.log(params.i_sigi_cutoff) x_cutoff = (y_cutoff - c) / m estimated_d_min = uctbx.d_star_sq_as_d(x_cutoff) print("estimated d_min: %.2f" % estimated_d_min) if params.plot: from matplotlib import pyplot fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.plot(list(d_star_sq_centre), list(log_i_over_sig_i), label=r"ln(I/sigI)") ax.plot(pyplot.xlim(), [(m * x + c) for x in pyplot.xlim()], color="red") ax.plot([x_cutoff, x_cutoff], pyplot.ylim(), color="grey", linestyle="dashed") ax.plot(pyplot.xlim(), [y_cutoff, y_cutoff], color="grey", linestyle="dashed") ax.set_xlabel("d_star_sq") ax.set_ylabel("ln(I/sigI)") ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis xticks = ax.get_xticks() xlim = ax.get_xlim() xticks_d = [ uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ] xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks] ax_.set_xticks(xticks) ax_.set_xlim(ax.get_xlim()) ax_.set_xlabel(r"Resolution ($\AA$)") ax_.set_xticklabels(["%.1f" % d for d in xticks_d]) pyplot.savefig("estimate_resolution_limit.png") pyplot.clf()
def demo(): # # create toy lists of Miller indices # crystal_symmetry = crystal.symmetry(unit_cell=(13, 14, 15, 90, 90, 90), space_group_symbol="P212121") miller_set_a = miller.set(crystal_symmetry=crystal_symmetry, anomalous_flag=False, indices=flex.miller_index([(0, -1, 2), (-1, -2, 3), (2, 3, 4), (-3, 4, 5), (4, -5, 6)])) miller_set_b = miller_set_a.customized_copy( indices=flex.miller_index([(-5, -6, 7), (0, 1, 2), (3, -4, 5), (-1, -2, 3), (-4, -5, 6)])) # # map all indices to the asymmetric unit # asu_a = miller_set_a.map_to_asu() asu_b = miller_set_b.map_to_asu() for h in asu_a.indices(): print("asu a:", h) print() for h in asu_b.indices(): print("asu b:", h) print() # # obtain the common index sets # common_a, common_b = asu_a.common_sets(asu_b) for h in common_a.indices(): print("common a:", h) print() for h in common_b.indices(): print("common b:", h) print() # # obtain the "lone" index sets # lone_set_a, lone_set_b = asu_a.lone_sets(asu_b) for h in lone_set_a.indices(): print("lone a:", h) print() for h in lone_set_b.indices(): print("lone b:", h) print() # # now the same again, but with data (i.e. miller.array instances) # miller_array_a = miller_set_a.array(data=flex.random_double( size=miller_set_a.indices().size())) miller_array_b = miller_set_b.array(data=flex.random_double( size=miller_set_a.indices().size())) # # map all indices to the asymmetric unit # asu_a = miller_array_a.map_to_asu() asu_b = miller_array_b.map_to_asu() asu_a.show_array(prefix="asu a: ") print() asu_b.show_array(prefix="asu b: ") print() # # obtain the common index sets # common_a, common_b = asu_a.common_sets(asu_b) common_a.show_array(prefix="common a: ") print() common_b.show_array(prefix="common b: ") print() # # obtain the "lone" index sets # lone_a, lone_b = asu_a.lone_sets(asu_b) lone_a.show_array(prefix="lone a: ") print() lone_b.show_array(prefix="lone b: ") print() print("OK")
def miller_set(self, anomalous_flag=True): return miller.set(crystal_symmetry=crystal.symmetry( unit_cell=self.unit_cell, space_group_symbol=self.space_group), indices=self.miller_indices, anomalous_flag=anomalous_flag)
def create_datastructures_for_target_mtz(experiments, mtz_file): """Read a merged mtz file and extract miller indices, intensities and variances.""" m = mtz.object(mtz_file) ind = m.extract_miller_indices() cols = m.columns() col_dict = {c.label(): c for c in cols} r_t = flex.reflection_table() if "I" in col_dict: # nice and simple r_t["miller_index"] = ind r_t["intensity"] = col_dict["I"].extract_values().as_double() r_t["variance"] = flex.pow2( col_dict["SIGI"].extract_values().as_double()) elif "IMEAN" in col_dict: # nice and simple r_t["miller_index"] = ind r_t["intensity"] = col_dict["IMEAN"].extract_values().as_double() r_t["variance"] = flex.pow2( col_dict["SIGIMEAN"].extract_values().as_double()) elif "I(+)" in col_dict: # need to combine I+ and I- together into target Ih if col_dict["I(+)"].n_valid_values() == 0: # use I(-) r_t["miller_index"] = ind r_t["intensity"] = col_dict["I(-)"].extract_values().as_double() r_t["variance"] = flex.pow2( col_dict["SIGI(-)"].extract_values().as_double()) elif col_dict["I(-)"].n_valid_values() == 0: # use I(+) r_t["miller_index"] = ind r_t["intensity"] = col_dict["I(+)"].extract_values().as_double() r_t["variance"] = flex.pow2( col_dict["SIGI(+)"].extract_values().as_double()) else: # Combine both - add together then use Ih table to calculate I and sigma r_tplus = flex.reflection_table() r_tminus = flex.reflection_table() r_tplus["miller_index"] = ind r_tplus["intensity"] = col_dict["I(+)"].extract_values().as_double( ) r_tplus["variance"] = flex.pow2( col_dict["SIGI(+)"].extract_values().as_double()) r_tminus["miller_index"] = ind r_tminus["intensity"] = col_dict["I(-)"].extract_values( ).as_double() r_tminus["variance"] = flex.pow2( col_dict["SIGI(-)"].extract_values().as_double()) r_tplus.extend(r_tminus) r_tplus.set_flags(flex.bool(r_tplus.size(), False), r_tplus.flags.bad_for_scaling) r_tplus = r_tplus.select(r_tplus["variance"] != 0.0) Ih_table = create_Ih_table([experiments[0]], [r_tplus], anomalous=True).blocked_data_list[0] r_t["intensity"] = Ih_table.Ih_values inv_var = Ih_table.sum_in_groups(Ih_table.weights, output="per_refl") r_t["variance"] = 1.0 / inv_var r_t["miller_index"] = Ih_table.miller_index else: raise KeyError( "Unable to find intensities (tried I, IMEAN, I(+)/I(-))") logger.info(f"Extracted {r_t.size()} intensities from target mtz") r_t = r_t.select(r_t["variance"] > 0.0) if r_t.size() == 0: raise ValueError( "No reflections with positive sigma remain after filtering") r_t["d"] = (miller.set( crystal_symmetry=crystal.symmetry( space_group=m.space_group(), unit_cell=m.crystals()[0].unit_cell()), indices=r_t["miller_index"], ).d_spacings().data()) r_t.set_flags(flex.bool(r_t.size(), True), r_t.flags.integrated) exp = Experiment() exp.crystal = deepcopy(experiments[0].crystal) exp.identifier = ersatz_uuid4() r_t.experiment_identifiers()[len(experiments)] = exp.identifier r_t["id"] = flex.int(r_t.size(), len(experiments)) # create a new KB scaling model for the target and set as scaled to fix scale # for targeted scaling. params = Mock() params.KB.decay_correction.return_value = False exp.scaling_model = KBScalingModel.from_data(params, [], []) exp.scaling_model.set_scaling_model_as_scaled( ) # Set as scaled to fix scale. return exp, r_t
bins = [] for i in range(nbins): b0, b1 = min_D2 + i * bin_size, min_D2 + (i + 1) * bin_size print 1 / sqrt(b0), 1 / sqrt(b1) bins.append((b0, b1)) def get_bin_index(h): d = unit_cell.d(h) d2 = 1 / d**2 bin_index = int(floor((d2 - min_D2) / bin_size)) if bin_index == nbins: bin_index = nbins - 1 return bin_index cs = intensities.crystal_symmetry() ms = miller.set(cs, intensities.indices()) ms_asu = ms.map_to_asu() asu_indices = ms_asu.indices() hkl = [] batch = [] intensity = [] for i in range(len(asu_indices)): hkl.append(asu_indices[i]) batch.append(batches.data()[i]) intensity.append(intensities.data()[i]) class Item(object): def __init__(self, h, batch, intensity): self.h = h self.batch = batch
def __init__(self, cif_block, base_array_info=None, wavelengths=None): crystal_symmetry_builder.__init__(self, cif_block) if base_array_info is not None: self.crystal_symmetry = self.crystal_symmetry.join_symmetry( other_symmetry=base_array_info.crystal_symmetry_from_file, force=True) self._arrays = OrderedDict() if (wavelengths is None): wavelengths = {} if base_array_info is None: base_array_info = miller.array_info(source_type="cif") refln_containing_loops = self.get_miller_indices_containing_loops() for self.indices, refln_loop in refln_containing_loops: self.wavelength_id_array = None self.crystal_id_array = None self.scale_group_array = None wavelength_ids = [None] crystal_ids = [None] scale_groups = [None] for key, value in refln_loop.iteritems(): # need to get these arrays first if (key.endswith('wavelength_id') or key.endswith('crystal_id') or key.endswith('scale_group_code')): data = as_int_or_none_if_all_question_marks( value, column_name=key) if data is None: continue counts = data.counts() if key.endswith('wavelength_id'): wavelength_ids = counts.keys() if len(counts) == 1: continue array = miller.array( miller.set(self.crystal_symmetry, self.indices).auto_anomalous(), data) if key.endswith('wavelength_id'): self.wavelength_id_array = array wavelength_ids = counts.keys() elif key.endswith('crystal_id'): self.crystal_id_array = array crystal_ids = counts.keys() elif key.endswith('scale_group_code'): self.scale_group_array = array scale_groups = counts.keys() for label, value in sorted(refln_loop.items()): for w_id in wavelength_ids: for crys_id in crystal_ids: for scale_group in scale_groups: if 'index_' in label: continue key = label labels = [label] wavelength = None if (key.endswith('wavelength_id') or key.endswith('crystal_id') or key.endswith('scale_group_code')): w_id = None crys_id = None scale_group = None key_suffix = '' if w_id is not None: key_suffix += '_%i' % w_id labels.insert(0, "wavelength_id=%i" % w_id) wavelength = wavelengths.get(w_id, None) if crys_id is not None: key_suffix += '_%i' % crys_id labels.insert(0, "crystal_id=%i" % crys_id) if scale_group is not None: key_suffix += '_%i' % scale_group labels.insert( 0, "scale_group_code=%i" % scale_group) key += key_suffix sigmas = None if key in self._arrays: continue array = self.flex_std_string_as_miller_array( value, wavelength_id=w_id, crystal_id=crys_id, scale_group_code=scale_group) if array is None: continue if '_sigma' in key: sigmas_label = label key = None for suffix in ('', '_meas', '_calc'): if sigmas_label.replace( '_sigma', suffix) in refln_loop: key = sigmas_label.replace( '_sigma', suffix) + key_suffix break if key is None: key = sigmas_label + key_suffix elif key in self._arrays and self._arrays[ key].sigmas() is None: sigmas = array array = self._arrays[key] check_array_sizes(array, sigmas, key, sigmas_label) sigmas = as_flex_double( sigmas, sigmas_label) array.set_sigmas(sigmas.data()) info = array.info() array.set_info( info.customized_copy( labels=info.labels + [sigmas_label], wavelength=wavelength)) continue elif 'PHWT' in key: phwt_label = label fwt_label = label.replace('PHWT', 'FWT') if fwt_label not in refln_loop: continue phwt_array = array if fwt_label in self._arrays: array = self._arrays[fwt_label] check_array_sizes(array, phwt_array, fwt_label, phwt_label) phases = as_flex_double( phwt_array, phwt_label) info = array.info() array = array.phase_transfer(phases, deg=True) array.set_info( info.customized_copy( labels=info.labels + [phwt_label])) self._arrays[fwt_label] = array continue elif 'HL_' in key: hl_letter = key[key.find('HL_') + 3] hl_key = 'HL_' + hl_letter key = key.replace(hl_key, 'HL_A') if key in self._arrays: continue # this array is already dealt with hl_labels = [ label.replace(hl_key, 'HL_' + letter) for letter in 'ABCD' ] hl_keys = [ key.replace(hl_key, 'HL_' + letter) for letter in 'ABCD' ] hl_values = [ cif_block.get(hl_key) for hl_key in hl_labels ] if hl_values.count(None) == 0: selection = self.get_selection( hl_values[0], wavelength_id=w_id, crystal_id=crys_id, scale_group_code=scale_group) hl_values = [ as_double_or_none_if_all_question_marks( hl.select(selection), column_name=lab) for hl, lab in zip( hl_values, hl_labels) ] array = miller.array( miller.set( self.crystal_symmetry, self.indices.select( selection)).auto_anomalous(), flex.hendrickson_lattman(*hl_values)) labels = labels[:-1] + hl_labels elif '.B_' in key or '_B_' in key: if '.B_' in key: key, key_b = key.replace('.B_', '.A_'), key label, label_b = label.replace( '.B_', '.A_'), label elif '_B_' in key: key, key_b = key.replace('_B', '_A'), key label, label_b = label.replace('_B', '_A'), label if key in refln_loop and key_b in refln_loop: b_part = array.data() if key in self._arrays: info = self._arrays[key].info() a_part = self._arrays[key].data() self._arrays[key] = self._arrays[ key].array( data=flex.complex_double( a_part, b_part)) self._arrays[key].set_info( info.customized_copy( labels=info.labels + [key_b])) continue elif ('phase_' in key and not "_meas" in key and self.crystal_symmetry.space_group() is not None): alt_key1 = label.replace('phase_', 'F_') alt_key2 = alt_key1 + '_au' if alt_key1 in refln_loop: phase_key = label key = alt_key1 + key_suffix elif alt_key2 in refln_loop: phase_key = label key = alt_key2 + key_suffix else: phase_key = None if phase_key is not None: phases = array.data() if key in self._arrays: array = self._arrays[key] array = as_flex_double(array, key) check_array_sizes( array, phases, key, phase_key) info = self._arrays[key].info() self._arrays[ key] = array.phase_transfer( phases, deg=True) self._arrays[key].set_info( info.customized_copy( labels=info.labels + [phase_key])) else: array = self.flex_std_string_as_miller_array( refln_loop[label], wavelength_id=w_id, crystal_id=crys_id, scale_group_code=scale_group) check_array_sizes( array, phases, key, phase_key) array.phase_transfer(phases, deg=True) labels = labels + [label, phase_key] if base_array_info.labels is not None: labels = base_array_info.labels + labels def rstrip_substrings(string, substrings): for substr in substrings: if substr == '': continue if string.endswith(substr): string = string[:-len(substr)] return string # determine observation type stripped_key = rstrip_substrings( key, [ key_suffix, '_au', '_meas', '_calc', '_plus', '_minus' ]) if (stripped_key.endswith('F_squared') or stripped_key.endswith('intensity') or stripped_key.endswith('.I') or stripped_key.endswith('_I')) and ( array.is_real_array() or array.is_integer_array()): array.set_observation_type_xray_intensity() elif (stripped_key.endswith('F') and (array.is_real_array() or array.is_integer_array())): array.set_observation_type_xray_amplitude() if (array.is_xray_amplitude_array() or array.is_xray_amplitude_array()): # e.g. merge_equivalents treats integer arrays differently, so must # convert integer observation arrays here to be safe if isinstance(array.data(), flex.int): array = array.customized_copy( data=array.data().as_double()) array.set_info( base_array_info.customized_copy(labels=labels)) if (array.is_xray_amplitude_array() or array.is_xray_amplitude_array()): info = array.info() array.set_info( info.customized_copy( wavelength=wavelength)) self._arrays.setdefault(key, array) for key, array in self._arrays.copy().iteritems(): if (key.endswith('_minus') or '_minus_' in key or key.endswith('_plus') or '_plus_' in key): if '_minus' in key: minus_key = key plus_key = key.replace('_minus', '_plus') elif '_plus' in key: plus_key = key minus_key = key.replace('_plus', '_minus') if plus_key in self._arrays and minus_key in self._arrays: plus_array = self._arrays.pop(plus_key) minus_array = self._arrays.pop(minus_key) minus_array = minus_array.customized_copy( indices=-minus_array.indices()).set_info( minus_array.info()) array = plus_array.concatenate( minus_array, assert_is_similar_symmetry=False) array = array.customized_copy(anomalous_flag=True) array.set_info( minus_array.info().customized_copy(labels=list( OrderedSet(plus_array.info().labels + minus_array.info().labels)))) array.set_observation_type(plus_array.observation_type()) self._arrays.setdefault(key, array) if len(self._arrays) == 0: raise CifBuilderError("No reflection data present in cif block")
def quasi_normalisation(reflection_table, experiment): """Calculate normalised intensity (Esq) values for reflections, for the purpose of selecting subsets based on Esq for scaling. If more involved analyses of normalised intensities are needed, then it may be necessary to split this procedure to handle acentric and centric reflections separately.""" logger.info( "Calculating normalised intensity values to select a reflection \n" "subset for scaling. \n") logger.debug( "Negative intensities are set to zero for the purpose of \n" "calculating mean intensity values for resolution bins. This is to avoid \n" "spuriously high E^2 values due to a mean close to zero and should only \n" "affect the E^2 values of the highest resolution bins. \n") good_refl_sel = ~reflection_table.get_flags( reflection_table.flags.bad_for_scaling, all=False) rt_subset = reflection_table.select(good_refl_sel) # Scaling subset is data that has not been flagged as bad or excluded miller_set = miller.set( crystal_symmetry=experiment.crystal.get_crystal_symmetry(), indices=rt_subset["miller_index"], ) # handle negative reflections to minimise effect on mean I values. rt_subset["intensity_for_norm"] = copy.deepcopy(rt_subset["intensity"]) rt_subset["intensity_for_norm"].set_selected(rt_subset["intensity"] < 0.0, 0.0) miller_array = miller.array(miller_set, data=rt_subset["intensity_for_norm"]) n_refl = rt_subset.size() # set up binning objects if n_refl > 20000: n_refl_shells = 20 elif n_refl > 15000: n_refl_shells = 15 elif n_refl > 10000: n_refl_shells = 10 else: logger.info( "No normalised intensity values were calculated, as an insufficient\n" "number of reflections were detected. All normalised intensity \n" "values will be set to 1 to allow use in scaling model determination. \n" ) reflection_table["Esq"] = flex.double(reflection_table.size(), 1.0) return reflection_table d_star_sq = miller_array.d_star_sq().data() d_star_sq_min = flex.min(d_star_sq) d_star_sq_max = flex.max(d_star_sq) span = d_star_sq_max - d_star_sq_min relative_tolerance = 1e-6 d_star_sq_max += span * relative_tolerance d_star_sq_min -= span * relative_tolerance step = (d_star_sq_max - d_star_sq_min) / n_refl_shells miller_array.setup_binner_d_star_sq_step( auto_binning=False, d_max=uctbx.d_star_sq_as_d(d_star_sq_max), d_min=uctbx.d_star_sq_as_d(d_star_sq_min), d_star_sq_step=step, ) normalisations = miller_array.intensity_quasi_normalisations() normalised_intensities = miller_array.customized_copy( data=(miller_array.data() / normalisations.data())) reflection_table["Esq"] = flex.double(reflection_table.size(), 0.0) reflection_table["Esq"].set_selected(good_refl_sel, normalised_intensities.data()) return reflection_table
def process_input_array(self): from cctbx.array_family import flex array = self.miller_array.deep_copy() multiplicities = None if self.merge_equivalents: if self.settings.show_anomalous_pairs: from cctbx import miller merge = array.merge_equivalents() multiplicities = merge.redundancies() asu, matches = multiplicities.match_bijvoet_mates() mult_plus, mult_minus = multiplicities.hemispheres_acentrics() anom_mult = flex.int( min(p, m) for (p, m) in zip(mult_plus.data(), mult_minus.data())) #flex.min_max_mean_double(anom_mult.as_double()).show() anomalous_multiplicities = miller.array( miller.set(asu.crystal_symmetry(), mult_plus.indices(), anomalous_flag=False), anom_mult) anomalous_multiplicities = anomalous_multiplicities.select( anomalous_multiplicities.data() > 0) array = anomalous_multiplicities multiplicities = anomalous_multiplicities else: merge = array.merge_equivalents() array = merge.array() multiplicities = merge.redundancies() settings = self.settings data = array.data() self.missing_set = oop.null() if (array.is_xray_intensity_array()): data.set_selected(data < 0, flex.double(data.size(), 0.)) if (array.is_unique_set_under_symmetry()) and (settings.map_to_asu): array = array.map_to_asu() if (multiplicities is not None): multiplicities = multiplicities.map_to_asu() if (settings.d_min is not None): array = array.resolution_filter(d_min=settings.d_min) if (multiplicities is not None): multiplicities = multiplicities.resolution_filter( d_min=settings.d_min) self.filtered_array = array.deep_copy() if (settings.expand_anomalous): array = array.generate_bijvoet_mates() if (multiplicities is not None): multiplicities = multiplicities.generate_bijvoet_mates() if (self.settings.show_missing): self.missing_set = array.complete_set().lone_set(array) if self.settings.show_anomalous_pairs: self.missing_set = self.missing_set.select( self.missing_set.centric_flags().data(), negate=True) if (settings.expand_to_p1): original_symmetry = array.crystal_symmetry() array = array.expand_to_p1().customized_copy( crystal_symmetry=original_symmetry) #array = array.niggli_cell().expand_to_p1() #self.missing_set = self.missing_set.niggli_cell().expand_to_p1() self.missing_set = self.missing_set.expand_to_p1().customized_copy( crystal_symmetry=original_symmetry) if (multiplicities is not None): multiplicities = multiplicities.expand_to_p1().customized_copy( crystal_symmetry=original_symmetry) data = array.data() self.r_free_mode = False if isinstance(data, flex.bool): self.r_free_mode = True data_as_float = flex.double(data.size(), 0.0) data_as_float.set_selected(data == True, flex.double(data.size(), 1.0)) data = data_as_float self.data = data.deep_copy() else: if isinstance(data, flex.double): self.data = data.deep_copy() elif isinstance(data, flex.complex_double): self.data = flex.abs(data) elif hasattr(array.data(), "as_double"): self.data = array.data().as_double() else: raise RuntimeError("Unexpected data type: %r" % data) if (settings.show_data_over_sigma): if (array.sigmas() is None): raise Sorry("sigmas not defined.") sigmas = array.sigmas() non_zero_sel = sigmas != 0 array = array.select(non_zero_sel) array = array.customized_copy(data=array.data() / array.sigmas()) self.data = array.data() if (multiplicities is not None): multiplicities = multiplicities.select(non_zero_sel) self.work_array = array self.multiplicities = multiplicities
def run(self, experiments, reflections): self.logger.log_step_time("POSTREFINEMENT") if not self.params.postrefinement.enable: self.logger.log("Postrefinement was not done") if self.mpi_helper.rank == 0: self.logger.main_log("Postrefinement was not done") return experiments, reflections target_symm = symmetry( unit_cell=self.params.scaling.unit_cell, space_group_info=self.params.scaling.space_group) i_model = self.params.scaling.i_model miller_set = self.params.scaling.miller_set # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. assert len(i_model.indices()) == len(miller_set.indices()) assert (i_model.indices() == miller_set.indices()).count(False) == 0 new_experiments = ExperimentList() new_reflections = flex.reflection_table() experiments_rejected_by_reason = {} # reason:how_many_rejected for experiment in experiments: exp_reflections = reflections.select( reflections['exp_id'] == experiment.identifier) # Build a miller array for the experiment reflections with original miller indexes exp_miller_indices_original = miller.set( target_symm, exp_reflections['miller_index'], True) observations_original_index = miller.array( exp_miller_indices_original, exp_reflections['intensity.sum.value'], flex.double( flex.sqrt(exp_reflections['intensity.sum.variance']))) assert exp_reflections.size() == exp_miller_indices_original.size() assert observations_original_index.size( ) == exp_miller_indices_original.size() # Build a miller array for the experiment reflections with asu miller indexes exp_miller_indices_asu = miller.set( target_symm, exp_reflections['miller_index_asymmetric'], True) observations = miller.array( exp_miller_indices_asu, exp_reflections['intensity.sum.value'], flex.double( flex.sqrt(exp_reflections['intensity.sum.variance']))) matches = miller.match_multi_indices( miller_indices_unique=miller_set.indices(), miller_indices=observations.indices()) pair1 = flex.int([pair[1] for pair in matches.pairs() ]) # refers to the observations pair0 = flex.int([pair[0] for pair in matches.pairs() ]) # refers to the model assert exp_reflections.size() == exp_miller_indices_original.size() assert observations_original_index.size( ) == exp_miller_indices_original.size() # narrow things down to the set that matches, only observations_pair1_selected = observations.customized_copy( indices=flex.miller_index( [observations.indices()[p] for p in pair1]), data=flex.double([observations.data()[p] for p in pair1]), sigmas=flex.double([observations.sigmas()[p] for p in pair1])) observations_original_index_pair1_selected = observations_original_index.customized_copy( indices=flex.miller_index( [observations_original_index.indices()[p] for p in pair1]), data=flex.double( [observations_original_index.data()[p] for p in pair1]), sigmas=flex.double( [observations_original_index.sigmas()[p] for p in pair1])) I_observed = observations_pair1_selected.data() MILLER = observations_original_index_pair1_selected.indices() ORI = crystal_orientation(experiment.crystal.get_A(), basis_type.reciprocal) Astar = matrix.sqr(ORI.reciprocal_matrix()) Astar_from_experiment = matrix.sqr(experiment.crystal.get_A()) assert Astar == Astar_from_experiment WAVE = experiment.beam.get_wavelength() BEAM = matrix.col((0.0, 0.0, -1. / WAVE)) BFACTOR = 0. MOSAICITY_DEG = experiment.crystal.get_half_mosaicity_deg() DOMAIN_SIZE_A = experiment.crystal.get_domain_size_ang() # calculation of correlation here I_reference = flex.double( [i_model.data()[pair[0]] for pair in matches.pairs()]) I_invalid = flex.bool( [i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()]) use_weights = False # New facility for getting variance-weighted correlation if use_weights: # variance weighting I_weight = flex.double([ 1. / (observations_pair1_selected.sigmas()[pair[1]])**2 for pair in matches.pairs() ]) else: I_weight = flex.double( len(observations_pair1_selected.sigmas()), 1.) I_weight.set_selected(I_invalid, 0.) """Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement: include_negatives = True + and - reflections both used for Rh distribution for initial estimate of RS parameter + and - reflections both used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) include_negatives = False + and - reflections both used for Rh distribution for initial estimate of RS parameter + reflections only used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) """ # RB: By design, for MPI-Merge "include negatives" is implicitly True SWC = simple_weighted_correlation(I_weight, I_reference, I_observed) if self.params.output.log_level == 0: self.logger.log("Old correlation is: %f" % SWC.corr) if self.params.postrefinement.algorithm == "rs": Rhall = flex.double() for mill in MILLER: H = matrix.col(mill) Xhkl = Astar * H Rh = (Xhkl + BEAM).length() - (1. / WAVE) Rhall.append(Rh) Rs = math.sqrt(flex.mean(Rhall * Rhall)) RS = 1. / 10000. # reciprocal effective domain size of 1 micron RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.]) parameterization_class = rs_parameterization refinery = rs_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) elif self.params.postrefinement.algorithm == "eta_deff": eta_init = 2. * MOSAICITY_DEG * math.pi / 180. D_eff_init = 2. * DOMAIN_SIZE_A current = flex.double( [SWC.slope, BFACTOR, eta_init, 0., 0., D_eff_init]) parameterization_class = eta_deff_parameterization refinery = eta_deff_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) func = refinery.fvec_callable(parameterization_class(current)) functional = flex.sum(func * func) if self.params.output.log_level == 0: self.logger.log("functional: %f" % functional) self.current = current self.parameterization_class = parameterization_class self.refinery = refinery self.observations_pair1_selected = observations_pair1_selected self.observations_original_index_pair1_selected = observations_original_index_pair1_selected error_detected = False try: self.run_plain() result_observations_original_index, result_observations, result_matches = self.result_for_cxi_merge( ) assert result_observations_original_index.size( ) == result_observations.size() assert result_matches.pairs().size( ) == result_observations_original_index.size() except (AssertionError, ValueError, RuntimeError) as e: error_detected = True reason = repr(e) if not reason: reason = "Unknown error" if not reason in experiments_rejected_by_reason: experiments_rejected_by_reason[reason] = 1 else: experiments_rejected_by_reason[reason] += 1 if not error_detected: new_experiments.append(experiment) new_exp_reflections = flex.reflection_table() new_exp_reflections[ 'miller_index_asymmetric'] = flex.miller_index( result_observations.indices()) new_exp_reflections['intensity.sum.value'] = flex.double( result_observations.data()) new_exp_reflections['intensity.sum.variance'] = flex.double( flex.pow(result_observations.sigmas(), 2)) new_exp_reflections['exp_id'] = flex.std_string( len(new_exp_reflections), experiment.identifier) new_reflections.extend(new_exp_reflections) ''' # debugging elif reason.startswith("ValueError"): self.logger.log("Rejected b/c of value error exp id: %s; unit cell: %s"%(exp_id, str(experiment.crystal.get_unit_cell())) ) ''' # report rejected experiments, reflections experiments_rejected_by_postrefinement = len(experiments) - len( new_experiments) reflections_rejected_by_postrefinement = reflections.size( ) - new_reflections.size() self.logger.log("Experiments rejected by post-refinement: %d" % experiments_rejected_by_postrefinement) self.logger.log("Reflections rejected by post-refinement: %d" % reflections_rejected_by_postrefinement) all_reasons = [] for reason, count in experiments_rejected_by_reason.iteritems(): self.logger.log("Experiments rejected due to %s: %d" % (reason, count)) all_reasons.append(reason) comm = self.mpi_helper.comm MPI = self.mpi_helper.MPI # Collect all rejection reasons from all ranks. Use allreduce to let each rank have all reasons. all_reasons = comm.allreduce(all_reasons, MPI.SUM) all_reasons = set(all_reasons) # Now that each rank has all reasons from all ranks, we can treat the reasons in a uniform way. total_experiments_rejected_by_reason = {} for reason in all_reasons: rejected_experiment_count = 0 if reason in experiments_rejected_by_reason: rejected_experiment_count = experiments_rejected_by_reason[ reason] total_experiments_rejected_by_reason[reason] = comm.reduce( rejected_experiment_count, MPI.SUM, 0) total_accepted_experiment_count = comm.reduce(len(new_experiments), MPI.SUM, 0) if self.mpi_helper.rank == 0: for reason, count in total_experiments_rejected_by_reason.iteritems( ): self.logger.main_log( "Total experiments rejected due to %s: %d" % (reason, count)) self.logger.main_log("Total experiments accepted: %d" % total_accepted_experiment_count) self.logger.log_step_time("POSTREFINEMENT", True) return new_experiments, new_reflections
def organize_input(self, observations_pickle, iparams, avg_mode, pickle_filename=None): """Given the pickle file, extract and prepare observations object and the alpha angle (meridional to equatorial). """ #get general parameters if iparams.isoform_name is not None: if "identified_isoform" not in observations_pickle: return None, "No identified isoform" if observations_pickle[ "identified_isoform"] != iparams.isoform_name: return None, "Identified isoform(%s) is not the requested isoform (%s)" % ( observations_pickle["identified_isoform"], iparams.isoform_name) if iparams.flag_weak_anomalous: if avg_mode == 'final': target_anomalous_flag = iparams.target_anomalous_flag else: target_anomalous_flag = False else: target_anomalous_flag = iparams.target_anomalous_flag img_filename_only = '' if pickle_filename: img_filename_only = os.path.basename(pickle_filename) txt_exception = ' {0:40} ==> '.format(img_filename_only) #for dials integration pickles - also look for experimentxxx.json if "miller_index" in observations_pickle: from dxtbx.model.experiment_list import ExperimentListFactory exp_json_file = os.path.join( os.path.dirname(pickle_filename), img_filename_only.split('_')[0] + '_refined_experiments.json') if os.path.isfile(exp_json_file): experiments = ExperimentListFactory.from_json_file( exp_json_file) dials_crystal = experiments[0].crystal detector = experiments[0].detector beam = experiments[0].beam crystal_symmetry = crystal.symmetry( unit_cell=dials_crystal.get_unit_cell().parameters(), space_group_symbol=iparams.target_space_group) miller_set_all = miller.set( crystal_symmetry=crystal_symmetry, indices=observations_pickle['miller_index'], anomalous_flag=target_anomalous_flag) observations = miller_set_all.array( data=observations_pickle['intensity.sum.value'], sigmas=flex.sqrt( observations_pickle['intensity.sum.variance']) ).set_observation_type_xray_intensity() detector_distance_mm = detector[0].get_distance() alpha_angle_obs = flex.double([0] * len(observations.data())) wavelength = beam.get_wavelength() spot_pred_x_mm = observations_pickle['s1'] #a disguise of s1 spot_pred_y_mm = flex.double([0] * len(observations.data())) #calculate the crystal orientation O = sqr(dials_crystal.get_unit_cell().orthogonalization_matrix( )).transpose() R = sqr(dials_crystal.get_U()).transpose() from cctbx.crystal_orientation import crystal_orientation, basis_type crystal_init_orientation = crystal_orientation( O * R, basis_type.direct) else: txt_exception += exp_json_file + ' not found' print(txt_exception) return None, txt_exception else: #for cctbx.xfel proceed as usual observations = observations_pickle["observations"][0] detector_distance_mm = observations_pickle['distance'] mm_predictions = iparams.pixel_size_mm * ( observations_pickle['mapped_predictions'][0]) xbeam = observations_pickle["xbeam"] ybeam = observations_pickle["ybeam"] alpha_angle_obs = flex.double([math.atan(abs(pred[0]-xbeam)/abs(pred[1]-ybeam)) \ for pred in mm_predictions]) spot_pred_x_mm = flex.double( [pred[0] - xbeam for pred in mm_predictions]) spot_pred_y_mm = flex.double( [pred[1] - ybeam for pred in mm_predictions]) #Polarization correction wavelength = observations_pickle["wavelength"] crystal_init_orientation = observations_pickle[ "current_orientation"][0] #continue reading... if iparams.flag_LP_correction and "observations" in observations_pickle: fx = 1 - iparams.polarization_horizontal_fraction fy = 1 - fx if fx > 1.0 or fx < 0: print( 'Horizontal polarization fraction is not correct. The value must be >= 0 and <= 1' ) print( 'No polarization correction. Continue with post-refinement' ) else: phi_angle_obs = flex.double([math.atan2(pred[1]-ybeam, pred[0]-xbeam) \ for pred in mm_predictions]) bragg_angle_obs = observations.two_theta(wavelength).data() P = ((fx*((flex.sin(phi_angle_obs)**2)+((flex.cos(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2)))+\ (fy*((flex.cos(phi_angle_obs)**2)+((flex.sin(phi_angle_obs)**2)*flex.cos(bragg_angle_obs)**2)))) I_prime = observations.data() / P sigI_prime = observations.sigmas() / P observations = observations.customized_copy( data=flex.double(I_prime), sigmas=flex.double(sigI_prime)) #set observations with target space group - !!! required for correct #merging due to map_to_asu command. if iparams.target_crystal_system is not None: target_crystal_system = iparams.target_crystal_system else: target_crystal_system = observations.crystal_symmetry( ).space_group().crystal_system() lph = lbfgs_partiality_handler() if iparams.flag_override_unit_cell: uc_constrained_inp = lph.prep_input( iparams.target_unit_cell.parameters(), target_crystal_system) else: uc_constrained_inp = lph.prep_input( observations.unit_cell().parameters(), target_crystal_system) uc_constrained = list( lph.prep_output(uc_constrained_inp, target_crystal_system)) try: #apply constrain using the crystal system miller_set = symmetry(unit_cell=uc_constrained, space_group_symbol=iparams.target_space_group ).build_miller_set( anomalous_flag=target_anomalous_flag, d_min=iparams.merge.d_min) observations = observations.customized_copy( anomalous_flag=target_anomalous_flag, crystal_symmetry=miller_set.crystal_symmetry()) except Exception: a, b, c, alpha, beta, gamma = uc_constrained txt_exception += 'Mismatch spacegroup (%6.2f,%6.2f,%6.2f,%6.2f,%6.2f,%6.2f)' % ( a, b, c, alpha, beta, gamma) print(txt_exception) return None, txt_exception #reset systematic absence sys_absent_negate_flags = flex.bool([ sys_absent_flag[1] == False for sys_absent_flag in observations.sys_absent_flags() ]) observations = observations.select(sys_absent_negate_flags) alpha_angle_obs = alpha_angle_obs.select(sys_absent_negate_flags) spot_pred_x_mm = spot_pred_x_mm.select(sys_absent_negate_flags) spot_pred_y_mm = spot_pred_y_mm.select(sys_absent_negate_flags) #remove observations from rejection list if iparams.rejections: if pickle_filename in iparams.rejections: miller_indices_ori_rejected = iparams.rejections[ pickle_filename] i_sel_flag = flex.bool([True] * len(observations.data())) cnrej = 0 for miller_index_ori_rejected in miller_indices_ori_rejected: for i_index_ori, miller_index_ori in enumerate( observations.indices()): if miller_index_ori_rejected == miller_index_ori: i_sel_flag[i_index_ori] = False cnrej += 1 observations = observations.customized_copy( indices=observations.indices().select(i_sel_flag), data=observations.data().select(i_sel_flag), sigmas=observations.sigmas().select(i_sel_flag)) alpha_angle_obs = alpha_angle_obs.select(i_sel_flag) spot_pred_x_mm = spot_pred_x_mm.select(i_sel_flag) spot_pred_y_mm = spot_pred_y_mm.select(i_sel_flag) #filter resolution i_sel_res = observations.resolution_filter_selection( d_max=iparams.merge.d_max, d_min=iparams.merge.d_min) observations = observations.select(i_sel_res) alpha_angle_obs = alpha_angle_obs.select(i_sel_res) spot_pred_x_mm = spot_pred_x_mm.select(i_sel_res) spot_pred_y_mm = spot_pred_y_mm.select(i_sel_res) #Filter weak i_sel = (observations.data() / observations.sigmas()) > iparams.merge.sigma_min observations = observations.select(i_sel) alpha_angle_obs = alpha_angle_obs.select(i_sel) spot_pred_x_mm = spot_pred_x_mm.select(i_sel) spot_pred_y_mm = spot_pred_y_mm.select(i_sel) #filter icering (if on) if iparams.icering.flag_on: miller_indices = flex.miller_index() I_set = flex.double() sigI_set = flex.double() alpha_angle_obs_set = flex.double() spot_pred_x_mm_set = flex.double() spot_pred_y_mm_set = flex.double() for miller_index, d, I, sigI, alpha, spot_x, spot_y in zip( observations.indices(), observations.d_spacings().data(), observations.data(), observations.sigmas(), alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm): if d > iparams.icering.d_upper or d < iparams.icering.d_lower: miller_indices.append(miller_index) I_set.append(I) sigI_set.append(sigI) alpha_angle_obs_set.append(alpha) spot_pred_x_mm_set.append(spot_x) spot_pred_y_mm_set.append(spot_y) observations = observations.customized_copy(indices=miller_indices, data=I_set, sigmas=sigI_set) alpha_angle_obs = alpha_angle_obs_set[:] spot_pred_x_mm = spot_pred_x_mm_set[:] spot_pred_y_mm = spot_pred_y_mm_set[:] #replacing sigI (if set) if iparams.flag_replace_sigI: observations = observations.customized_copy( sigmas=flex.sqrt(observations.data())) inputs = observations, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, wavelength, crystal_init_orientation return inputs, 'OK'
def refine_subgroup(args): assert len(args) == 4 from dials.command_line.check_indexing_symmetry import ( get_symop_correlation_coefficients, normalise_intensities, ) params, subgroup, used_reflections, experiments = args used_reflections = copy.deepcopy(used_reflections) triclinic_miller = used_reflections["miller_index"] cb_op = subgroup["cb_op_inp_best"] higher_symmetry_miller = cb_op.apply(triclinic_miller) used_reflections["miller_index"] = higher_symmetry_miller unrefined_crystal = copy.deepcopy(subgroup.unrefined_crystal) for expt in experiments: expt.crystal = unrefined_crystal from dials.algorithms.indexing.refinement import refine subgroup.max_cc = None subgroup.min_cc = None subgroup.correlation_coefficients = [] subgroup.cc_nrefs = [] with LoggingContext(logging.getLogger(), level=logging.ERROR): try: outlier_algorithm = params.refinement.reflections.outlier.algorithm sel = used_reflections.get_flags( used_reflections.flags.used_in_refinement) if sel.all_eq(False): # Soft outlier rejection if no used_in_refinement flag is set params.refinement.reflections.outlier.algorithm = "tukey" iqr_multiplier = ( params.refinement.reflections.outlier.tukey.iqr_multiplier) params.refinement.reflections.outlier.tukey.iqr_multiplier = ( 2 * iqr_multiplier) sel = ~sel else: # Remove reflections not previously used in refinement params.refinement.reflections.outlier.algorithm = "null" refinery, refined, outliers = refine(params, used_reflections.select(sel), experiments) params.refinement.reflections.outlier.algorithm = outlier_algorithm refinery, refined, outliers = refine(params, used_reflections, refinery.get_experiments()) except RuntimeError as e: if (str(e) == "scitbx Error: g0 - astry*astry -astrz*astrz <= 0." or str(e) == "scitbx Error: g1-bstrz*bstrz <= 0."): subgroup.refined_experiments = None subgroup.rmsd = None subgroup.Nmatches = None else: raise else: dall = refinery.rmsds() dx = dall[0] dy = dall[1] subgroup.rmsd = math.sqrt(dx * dx + dy * dy) subgroup.Nmatches = len(refinery.get_matches()) subgroup.refined_experiments = refinery.get_experiments() assert len(subgroup.refined_experiments.crystals()) == 1 subgroup.refined_crystal = subgroup.refined_experiments.crystals( )[0] cs = crystal.symmetry( unit_cell=subgroup.refined_crystal.get_unit_cell(), space_group=subgroup.refined_crystal.get_space_group(), ) if "intensity.sum.value" in used_reflections: # remove refl with -ve variance sel = used_reflections["intensity.sum.variance"] > 0 good_reflections = used_reflections.select(sel) from cctbx import miller ms = miller.set(cs, good_reflections["miller_index"]) ms = ms.array( good_reflections["intensity.sum.value"] / flex.sqrt(good_reflections["intensity.sum.variance"])) if params.normalise: if params.normalise_bins: ms = normalise_intensities( ms, n_bins=params.normalise_bins) else: ms = normalise_intensities(ms) if params.cc_n_bins is not None: ms.setup_binner(n_bins=params.cc_n_bins) ccs, nrefs = get_symop_correlation_coefficients( ms, use_binning=(params.cc_n_bins is not None)) subgroup.correlation_coefficients = ccs subgroup.cc_nrefs = nrefs ccs = ccs.select(nrefs > 10) if len(ccs) > 1: subgroup.max_cc = flex.max(ccs[1:]) subgroup.min_cc = flex.min(ccs[1:]) return subgroup
def prepare_output(self, results, iparams, avg_mode): if avg_mode == 'average': cc_thres = 0 else: cc_thres = iparams.frame_accept_min_cc std_filter = iparams.sigma_rejection if iparams.flag_weak_anomalous: if avg_mode == 'final': target_anomalous_flag = iparams.target_anomalous_flag else: target_anomalous_flag = False else: target_anomalous_flag = iparams.target_anomalous_flag pr_params_mean, pr_params_med, pr_params_std = self.calc_mean_postref_parameters( results) G_mean, B_mean, ry_mean, rz_mean, re_mean, r0_mean, voigt_nu_mean, rotx_mean, roty_mean, R_mean, R_xy_mean, SE_mean = pr_params_mean G_med, B_med, ry_med, rz_med, re_med, r0_med, voigt_nu_med, rotx_med, roty_med, R_med, R_xy_med, SE_med = pr_params_med G_std, B_std, ry_std, rz_std, re_std, r0_std, voigt_nu_std, rotx_std, roty_std, R_std, R_xy_std, SE_std = pr_params_std #prepare data for merging miller_indices_all = flex.miller_index() miller_indices_ori_all = flex.miller_index() I_all = flex.double() sigI_all = flex.double() G_all = flex.double() B_all = flex.double() p_all = flex.double() rx_all = flex.double() rs_all = flex.double() rh_all = flex.double() SE_all = flex.double() sin_sq_all = flex.double() wavelength_all = flex.double() detector_distance_set = flex.double() R_init_all = flex.double() R_final_all = flex.double() R_xy_init_all = flex.double() R_xy_final_all = flex.double() pickle_filename_all = flex.std_string() filtered_results = [] cn_good_frame, cn_bad_frame_SE, cn_bad_frame_uc, cn_bad_frame_cc, cn_bad_frame_G, cn_bad_frame_re = ( 0, 0, 0, 0, 0, 0) crystal_orientation_dict = {} for pres in results: if pres is not None: pickle_filepath = pres.pickle_filename.split('/') img_filename = pickle_filepath[len(pickle_filepath) - 1] flag_pres_ok = True #check SE, CC, UC, G, B, gamma_e if math.isnan(pres.G): flag_pres_ok = False if math.isnan(pres.SE) or np.isinf(pres.SE): flag_pres_ok = False if flag_pres_ok and SE_std > 0: if abs(pres.SE - SE_med) / SE_std > std_filter: flag_pres_ok = False cn_bad_frame_SE += 1 if flag_pres_ok and pres.CC_final < cc_thres: flag_pres_ok = False cn_bad_frame_cc += 1 if flag_pres_ok: if G_std > 0: if abs(pres.G - G_med) / G_std > std_filter: flag_pres_ok = False cn_bad_frame_G += 1 if flag_pres_ok: if re_std > 0: if abs(pres.re - re_med) / re_std > std_filter: flag_pres_ok = False cn_bad_frame_re += 1 if flag_pres_ok and not good_unit_cell( pres.uc_params, iparams, iparams.merge.uc_tolerance): flag_pres_ok = False cn_bad_frame_uc += 1 data_size = pres.observations.size() if flag_pres_ok: cn_good_frame += 1 filtered_results.append(pres) R_init_all.append(pres.R_init) R_final_all.append(pres.R_final) R_xy_init_all.append(pres.R_xy_init) R_xy_final_all.append(pres.R_xy_final) miller_indices_all.extend(pres.observations.indices()) miller_indices_ori_all.extend( pres.observations_original.indices()) I_all.extend(pres.observations.data()) sigI_all.extend(pres.observations.sigmas()) G_all.extend(flex.double([pres.G] * data_size)) B_all.extend(flex.double([pres.B] * data_size)) p_all.extend(pres.partiality) rs_all.extend(pres.rs_set) rh_all.extend(pres.rh_set) sin_sq_all.extend( pres.observations.two_theta(wavelength=pres.wavelength) .sin_theta_over_lambda_sq().data()) SE_all.extend(flex.double([pres.SE] * data_size)) wavelength_all.extend( flex.double([pres.wavelength] * data_size)) detector_distance_set.append(pres.detector_distance_mm) pickle_filename_all.extend( flex.std_string([pres.pickle_filename] * data_size)) crystal_orientation_dict[ pres.pickle_filename] = pres.crystal_orientation #plot stats self.plot_stats(filtered_results, iparams) #write out updated crystal orientation as a pickle file if not iparams.flag_hush: pickle.dump(crystal_orientation_dict, open(iparams.run_no + '/' + "crystal.o", "wb"), pickle.HIGHEST_PROTOCOL) #calculate average unit cell uc_mean, uc_med, uc_std = self.calc_mean_unit_cell(filtered_results) unit_cell_mean = unit_cell(tuple(uc_mean)) #recalculate stats for pr parameters pr_params_mean, pr_params_med, pr_params_std = self.calc_mean_postref_parameters( filtered_results) G_mean, B_mean, ry_mean, rz_mean, re_mean, r0_mean, voigt_nu_mean, rotx_mean, roty_mean, R_mean, R_xy_mean, SE_mean = pr_params_mean G_med, B_med, ry_med, rz_med, re_med, r0_med, voigt_nu_med, rotx_med, roty_med, R_med, R_xy_med, SE_med = pr_params_med G_std, B_std, ry_std, rz_std, re_std, r0_std, voigt_nu_std, rotx_std, roty_std, R_std, R_xy_std, SE_std = pr_params_std #from all observations merge them crystal_symmetry = crystal.symmetry(unit_cell=tuple(uc_mean), space_group_symbol=str( iparams.target_space_group)) miller_set_all = miller.set(crystal_symmetry=crystal_symmetry, indices=miller_indices_all, anomalous_flag=target_anomalous_flag) miller_array_all = miller_set_all.array( data=I_all, sigmas=sigI_all).set_observation_type_xray_intensity() #sort reflections according to asymmetric-unit symmetry hkl perm = miller_array_all.sort_permutation(by_value="packed_indices") miller_indices_all_sort = miller_array_all.indices().select(perm) miller_indices_ori_all_sort = miller_indices_ori_all.select(perm) I_obs_all_sort = miller_array_all.data().select(perm) sigI_obs_all_sort = miller_array_all.sigmas().select(perm) G_all_sort = G_all.select(perm) B_all_sort = B_all.select(perm) p_all_sort = p_all.select(perm) rs_all_sort = rs_all.select(perm) wavelength_all_sort = wavelength_all.select(perm) sin_sq_all_sort = sin_sq_all.select(perm) SE_all_sort = SE_all.select(perm) pickle_filename_all_sort = pickle_filename_all.select(perm) miller_array_uniq = miller_array_all.merge_equivalents().array( ).complete_array(d_min=iparams.merge.d_min, d_max=iparams.merge.d_max) matches_uniq = miller.match_multi_indices( miller_indices_unique=miller_array_uniq.indices(), miller_indices=miller_indices_all_sort) pair_0 = flex.int([pair[0] for pair in matches_uniq.pairs()]) pair_1 = flex.int([pair[1] for pair in matches_uniq.pairs()]) group_id_list = flex.int( [pair_0[pair_1[i]] for i in range(len(matches_uniq.pairs()))]) tally = Counter() for elem in group_id_list: tally[elem] += 1 cn_group = len(tally) #preparte txt out stat txt_out = 'Summary of refinement and merging\n' txt_out += ' No. good frames: %12.0f\n' % (cn_good_frame) txt_out += ' No. bad cc frames: %12.0f\n' % (cn_bad_frame_cc) txt_out += ' No. bad G frames) : %12.0f\n' % (cn_bad_frame_G) txt_out += ' No. bad unit cell frames: %12.0f\n' % (cn_bad_frame_uc) txt_out += ' No. bad gamma_e frames: %12.0f\n' % (cn_bad_frame_re) txt_out += ' No. bad SE: %12.0f\n' % (cn_bad_frame_SE) txt_out += ' No. observations: %12.0f\n' % ( len(I_obs_all_sort)) txt_out += 'Mean target value (BEFORE: Mean Median (Std.))\n' txt_out += ' post-refinement: %12.2f %12.2f (%9.2f)\n' % ( np.mean(R_init_all), np.median(R_init_all), np.std(R_init_all)) txt_out += ' (x,y) restraints: %12.2f %12.2f (%9.2f)\n' % ( np.mean(R_xy_init_all), np.median(R_xy_init_all), np.std(R_xy_init_all)) txt_out += 'Mean target value (AFTER: Mean Median (Std.))\n' txt_out += ' post-refinement: %12.2f %12.2f (%9.2f)\n' % ( np.mean(R_final_all), np.median(R_final_all), np.std(R_final_all)) txt_out += ' (x,y) restraints: %12.2f %12.2f (%9.2f)\n' % ( np.mean(R_xy_final_all), np.median(R_xy_final_all), np.std(R_xy_final_all)) txt_out += ' SE: %12.2f %12.2f (%9.2f)\n' % ( SE_mean, SE_med, SE_std) txt_out += ' G: %12.3e %12.3e (%9.2e)\n' % ( G_mean, G_med, G_std) txt_out += ' B: %12.2f %12.2f (%9.2f)\n' % ( B_mean, B_med, B_std) txt_out += ' Rot.x: %12.2f %12.2f (%9.2f)\n' % ( rotx_mean * 180 / math.pi, rotx_med * 180 / math.pi, rotx_std * 180 / math.pi) txt_out += ' Rot.y: %12.2f %12.2f (%9.2f)\n' % ( roty_mean * 180 / math.pi, roty_med * 180 / math.pi, roty_std * 180 / math.pi) txt_out += ' gamma_y: %12.5f %12.5f (%9.5f)\n' % ( ry_mean, ry_med, ry_std) txt_out += ' gamma_z: %12.5f %12.5f (%9.5f)\n' % ( rz_mean, rz_med, rz_std) txt_out += ' gamma_0: %12.5f %12.5f (%9.5f)\n' % ( r0_mean, r0_med, r0_std) txt_out += ' gamma_e: %12.5f %12.5f (%9.5f)\n' % ( re_mean, re_med, re_std) txt_out += ' voigt_nu: %12.5f %12.5f (%9.5f)\n' % ( voigt_nu_mean, voigt_nu_med, voigt_nu_std) txt_out += ' unit cell\n' txt_out += ' a: %12.2f %12.2f (%9.2f)\n' % ( uc_mean[0], uc_med[0], uc_std[0]) txt_out += ' b: %12.2f %12.2f (%9.2f)\n' % ( uc_mean[1], uc_med[1], uc_std[1]) txt_out += ' c: %12.2f %12.2f (%9.2f)\n' % ( uc_mean[2], uc_med[2], uc_std[2]) txt_out += ' alpha: %12.2f %12.2f (%9.2f)\n' % ( uc_mean[3], uc_med[3], uc_std[3]) txt_out += ' beta: %12.2f %12.2f (%9.2f)\n' % ( uc_mean[4], uc_med[4], uc_std[4]) txt_out += ' gamma: %12.2f %12.2f (%9.2f)\n' % ( uc_mean[5], uc_med[5], uc_std[5]) txt_out += 'Parmeters from integration (not-refined)\n' txt_out += ' Wavelength: %12.5f %12.5f (%9.5f)\n' % ( np.mean(wavelength_all), np.median(wavelength_all), np.std(wavelength_all)) txt_out += ' Detector distance: %12.5f %12.5f (%9.5f)\n' % ( np.mean(detector_distance_set), np.median(detector_distance_set), np.std(detector_distance_set)) txt_out += '* (standard deviation)\n' #write out stat. pickle if not iparams.flag_hush: stat_dict = {"n_frames_good": [cn_good_frame], \ "n_frames_bad_cc": [cn_bad_frame_cc], \ "n_frames_bad_G": [cn_bad_frame_G], \ "n_frames_bad_uc": [cn_bad_frame_uc], \ "n_frames_bad_gamma_e": [cn_bad_frame_re], \ "n_frames_bad_SE": [cn_bad_frame_SE], \ "n_observations": [len(I_obs_all_sort)], \ "R_start": [np.mean(R_init_all)], \ "R_end": [np.mean(R_final_all)], \ "R_xy_start": [np.mean(R_xy_init_all)], \ "R_xy_end": [np.mean(R_xy_final_all)], \ "mean_gamma_y": [ry_mean], \ "std_gamma_y": [ry_std], \ "mean_gamma_z": [rz_mean], \ "std_gamma_z": [rz_std], \ "mean_gamma_0": [r0_mean], \ "std_gamma_0": [r0_std], \ "mean_gamma_e": [re_mean], \ "std_gamma_e": [re_std], \ "mean_voigt_nu": [voigt_nu_mean], \ "std_voigt_nu": [voigt_nu_std], \ "mean_a": [uc_mean[0]], \ "std_a": [uc_std[0]], \ "mean_b": [uc_mean[1]], \ "std_b": [uc_std[1]], \ "mean_c": [uc_mean[2]], \ "std_c": [uc_std[2]], \ "mean_alpha": [uc_mean[3]], \ "std_alpha": [uc_std[3]], \ "mean_beta": [uc_mean[4]], \ "std_beta": [uc_std[4]], \ "mean_gamma": [uc_mean[5]], \ "std_gamma": [uc_std[5]]} self.write_stat_pickle(iparams, stat_dict) return cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, \ I_obs_all_sort, sigI_obs_all_sort,G_all_sort, B_all_sort, \ p_all_sort, rs_all_sort, wavelength_all_sort, sin_sq_all_sort, SE_all_sort, uc_mean, \ np.mean(wavelength_all), pickle_filename_all_sort, txt_out
def refine_subgroup(args): assert len(args) == 5 from dials.command_line.check_indexing_symmetry \ import get_symop_correlation_coefficients, normalise_intensities params, subgroup, used_reflections, experiments, refiner_verbosity = args used_reflections = copy.deepcopy(used_reflections) triclinic_miller = used_reflections['miller_index'] cb_op = subgroup['cb_op_inp_best'] higher_symmetry_miller = cb_op.apply(triclinic_miller) used_reflections['miller_index'] = higher_symmetry_miller unrefined_crystal = copy.deepcopy(subgroup.unrefined_crystal) for expt in experiments: expt.crystal = unrefined_crystal from dials.algorithms.indexing.refinement import refine subgroup.max_cc = None subgroup.min_cc = None subgroup.correlation_coefficients = [] subgroup.cc_nrefs = [] try: logger = logging.getLogger() disabled = logger.disabled logger.disabled = True iqr_multiplier = params.refinement.reflections.outlier.tukey.iqr_multiplier params.refinement.reflections.outlier.tukey.iqr_multiplier = 2 * iqr_multiplier refinery, refined, outliers = refine(params, used_reflections, experiments, verbosity=refiner_verbosity) params.refinement.reflections.outlier.tukey.iqr_multiplier = iqr_multiplier refinery, refined, outliers = refine(params, used_reflections, refinery.get_experiments(), verbosity=refiner_verbosity) except RuntimeError as e: if (str(e) == "scitbx Error: g0 - astry*astry -astrz*astrz <= 0." or str(e) == "scitbx Error: g1-bstrz*bstrz <= 0."): subgroup.refined_crystal = None subgroup.rmsd = None subgroup.Nmatches = None subgroup.scan = None subgroup.goniometer = None subgroup.beam = None subgroup.detector = None else: raise else: dall = refinery.rmsds() dx = dall[0] dy = dall[1] subgroup.rmsd = math.sqrt(dx * dx + dy * dy) subgroup.Nmatches = len(refinery.get_matches()) refined_exps = refinery.get_experiments() subgroup.scan = refined_exps[0].scan subgroup.goniometer = refined_exps[0].goniometer subgroup.beam = refined_exps[0].beam subgroup.detector = refined_exps[0].detector subgroup.refined_crystal = refined_exps[0].crystal cs = crystal.symmetry( unit_cell=subgroup.refined_crystal.get_unit_cell(), space_group=subgroup.refined_crystal.get_space_group()) if 'intensity.sum.value' in used_reflections: # remove refl with -ve variance sel = used_reflections['intensity.sum.variance'] > 0 good_reflections = used_reflections.select(sel) from cctbx import miller ms = miller.set(cs, good_reflections['miller_index']) ms = ms.array( good_reflections['intensity.sum.value'] / flex.sqrt(good_reflections['intensity.sum.variance'])) if params.normalise: if params.normalise_bins: ms = normalise_intensities(ms, n_bins=params.normalise_bins) else: ms = normalise_intensities(ms) if params.cc_n_bins is not None: ms.setup_binner(n_bins=params.cc_n_bins) ccs, nrefs = get_symop_correlation_coefficients( ms, use_binning=(params.cc_n_bins is not None)) subgroup.correlation_coefficients = ccs subgroup.cc_nrefs = nrefs ccs = ccs.select(nrefs > 10) if len(ccs) > 1: subgroup.max_cc = flex.max(ccs[1:]) subgroup.min_cc = flex.min(ccs[1:]) finally: logger.disabled = disabled return subgroup
def run(args, command_name = "mmtbx.csv_to_mtz"): if (len(args) == 0): args = ["--help"] try: command_line = (iotbx_option_parser( usage="%s [reflection_csv_file] [options]" % command_name, description='Example: %s 1m5u-sf.csv --use_model=1m5u.pdb'%command_name) .enable_symmetry_comprehensive() .option(None, "--output_file", action="store", default=False, type="string", help="Output mtz file name.") .option(None, "--use_model", action="store", default=False, type="string", help="Use PDB model to make better guess about reflection data type.") ).process(args=args) except Exception as e: if(str(e) != "0"): print(str(e)) sys.exit(0) crystal_symmetry = command_line.symmetry if(command_line.symmetry.unit_cell() is None or command_line.symmetry.space_group_info() is None): if(command_line.options.use_model): crystal_symmetry = crystal_symmetry_from_pdb.extract_from( file_name=command_line.options.use_model) if(crystal_symmetry.unit_cell() is None or crystal_symmetry.space_group_info() is None): raise Sorry( "Crystal symmetry is not defined. Please use the --symmetry option.\n" "Type %s without arguments to see more options."%command_name) if(len(command_line.args) > 1): print("%d arguments are given from the command line:"% \ len(command_line.args), command_line.args) raise Sorry("Please specify one reflection csv file.") file_name = command_line.args[0] if(not os.path.isfile(file_name)): raise Sorry("File is not found: %s"%file_name) data = flex.double() sigmas = flex.double() flags = flex.int() column_ids, columns = parse_csv_file(file_name=file_name) data_label_root = column_ids[3] ms = miller.set(crystal_symmetry, flex.miller_index(columns[0])) for d in columns[1]: data.append(float(d)) for sig in columns[2]: sigmas.append(float(sig)) for flag in columns[3]: flags.append(int(flag)) assert len(data) == len(sigmas) assert len(flags) == len(data) ma = miller.array(ms, data, sigmas) if data_label_root.startswith('F'): ma.set_observation_type_xray_amplitude() elif data_label_root.startswith('I'): ma.set_observation_type_xray_intensity() else: ma.set_observation_type_xray_amplitude() flags_ma = miller.set( crystal_symmetry = crystal_symmetry, indices = ma.indices()).array(data = flags) mtz_dataset = ma.as_mtz_dataset( column_root_label = data_label_root) mtz_dataset.add_miller_array( miller_array = flags_ma, column_root_label = "R-free-flags") mtz_object = mtz_dataset.mtz_object() mtz_object.write(file_name = command_line.options.output_file)
def run(args): from dials.util.options import OptionParser import libtbx.load_env usage = "%s [options] find_spots.json" % (libtbx.env.dispatcher_name) parser = OptionParser(usage=usage, phil=phil_scope, epilog=help_message) params, options, args = parser.parse_args(show_diff_phil=True, return_unhandled=True) positions = None if params.positions is not None: with open(params.positions, 'rb') as f: positions = flex.vec2_double() for line in f.readlines(): line = line.replace('(', ' ').replace(')', '').replace( ',', ' ').strip().split() assert len(line) == 3 i, x, y = [float(l) for l in line] positions.append((x, y)) assert len(args) == 1 json_file = args[0] import json with open(json_file, 'rb') as f: results = json.load(f) n_indexed = flex.double() fraction_indexed = flex.double() n_spots = flex.double() n_lattices = flex.double() crystals = [] image_names = flex.std_string() for r in results: n_spots.append(r['n_spots_total']) image_names.append(str(r['image'])) if 'n_indexed' in r: n_indexed.append(r['n_indexed']) n_lattices.append(len(r['lattices'])) for d in r['lattices']: from dxtbx.model.crystal import CrystalFactory crystals.append(CrystalFactory.from_dict(d['crystal'])) else: n_indexed.append(0) n_lattices.append(0) if n_indexed.size(): sel = n_spots > 0 fraction_indexed = flex.double(n_indexed.size(), 0) fraction_indexed.set_selected( sel, n_indexed.select(sel) / n_spots.select(sel)) import matplotlib matplotlib.use('Agg') from matplotlib import pyplot blue = '#3498db' red = '#e74c3c' marker = 'o' alpha = 0.5 lw = 0 plot = True table = True grid = params.grid from libtbx import group_args from dials.algorithms.spot_finding.per_image_analysis \ import plot_stats, print_table estimated_d_min = flex.double() d_min_distl_method_1 = flex.double() d_min_distl_method_2 = flex.double() n_spots_total = flex.int() n_spots_no_ice = flex.int() total_intensity = flex.double() for d in results: estimated_d_min.append(d['estimated_d_min']) d_min_distl_method_1.append(d['d_min_distl_method_1']) d_min_distl_method_2.append(d['d_min_distl_method_2']) n_spots_total.append(d['n_spots_total']) n_spots_no_ice.append(d['n_spots_no_ice']) total_intensity.append(d['total_intensity']) stats = group_args(image=image_names, n_spots_total=n_spots_total, n_spots_no_ice=n_spots_no_ice, n_spots_4A=None, n_indexed=n_indexed, fraction_indexed=fraction_indexed, total_intensity=total_intensity, estimated_d_min=estimated_d_min, d_min_distl_method_1=d_min_distl_method_1, d_min_distl_method_2=d_min_distl_method_2, noisiness_method_1=None, noisiness_method_2=None) if plot: plot_stats(stats) pyplot.clf() if table: print_table(stats) n_rows = 10 n_rows = min(n_rows, len(n_spots_total)) perm_n_spots_total = flex.sort_permutation(n_spots_total, reverse=True) print('Top %i images sorted by number of spots:' % n_rows) print_table(stats, perm=perm_n_spots_total, n_rows=n_rows) if flex.max(n_indexed) > 0: perm_n_indexed = flex.sort_permutation(n_indexed, reverse=True) print('Top %i images sorted by number of indexed reflections:' % n_rows) print_table(stats, perm=perm_n_indexed, n_rows=n_rows) print("Number of indexed lattices: ", (n_indexed > 0).count(True)) print("Number with valid d_min but failed indexing: ", ((d_min_distl_method_1 > 0) & (d_min_distl_method_2 > 0) & (estimated_d_min > 0) & (n_indexed == 0)).count(True)) n_bins = 20 spot_count_histogram(n_spots_total, n_bins=n_bins, filename='hist_n_spots_total.png', log=True) spot_count_histogram(n_spots_no_ice, n_bins=n_bins, filename='hist_n_spots_no_ice.png', log=True) spot_count_histogram(n_indexed.select(n_indexed > 0), n_bins=n_bins, filename='hist_n_indexed.png', log=False) if len(crystals): plot_unit_cell_histograms(crystals) if params.stereographic_projections and len(crystals): from dxtbx.datablock import DataBlockFactory datablocks = DataBlockFactory.from_filenames([image_names[0]], verbose=False) assert len(datablocks) == 1 imageset = datablocks[0].extract_imagesets()[0] s0 = imageset.get_beam().get_s0() # XXX what if no goniometer? rotation_axis = imageset.get_goniometer().get_rotation_axis() indices = ((1, 0, 0), (0, 1, 0), (0, 0, 1)) for i, index in enumerate(indices): from cctbx import crystal, miller from scitbx import matrix miller_indices = flex.miller_index([index]) symmetry = crystal.symmetry( unit_cell=crystals[0].get_unit_cell(), space_group=crystals[0].get_space_group()) miller_set = miller.set(symmetry, miller_indices) d_spacings = miller_set.d_spacings() d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1() d_spacings = d_spacings.generate_bijvoet_mates() miller_indices = d_spacings.indices() # plane normal d0 = matrix.col(s0).normalize() d1 = d0.cross(matrix.col(rotation_axis)).normalize() d2 = d1.cross(d0).normalize() reference_poles = (d0, d1, d2) from dials.command_line.stereographic_projection import stereographic_projection projections = [] for cryst in crystals: reciprocal_space_points = list( cryst.get_U() * cryst.get_B()) * miller_indices.as_vec3_double() projections.append( stereographic_projection(reciprocal_space_points, reference_poles)) #from dials.algorithms.indexing.compare_orientation_matrices import \ # difference_rotation_matrix_and_euler_angles #R_ij, euler_angles, cb_op = difference_rotation_matrix_and_euler_angles( # crystals[0], cryst) #print max(euler_angles) from dials.command_line.stereographic_projection import plot_projections plot_projections(projections, filename='projections_%s.png' % ('hkl'[i])) pyplot.clf() def plot_grid(values, grid, file_name, cmap=pyplot.cm.Reds, vmin=None, vmax=None, invalid='white'): values = values.as_double() # At DLS, fast direction appears to be largest direction if grid[0] > grid[1]: values.reshape(flex.grid(reversed(grid))) values = values.matrix_transpose() else: values.reshape(flex.grid(grid)) Z = values.as_numpy_array() #f, (ax1, ax2) = pyplot.subplots(2) f, ax1 = pyplot.subplots(1) mesh1 = ax1.pcolormesh(values.as_numpy_array(), cmap=cmap, vmin=vmin, vmax=vmax) mesh1.cmap.set_under(color=invalid, alpha=None) mesh1.cmap.set_over(color=invalid, alpha=None) #mesh2 = ax2.contour(Z, cmap=cmap, vmin=vmin, vmax=vmax) #mesh2 = ax2.contourf(Z, cmap=cmap, vmin=vmin, vmax=vmax) ax1.set_aspect('equal') ax1.invert_yaxis() #ax2.set_aspect('equal') #ax2.invert_yaxis() pyplot.colorbar(mesh1, ax=ax1) #pyplot.colorbar(mesh2, ax=ax2) pyplot.savefig(file_name, dpi=600) pyplot.clf() def plot_positions(values, positions, file_name, cmap=pyplot.cm.Reds, vmin=None, vmax=None, invalid='white'): values = values.as_double() assert positions.size() >= values.size() positions = positions[:values.size()] if vmin is None: vmin = flex.min(values) if vmax is None: vmax = flex.max(values) x, y = positions.parts() dx = flex.abs(x[1:] - x[:-1]) dy = flex.abs(y[1:] - y[:-1]) dx = dx.select(dx > 0) dy = dy.select(dy > 0) scale = 1 / flex.min(dx) #print scale x = (x * scale).iround() y = (y * scale).iround() from libtbx.math_utils import iceil z = flex.double( flex.grid(iceil(flex.max(y)) + 1, iceil(flex.max(x)) + 1), -2) #print z.all() for x_, y_, z_ in zip(x, y, values): z[y_, x_] = z_ plot_grid(z.as_1d(), z.all(), file_name, cmap=cmap, vmin=vmin, vmax=vmax, invalid=invalid) return if grid is not None or positions is not None: if grid is not None: positions = tuple(reversed(grid)) plotter = plot_grid else: plotter = plot_positions cmap = pyplot.get_cmap(params.cmap) plotter(n_spots_total, positions, 'grid_spot_count_total.png', cmap=cmap, invalid=params.invalid) plotter(n_spots_no_ice, positions, 'grid_spot_count_no_ice.png', cmap=cmap, invalid=params.invalid) plotter(total_intensity, positions, 'grid_total_intensity.png', cmap=cmap, invalid=params.invalid) if flex.max(n_indexed) > 0: plotter(n_indexed, positions, 'grid_n_indexed.png', cmap=cmap, invalid=params.invalid) plotter(fraction_indexed, positions, 'grid_fraction_indexed.png', cmap=cmap, vmin=0, vmax=1, invalid=params.invalid) for i, d_min in enumerate( (estimated_d_min, d_min_distl_method_1, d_min_distl_method_2)): from cctbx import uctbx d_star_sq = uctbx.d_as_d_star_sq(d_min) d_star_sq.set_selected(d_star_sq == 1, 0) vmin = flex.min(d_star_sq.select(d_star_sq > 0)) vmax = flex.max(d_star_sq) vmin = flex.min(d_min.select(d_min > 0)) vmax = flex.max(d_min) cmap = pyplot.get_cmap('%s_r' % params.cmap) d_min.set_selected(d_min <= 0, vmax) if i == 0: plotter(d_min, positions, 'grid_d_min.png', cmap=cmap, vmin=vmin, vmax=vmax, invalid=params.invalid) else: plotter(d_min, positions, 'grid_d_min_method_%i.png' % i, cmap=cmap, vmin=vmin, vmax=vmax, invalid=params.invalid) if flex.max(n_indexed) > 0: pyplot.hexbin(n_spots, n_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter(n_spots, n_indexed, marker=marker, alpha=alpha, c=blue, lw=lw) xlim = pyplot.xlim() ylim = pyplot.ylim() pyplot.plot([0, max(n_spots)], [0, max(n_spots)], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel('# spots') pyplot.ylabel('# indexed') pyplot.savefig('n_spots_vs_n_indexed.png') pyplot.clf() pyplot.hexbin(n_spots, fraction_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_spots, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# spots') pyplot.ylabel('Fraction indexed') pyplot.savefig('n_spots_vs_fraction_indexed.png') pyplot.clf() pyplot.hexbin(n_indexed, fraction_indexed, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_indexed, fraction_indexed, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# indexed') pyplot.ylabel('Fraction indexed') pyplot.savefig('n_indexed_vs_fraction_indexed.png') pyplot.clf() pyplot.hexbin(n_spots, n_lattices, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_spots, n_lattices, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# spots') pyplot.ylabel('# lattices') pyplot.savefig('n_spots_vs_n_lattices.png') pyplot.clf() #pyplot.scatter( # estimated_d_min, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.hexbin(estimated_d_min, d_min_distl_method_1, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.gca().set_aspect('equal') xlim = pyplot.xlim() ylim = pyplot.ylim() m = max(max(estimated_d_min), max(d_min_distl_method_1)) pyplot.plot([0, m], [0, m], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel('estimated_d_min') pyplot.ylabel('d_min_distl_method_1') pyplot.savefig('d_min_vs_distl_method_1.png') pyplot.clf() #pyplot.scatter( # estimated_d_min, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.hexbin(estimated_d_min, d_min_distl_method_2, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.gca().set_aspect('equal') xlim = pyplot.xlim() ylim = pyplot.ylim() m = max(max(estimated_d_min), max(d_min_distl_method_2)) pyplot.plot([0, m], [0, m], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel('estimated_d_min') pyplot.ylabel('d_min_distl_method_2') pyplot.savefig('d_min_vs_distl_method_2.png') pyplot.clf() #pyplot.scatter( # d_min_distl_method_1, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.hexbin(d_min_distl_method_1, d_min_distl_method_2, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.gca().set_aspect('equal') xlim = pyplot.xlim() ylim = pyplot.ylim() m = max(max(d_min_distl_method_1), max(d_min_distl_method_2)) pyplot.plot([0, m], [0, m], c=red) pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.xlabel('d_min_distl_method_1') pyplot.ylabel('d_min_distl_method_2') pyplot.savefig('distl_method_1_vs_distl_method_2.png') pyplot.clf() pyplot.hexbin(n_spots, estimated_d_min, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_spots, estimated_d_min, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# spots') pyplot.ylabel('estimated_d_min') pyplot.savefig('n_spots_vs_d_min.png') pyplot.clf() pyplot.hexbin(n_spots, d_min_distl_method_1, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_spots, d_min_distl_method_1, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# spots') pyplot.ylabel('d_min_distl_method_1') pyplot.savefig('n_spots_vs_distl_method_1.png') pyplot.clf() pyplot.hexbin(n_spots, d_min_distl_method_2, bins='log', cmap=pyplot.cm.jet, gridsize=50) pyplot.colorbar() #pyplot.scatter( #n_spots, d_min_distl_method_2, marker=marker, alpha=alpha, c=blue, lw=lw) pyplot.xlim(0, pyplot.xlim()[1]) pyplot.ylim(0, pyplot.ylim()[1]) pyplot.xlabel('# spots') pyplot.ylabel('d_min_distl_method_2') pyplot.savefig('n_spots_vs_distl_method_2.png') pyplot.clf()
def exercise(space_group_info, anomalous_flag, n_scatterers=8, d_min=2, verbose=0): structure = random_structure.xray_structure(space_group_info, elements=["const"] * n_scatterers) f_calc = structure.structure_factors( d_min=d_min, anomalous_flag=anomalous_flag).f_calc() f = abs(f_calc) fs = miller.array(miller_set=f, data=f.data(), sigmas=flex.sqrt(f.data())) assert fs.is_unique_set_under_symmetry() for a in (f, fs): for algorithm in ["gaussian", "shelx"]: m = a.merge_equivalents(algorithm=algorithm) m.show_summary(out=StringIO()) j = m.array().adopt_set(a) assert flex.linear_correlation(j.data(), a.data()).coefficient() > 1 - 1.e-6 if (a.sigmas() is not None): assert flex.linear_correlation( j.sigmas(), a.sigmas()).coefficient() > 1 - 1.e-6 redundancies = flex.size_t() for i in xrange(fs.indices().size()): redundancies.append(random.randrange(5) + 1) space_group = space_group_info.group() r_indices = flex.miller_index() r_data = flex.double() r_sigmas = flex.double() for i, n in enumerate(redundancies): h = fs.indices()[i] h_eq = miller.sym_equiv_indices(space_group, h).indices() for j in xrange(n): r_indices.append(h_eq[random.randrange(len(h_eq))].h()) r_data.append(fs.data()[i]) r_sigmas.append(fs.sigmas()[i]) r = miller.array(miller_set=miller.set(crystal_symmetry=fs, indices=r_indices, anomalous_flag=fs.anomalous_flag()), data=r_data, sigmas=r_sigmas) assert not r.is_unique_set_under_symmetry() noise = flex.random_double(size=r.indices().size()) r = r.sort(by_value=noise) for algorithm in ["gaussian", "shelx"]: m = r.merge_equivalents(algorithm=algorithm) m.show_summary(out=StringIO()) j = m.array().adopt_set(fs) assert j.is_unique_set_under_symmetry() assert flex.linear_correlation(j.data(), fs.data()).coefficient() > 1 - 1.e-6 fssr = fs.sigmas() / flex.sqrt(redundancies.as_double()) assert flex.linear_correlation(j.sigmas(), fssr).coefficient() > 1 - 1.e-6 # if (anomalous_flag): f_calc_ave = f_calc.average_bijvoet_mates() # uses merge_equivalents f_calc_com = f_calc.as_non_anomalous_array().common_set(f_calc_ave) assert f_calc_com.indices().all_eq(f_calc_ave.indices()) for part in [flex.real, flex.imag]: assert flex.linear_correlation(part( f_calc_com.data()), part( f_calc_ave.data())).coefficient() > 1 - 1.e-6 # test use_internal_variance=False m = r.merge_equivalents(algorithm="gaussian", use_internal_variance=False) j = m.array().adopt_set(fs) fssr = fs.sigmas() / flex.sqrt(redundancies.as_double()) assert flex.linear_correlation(j.sigmas(), fssr).coefficient() > 1 - 1.e-6
def run(args): from dials.util.options import OptionParser from dials.util.options import flatten_experiments # The script usage usage = "dials.stereographic_projection [options] [param.phil] indexed.expt" parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, check_format=False, epilog=help_message, ) params, options = parser.parse_args(args=args, show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) if not experiments: parser.print_help() return if not params.hkl and params.hkl_limit is None: sys.exit("Please provide hkl or hkl_limit parameters.") if params.plot.labels and len(params.plot.labels) != len(experiments): sys.exit( "Number of labels (%i) must equal number of experiments (%i)" % (len(params.plot.labels), len(experiments)) ) if params.hkl is not None and len(params.hkl): miller_indices = flex.miller_index(params.hkl) elif params.hkl_limit is not None: limit = params.hkl_limit miller_indices = flex.miller_index() for h in range(-limit, limit + 1): for k in range(-limit, limit + 1): for l in range(-limit, limit + 1): if (h, k, l) == (0, 0, 0): continue miller_indices.append((h, k, l)) crystals = experiments.crystals() symmetry = crystal.symmetry( unit_cell=crystals[0].get_unit_cell(), space_group=crystals[0].get_space_group() ) miller_set = miller.set(symmetry, miller_indices) d_spacings = miller_set.d_spacings() if params.eliminate_sys_absent: d_spacings = d_spacings.eliminate_sys_absent() if params.expand_to_p1: d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1() miller_indices = d_spacings.indices() # find the greatest common factor (divisor) between miller indices miller_indices_unique = flex.miller_index() for hkl in miller_indices: gcd = gcd_list(hkl) if gcd > 1: miller_indices_unique.append(tuple(int(h / gcd) for h in hkl)) elif gcd < 1: pass else: miller_indices_unique.append(hkl) miller_indices = miller_indices_unique miller_indices = flex.miller_index(list(set(miller_indices))) ref_crystal = crystals[0] U = matrix.sqr(ref_crystal.get_U()) B = matrix.sqr(ref_crystal.get_B()) R = matrix.identity(3) if params.frame == "laboratory": reference_poles = reference_poles_perpendicular_to_beam( experiments[0].beam, experiments[0].goniometer ) if params.use_starting_angle: rotation_axis = matrix.col(experiments[0].goniometer.get_rotation_axis()) R = rotation_axis.axis_and_angle_as_r3_rotation_matrix( experiments[0].scan.get_oscillation()[0], deg=True ) elif params.phi_angle != 0: rotation_axis = matrix.col(experiments[0].goniometer.get_rotation_axis()) R = rotation_axis.axis_and_angle_as_r3_rotation_matrix( params.phi_angle, deg=True ) else: if params.plane_normal is not None: plane_normal = params.plane_normal else: plane_normal = (0, 0, 1) reference_poles = reference_poles_crystal( ref_crystal, plane_normal=plane_normal ) if params.frame == "crystal": U = matrix.identity(3) reciprocal_space_points = list(R * U * B) * miller_indices.as_vec3_double() projections_ref = stereographic_projection(reciprocal_space_points, reference_poles) projections_all = [projections_ref] if experiments: from dials.algorithms.indexing.compare_orientation_matrices import ( difference_rotation_matrix_axis_angle, ) for expt in experiments[1:]: cryst = expt.crystal if params.frame == "crystal": R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle( ref_crystal, cryst ) U = R_ij elif params.use_starting_angle: if params.use_starting_angle: rotation_axis = matrix.col(expt.goniometer.get_rotation_axis()) R = rotation_axis.axis_and_angle_as_r3_rotation_matrix( expt.scan.get_oscillation()[0], deg=True ) else: U = matrix.sqr(cryst.get_U()) reciprocal_space_points = ( list(R * U * matrix.sqr(cryst.get_B())) * miller_indices.as_vec3_double() ) projections = stereographic_projection( reciprocal_space_points, reference_poles ) projections_all.append(projections) if params.save_coordinates: with open("projections.txt", "w") as f: f.write("crystal h k l x y" + os.linesep) for i_cryst, projections in enumerate(projections_all): for hkl, proj in zip(miller_indices, projections): f.write("%i " % (i_cryst + 1)) f.write("%i %i %i " % hkl) f.write(("%f %f" + os.linesep) % proj) if params.plot.filename: epochs = None if params.plot.colour_map is not None: if experiments[0].scan is not None: epochs = [expt.scan.get_epochs()[0] for expt in experiments] else: epochs = [i for i, expt in enumerate(experiments)] plot_projections( projections_all, filename=params.plot.filename, colours=params.plot.colours, marker_size=params.plot.marker_size, font_size=params.plot.font_size, gridsize=params.plot.gridsize, label_indices=miller_indices if params.plot.label_indices else False, epochs=epochs, colour_map=params.plot.colour_map, ) if params.json.filename: projections_as_json( projections_all, filename=params.json.filename, labels=params.plot.labels )
def run(self, experiments, reflections): self.logger.log_step_time("SCALE_FRAMES") if self.params.scaling.algorithm != "mark0": # mark1 implies no scaling/post-refinement self.logger.log("No scaling was done") if self.mpi_helper.rank == 0: self.logger.main_log("No scaling was done") return experiments, reflections new_experiments = ExperimentList() new_reflections = flex.reflection_table() # scale experiments, one at a time. Reject experiments that do not correlate with the reference or fail to scale. results = [] slopes = [] correlations = [] high_res_experiments = 0 experiments_rejected_because_of_low_signal = 0 experiments_rejected_because_of_low_correlation_with_reference = 0 target_symm = symmetry( unit_cell=self.params.scaling.unit_cell, space_group_info=self.params.scaling.space_group) for experiment in experiments: exp_reflections = reflections.select( reflections['exp_id'] == experiment.identifier) # Build a miller array for the experiment reflections exp_miller_indices = miller.set( target_symm, exp_reflections['miller_index_asymmetric'], True) exp_intensities = miller.array( exp_miller_indices, exp_reflections['intensity.sum.value'], flex.double( flex.sqrt(exp_reflections['intensity.sum.variance']))) model_intensities = self.params.scaling.i_model # Extract an array of HKLs from the model to match the experiment HKLs matching_indices = miller.match_multi_indices( miller_indices_unique=model_intensities.indices(), miller_indices=exp_intensities.indices()) # Least squares if self.params.scaling.mark0.fit_reference_to_experiment: # RB: in cxi-merge we fit reference to experiment, but we should really do it the other way result = self.fit_reference_to_experiment( model_intensities, exp_intensities, matching_indices) else: result = self.fit_experiment_to_reference( model_intensities, exp_intensities, matching_indices) if result.error == scaling_result.err_low_signal: experiments_rejected_because_of_low_signal += 1 continue elif result.error == scaling_result.err_low_correlation: experiments_rejected_because_of_low_correlation_with_reference += 1 continue slopes.append(result.slope) correlations.append(result.correlation) if self.params.output.log_level == 0: self.logger.log( "Experiment ID: %s; Slope: %f; Correlation %f" % (experiment.identifier, result.slope, result.correlation)) # count high resolution experiments if exp_intensities.d_min() <= self.params.merging.d_min: high_res_experiments += 1 # apply scale factors if not self.params.postrefinement.enable: if self.params.scaling.mark0.fit_reference_to_experiment: exp_reflections['intensity.sum.value'] /= result.slope exp_reflections['intensity.sum.variance'] /= ( result.slope**2) else: exp_reflections['intensity.sum.value'] *= result.slope exp_reflections['intensity.sum.variance'] *= ( result.slope**2) new_experiments.append(experiment) new_reflections.extend(exp_reflections) rejected_experiments = len(experiments) - len(new_experiments) assert rejected_experiments == experiments_rejected_because_of_low_signal + \ experiments_rejected_because_of_low_correlation_with_reference reflections_removed_because_of_rejected_experiments = reflections.size( ) - new_reflections.size() self.logger.log("Experiments rejected because of low signal: %d" % experiments_rejected_because_of_low_signal) self.logger.log( "Experiments rejected because of low correlation with reference: %d" % experiments_rejected_because_of_low_correlation_with_reference) self.logger.log( "Reflections rejected because of rejected experiments: %d" % reflections_removed_because_of_rejected_experiments) self.logger.log("High resolution experiments: %d" % high_res_experiments) if self.params.postrefinement.enable: self.logger.log( "Note: scale factors were not applied, because postrefinement is enabled" ) # MPI-reduce all counts comm = self.mpi_helper.comm MPI = self.mpi_helper.MPI total_experiments_rejected_because_of_low_signal = comm.reduce( experiments_rejected_because_of_low_signal, MPI.SUM, 0) total_experiments_rejected_because_of_low_correlation_with_reference = comm.reduce( experiments_rejected_because_of_low_correlation_with_reference, MPI.SUM, 0) total_reflections_removed_because_of_rejected_experiments = comm.reduce( reflections_removed_because_of_rejected_experiments, MPI.SUM, 0) total_high_res_experiments = comm.reduce(high_res_experiments, MPI.SUM, 0) all_slopes = comm.reduce(slopes, MPI.SUM, 0) all_correlations = comm.reduce(correlations, MPI.SUM, 0) # rank 0: log data statistics if self.mpi_helper.rank == 0: self.logger.main_log( 'Experiments rejected because of low signal: %d' % total_experiments_rejected_because_of_low_signal) self.logger.main_log( 'Experiments rejected because of low correlation with reference: %d' % total_experiments_rejected_because_of_low_correlation_with_reference ) self.logger.main_log( 'Reflections rejected because of rejected experiments: %d' % total_reflections_removed_because_of_rejected_experiments) self.logger.main_log( 'Experiments with high resolution of %5.2f Angstrom or better: %d' % (self.params.merging.d_min, total_high_res_experiments)) if len(all_slopes) > 0: stats_slope = flex.mean_and_variance(flex.double(all_slopes)) self.logger.main_log( 'Average experiment scale factor wrt reference: %f' % (stats_slope.mean())) if len(all_correlations) > 0: stats_correlation = flex.mean_and_variance( flex.double(all_correlations)) self.logger.main_log( 'Average experiment correlation with reference: %f +/- %f' % (stats_correlation.mean(), stats_correlation.unweighted_sample_standard_deviation())) if self.params.postrefinement.enable: self.logger.main_log( "Note: scale factors were not applied, because postrefinement is enabled" ) self.logger.log_step_time("SCALE_FRAMES", True) return new_experiments, new_reflections
def finite_diffs_aniso(p_scale, u_star, centric=False, h=0.0001): d_star_sq = flex.double(2, 0.25) f_obs = flex.double(2, 1.0) centric_array = flex.bool(2, centric) sigma_f_obs = f_obs / 10.0 sigma_sq = flex.double(2, 1.0) epsilon = flex.double(2, 1.0) gamma = flex.double(2, 0.0) unit_cell = uctbx.unit_cell('20, 30, 40, 90.0, 90.0, 90.0') mi = flex.miller_index(((1, 2, 3), (1, 2, 3))) xs = crystal.symmetry((20, 30, 40), "P 2 2 2") ms = miller.set(xs, mi) nll_norm = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0], sigma_f_obs[0], epsilon[0], sigma_sq[0], gamma[0], centric_array[0], p_scale, unit_cell, u_star) nll_scale = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0], sigma_f_obs[0], epsilon[0], sigma_sq[0], gamma[0], centric_array[0], p_scale + h, unit_cell, u_star) u_star[0] += h nll_u11 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0], sigma_f_obs[0], epsilon[0], sigma_sq[0], gamma[0], centric_array[0], p_scale, unit_cell, u_star) u_star[0] -= h u_star[1] += h nll_u22 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0], sigma_f_obs[0], epsilon[0], sigma_sq[0], gamma[0], centric_array[0], p_scale, unit_cell, u_star) u_star[1] -= h u_star[2] += h nll_u33 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0], sigma_f_obs[0], epsilon[0], sigma_sq[0], gamma[0], centric_array[0], p_scale, unit_cell, u_star) u_star[2] -= h u_star[3] += h nll_u12 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0], sigma_f_obs[0], epsilon[0], sigma_sq[0], gamma[0], centric_array[0], p_scale, unit_cell, u_star) u_star[3] -= h u_star[4] += h nll_u13 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0], sigma_f_obs[0], epsilon[0], sigma_sq[0], gamma[0], centric_array[0], p_scale, unit_cell, u_star) u_star[4] -= h u_star[5] += h nll_u23 = scaling.wilson_single_nll_aniso(ms.indices()[0], f_obs[0], sigma_f_obs[0], epsilon[0], sigma_sq[0], gamma[0], centric_array[0], p_scale, unit_cell, u_star) g = scaling.wilson_single_nll_aniso_gradient(ms.indices()[0], f_obs[0], sigma_f_obs[0], epsilon[0], sigma_sq[0], gamma[0], centric_array[0], p_scale, unit_cell, u_star) g2 = scaling.wilson_total_nll_aniso_gradient(ms.indices(), f_obs, sigma_f_obs, epsilon, sigma_sq, gamma, centric_array, p_scale, unit_cell, u_star) ds = (nll_norm - nll_scale) / -h du11 = (nll_norm - nll_u11) / -h du22 = (nll_norm - nll_u22) / -h du33 = (nll_norm - nll_u33) / -h du12 = (nll_norm - nll_u12) / -h du13 = (nll_norm - nll_u13) / -h du23 = (nll_norm - nll_u23) / -h assert approx_equal(ds, g[0]), (ds, g[0]) assert approx_equal(du11, g[1]), (du11, g[1]) assert approx_equal(du22, g[2]) assert approx_equal(du33, g[3]) assert approx_equal(du12, g[4]) assert approx_equal(du13, g[5]) assert approx_equal(du23, g[6]) assert approx_equal(ds, g2[0] / 2.0) assert approx_equal(du11, g2[1] / 2.0) assert approx_equal(du22, g2[2] / 2.0) assert approx_equal(du33, g2[3] / 2.0) assert approx_equal(du12, g2[4] / 2.0) assert approx_equal(du13, g2[5] / 2.0) assert approx_equal(du23, g2[6] / 2.0)
def filtered_arrays_from_experiments_reflections( experiments, reflections, outlier_rejection_after_filter=False, partiality_threshold=0.99, ): """Create a list of filtered arrays from experiments and reflections. A partiality threshold can be set, and if outlier_rejection_after_filter is True, and intensity.scale values are not present, then a round of outlier rejection will take place. Raises: ValueError: if no datasets remain after filtering. """ miller_arrays = [] ids_to_del = [] for idx, (expt, refl) in enumerate(zip(experiments, reflections)): crystal_symmetry = crystal.symmetry( unit_cell=expt.crystal.get_unit_cell(), space_group=expt.crystal.get_space_group(), ) # want to use scale intensities if present, else sum + prf (if available) if "intensity.scale.value" in refl: intensity_choice = ["scale"] intensity_to_use = "intensity.scale" else: assert "intensity.sum.value" in refl intensity_to_use = "intensity.sum" intensity_choice = ["sum"] if "intensity.prf.value" in refl: intensity_choice.append("profile") intensity_to_use = "intensity.prf" try: logger.info("Filtering reflections for dataset %s" % (expt.identifier if expt.identifier else idx)) refl = filter_reflection_table( refl, intensity_choice, min_isigi=-5, filter_ice_rings=False, combine_partials=True, partiality_threshold=partiality_threshold, ) except ValueError: logger.info( "Dataset %s removed as no reflections left after filtering", idx) ids_to_del.append(idx) else: # If scale was chosen - will return scale or have raised ValueError # If prf or sum, possible was no prf but want to continue. try: refl["intensity"] = refl[intensity_to_use + ".value"] refl["variance"] = refl[intensity_to_use + ".variance"] except KeyError: # catch case where prf were removed. refl["intensity"] = refl["intensity.sum.value"] refl["variance"] = refl["intensity.sum.variance"] if outlier_rejection_after_filter and intensity_to_use != "intensity.scale": refl = reject_outliers(refl, expt, method="simple", zmax=12.0) refl = refl.select( ~refl.get_flags(refl.flags.outlier_in_scaling)) miller_set = miller.set(crystal_symmetry, refl["miller_index"], anomalous_flag=False) intensities = miller.array(miller_set, data=refl["intensity"], sigmas=flex.sqrt(refl["variance"])) intensities.set_observation_type_xray_intensity() intensities.set_info( miller.array_info(source="DIALS", source_type="pickle")) miller_arrays.append(intensities) if not miller_arrays: raise ValueError( """No datasets remain after pre-filtering. Please check input data. The datasets may not contain any full reflections; the command line option partiality_threshold can be lowered to include partials.""") for id_ in ids_to_del[::-1]: del experiments[id_] del reflections[id_] return miller_arrays
def run(args): import libtbx.load_env from dials.util.options import OptionParser from dials.util.options import flatten_experiments # The script usage usage = "usage: %s [options] [param.phil] experiments.json" % libtbx.env.dispatcher_name parser = OptionParser(usage=usage, phil=phil_scope, read_experiments=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) if not experiments: parser.print_help() return if not params.hkl and params.hkl_limit is None: from libtbx.utils import Sorry raise Sorry("Please provide hkl or hkl_limit parameters.") if params.hkl is not None and len(params.hkl): miller_indices = flex.miller_index(params.hkl) elif params.hkl_limit is not None: limit = params.hkl_limit miller_indices = flex.miller_index() for h in range(-limit, limit + 1): for k in range(-limit, limit + 1): for l in range(-limit, limit + 1): if (h, k, l) == (0, 0, 0): continue miller_indices.append((h, k, l)) crystals = experiments.crystals() symmetry = crystal.symmetry(unit_cell=crystals[0].get_unit_cell(), space_group=crystals[0].get_space_group()) miller_set = miller.set(symmetry, miller_indices) d_spacings = miller_set.d_spacings() if params.eliminate_sys_absent: d_spacings = d_spacings.eliminate_sys_absent() if params.expand_to_p1: d_spacings = d_spacings.as_non_anomalous_array().expand_to_p1() d_spacings = d_spacings.generate_bijvoet_mates() miller_indices = d_spacings.indices() # find the greatest common factor (divisor) between miller indices miller_indices_unique = flex.miller_index() for hkl in miller_indices: gcd = gcd_list(hkl) if gcd > 1: miller_indices_unique.append(tuple(int(h / gcd) for h in hkl)) elif gcd < 1: pass else: miller_indices_unique.append(hkl) miller_indices = miller_indices_unique miller_indices = flex.miller_index(list(set(miller_indices))) ref_crystal = crystals[0] A = ref_crystal.get_A() U = ref_crystal.get_U() B = ref_crystal.get_B() R = matrix.identity(3) if params.frame == 'laboratory': reference_poles = reference_poles_perpendicular_to_beam( experiments[0].beam, experiments[0].goniometer) if params.use_starting_angle: rotation_axis = matrix.col( experiments[0].goniometer.get_rotation_axis()) R = rotation_axis.axis_and_angle_as_r3_rotation_matrix( experiments[0].scan.get_oscillation()[0], deg=True) elif params.phi_angle != 0: rotation_axis = matrix.col( experiments[0].goniometer.get_rotation_axis()) R = rotation_axis.axis_and_angle_as_r3_rotation_matrix( params.phi_angle, deg=True) else: if params.plane_normal is not None: plane_normal = params.plane_normal else: plane_normal = (0, 0, 1) reference_poles = reference_poles_crystal(ref_crystal, plane_normal=plane_normal) if params.frame == 'crystal': U = matrix.identity(3) reciprocal_space_points = list(R * U * B) * miller_indices.as_vec3_double() projections_ref = stereographic_projection(reciprocal_space_points, reference_poles) projections_all = [projections_ref] if experiments: from dials.algorithms.indexing.compare_orientation_matrices import \ difference_rotation_matrix_axis_angle for expt in experiments[1:]: cryst = expt.crystal if params.frame == 'crystal': R_ij, axis, angle, cb_op = difference_rotation_matrix_axis_angle( ref_crystal, cryst) U = R_ij elif params.use_starting_angle: if params.use_starting_angle: rotation_axis = matrix.col( expt.goniometer.get_rotation_axis()) R = rotation_axis.axis_and_angle_as_r3_rotation_matrix( expt.scan.get_oscillation()[0], deg=True) else: U = cryst.get_U() reciprocal_space_points = list( R * U * cryst.get_B()) * miller_indices.as_vec3_double() projections = stereographic_projection(reciprocal_space_points, reference_poles) projections_all.append(projections) if params.save_coordinates: with open('projections.txt', 'wb') as f: print >> f, "crystal h k l x y" for i_cryst, projections in enumerate(projections_all): for hkl, proj in zip(miller_indices, projections): print >> f, "%i" % (i_cryst + 1), print >> f, "%i %i %i" % hkl, print >> f, "%f %f" % proj if params.plot.show or params.plot.filename: plot_projections(projections_all, filename=params.plot.filename, show=params.plot.show, colours=params.plot.colours, marker_size=params.plot.marker_size, font_size=params.plot.font_size, label_indices=params.plot.label_indices)
def add_miller_array(self, array, array_type=None, column_name=None, column_names=None): """ Accepts a miller array, and one of array_type, column_name or column_names. """ assert [array_type, column_name, column_names].count(None) == 2 if array_type is not None: assert array_type in ('calc', 'meas') elif column_name is not None: column_names = [column_name] if array.is_complex_array(): if column_names is None: column_names = [ self.prefix + 'F_' + array_type, self.prefix + 'phase_' + array_type ] else: assert len(column_names) == 2 if (('_A_' in column_names[0] and '_B_' in column_names[1]) or ('.A_' in column_names[0] and '.B_' in column_names[1])): data = [ flex.real(array.data()).as_string(), flex.imag(array.data()).as_string() ] else: data = [ flex.abs(array.data()).as_string(), array.phases(deg=True).data().as_string() ] elif array.is_hendrickson_lattman_array(): if column_names is None: column_names = [ self.prefix + 'HL_%s_iso' % abcd for abcd in 'ABCD' ] else: assert len(column_names) == 4 data = [d.as_string() for d in array.data().as_abcd()] else: if array_type is not None: if array.is_xray_intensity_array(): obs_ext = 'squared_' else: obs_ext = '' column_names = [self.prefix + 'F_' + obs_ext + array_type] if array.sigmas() is not None: column_names.append(self.prefix + 'F_' + obs_ext + 'sigma') if isinstance(array.data(), flex.std_string): data = [array.data()] else: data = [array.data().as_string()] if array.anomalous_flag(): if ((array.sigmas() is not None and len(column_names) == 4) or (array.sigmas() is None and len(column_names) == 2)): data = [] asu, matches = array.match_bijvoet_mates() for anomalous_sign in ("+", "-"): sel = matches.pairs_hemisphere_selection( anomalous_sign) sel.extend( matches.singles_hemisphere_selection( anomalous_sign)) if (anomalous_sign == "+"): indices = asu.indices().select(sel) hemisphere_column_names = column_names[:len( column_names) // 2] else: indices = -asu.indices().select(sel) hemisphere_column_names = column_names[ len(column_names) // 2:] hemisphere_data = asu.data().select(sel) hemisphere_array = miller.array( miller.set(array.crystal_symmetry(), indices), hemisphere_data) if array.sigmas() is not None: hemisphere_array.set_sigmas( asu.sigmas().select(sel)) if self.refln_loop is None: # then this is the first array to be added to the loop, # hack so we don't have both hemispheres of indices self.indices = indices self.add_miller_array( hemisphere_array, column_names=hemisphere_column_names) return if array.sigmas() is not None and len(column_names) == 2: data.append(array.sigmas().as_string()) if not (self.indices.size() == array.indices().size() and self.indices.all_eq(array.indices())): from cctbx.miller import match_indices other_indices = array.indices().deep_copy() match = match_indices(self.indices, other_indices) if match.singles(0).size(): # array is missing some reflections indices that already appear in the loop # therefore pad the data with '?' values other_indices.extend( self.indices.select(match.single_selection(0))) for d in data: d.extend( flex.std_string(['?'] * (other_indices.size() - d.size()))) for d in data: assert d.size() == other_indices.size() match = match_indices(self.indices, other_indices) if match.singles(1).size(): # this array contains some reflections that are not already present in the # cif loop, therefore need to add rows of '?' values single_indices = other_indices.select( match.single_selection(1)) self.indices.extend(single_indices) n_data_columns = len(self.refln_loop) - 3 for hkl in single_indices: row = list(hkl) + ['?'] * n_data_columns self.refln_loop.add_row(row) match = match_indices(self.indices, other_indices) match = match_indices(self.indices, other_indices) perm = match.permutation() data = [d.select(perm) for d in data] if self.refln_loop is None: self.refln_loop = miller_indices_as_cif_loop(self.indices, prefix=self.prefix) columns = OrderedDict(zip(column_names, data)) for key in columns: assert key not in self.refln_loop self.refln_loop.add_columns(columns)
def populate_observations(self): intensities = self.reflections['intensity.' + self.method + '.value'] variances = self.reflections['intensity.' + self.method + '.variance'] space_group = crystal.symmetry(self.xtal.get_unit_cell(), str(self.xtal.get_space_group().info())) miller_set = miller.set(space_group, self.reflections['miller_index']) self.frame['observations'][0] = cctbx.miller.array(miller_set, intensities, flex.sqrt(variances)).set_observation_type_xray_intensity()
def make_cif_block(self, experiments, reflections): """Write the data to a cif block""" # Select reflections selection = reflections.get_flags(reflections.flags.integrated, all=True) reflections = reflections.select(selection) # Filter out bad variances and other issues, but don't filter on ice rings # or alter partialities. # Assumes you want to apply the lp and dqe corrections to sum and prf # Do we want to combine partials? reflections = filter_reflection_table( reflections, self.params.intensity, combine_partials=False, partiality_threshold=0.0, d_min=self.params.mtz.d_min, ) # Get the cif block cif_block = iotbx.cif.model.block() # Audit trail dials_version = dials.util.version.dials_version() cif_block["_audit.revision_id"] = 1 cif_block["_audit.creation_method"] = dials_version cif_block["_audit.creation_date"] = datetime.date.today().isoformat() cif_block["_entry.id"] = "DIALS" # add software loop mmcif_software_header = ( "_software.pdbx_ordinal", "_software.citation_id", "_software.name", # as defined at [1] "_software.version", "_software.type", "_software.classification", "_software.description", ) mmcif_citations_header = ( "_citation.id", "_citation.journal_abbrev", "_citation.journal_volume", "_citation.journal_issue", "_citation.page_first", "_citation.page_last", "_citation.year", "_citation.title", ) software_loop = iotbx.cif.model.loop(header=mmcif_software_header) citations_loop = iotbx.cif.model.loop(header=mmcif_citations_header) software_loop.add_row(( 1, 1, "DIALS", dials_version, "package", "data processing", "Data processing and integration within the DIALS software package", )) citations_loop.add_row(( 1, "Acta Cryst. D", 74, 2, 85, 97, 2018, "DIALS: implementation and evaluation of a new integration package", )) if "scale" in self.params.intensity: software_loop.add_row(( 2, 2, "DIALS", dials_version, "program", "data scaling", "Data scaling and merging within the DIALS software package", )) citations_loop.add_row(( 2, "Acta Cryst. D", 76, 4, 385, 399, 2020, "Scaling diffraction data in the DIALS software package: algorithms and new approaches for multi-crystal scaling", )) cif_block.add_loop(software_loop) cif_block.add_loop(citations_loop) # Hard coding X-ray if self.params.mmcif.pdb_version == "v5_next": cif_block["_pdbx_diffrn_data_section.id"] = "dials" cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray" cif_block["_pdbx_diffrn_data_section.type_merged"] = "false" cif_block["_pdbx_diffrn_data_section.type_scaled"] = str( "scale" in self.params.intensity).lower() # FIXME finish metadata addition - detector and source details needed # http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html # Add source information; # _diffrn_source.pdbx_wavelength_list = (list of wavelengths) # _diffrn_source.source = (general class of source e.g. synchrotron) # _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04) wls = [] epochs = [] for exp in experiments: wls.append(round(exp.beam.get_wavelength(), 5)) epochs.append(exp.scan.get_epochs()[0]) unique_wls = set(wls) cif_block["_exptl_crystal.id"] = 1 # links to crystal_id cif_block["_diffrn.id"] = 1 # links to diffrn_id cif_block["_diffrn.crystal_id"] = 1 cif_block["_diffrn_source.diffrn_id"] = 1 cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join( str(w) for w in unique_wls) # Add detector information; # _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc) # _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd) # _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M) # One date is required, so if multiple just use the first date. min_epoch = min(epochs) date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch)) cif_block["_diffrn_detector.diffrn_id"] = 1 cif_block["_diffrn_detector.pdbx_collection_date"] = date_str # Write reflection data # Required columns header = ( "_pdbx_diffrn_unmerged_refln.reflection_id", "_pdbx_diffrn_unmerged_refln.scan_id", "_pdbx_diffrn_unmerged_refln.image_id_begin", "_pdbx_diffrn_unmerged_refln.image_id_end", "_pdbx_diffrn_unmerged_refln.index_h", "_pdbx_diffrn_unmerged_refln.index_k", "_pdbx_diffrn_unmerged_refln.index_l", ) extra_items = { "scales": ("_pdbx_diffrn_unmerged_refln.scale_value", "%5.3f"), "intensity.scale.value": ( "_pdbx_diffrn_unmerged_refln.intensity_meas", "%8.3f", ), "intensity.scale.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_sigma", "%8.3f", ), "intensity.sum.value": ( "_pdbx_diffrn_unmerged_refln.intensity_sum", "%8.3f", ), "intensity.sum.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_sum_sigma", "%8.3f", ), "intensity.prf.value": ( "_pdbx_diffrn_unmerged_refln.intensity_prf", "%8.3f", ), "intensity.prf.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_prf_sigma", "%8.3f", ), "angle": ("_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "%7.4f"), "partiality": ("_pdbx_diffrn_unmerged_refln.partiality", "%7.4f"), } variables_present = [] if "scale" in self.params.intensity: reflections["scales"] = 1.0 / reflections["inverse_scale_factor"] reflections["intensity.scale.sigma"] = flex.sqrt( reflections["intensity.scale.variance"]) variables_present.extend( ["scales", "intensity.scale.value", "intensity.scale.sigma"]) if "sum" in self.params.intensity: reflections["intensity.sum.sigma"] = flex.sqrt( reflections["intensity.sum.variance"]) variables_present.extend( ["intensity.sum.value", "intensity.sum.sigma"]) if "profile" in self.params.intensity: reflections["intensity.prf.sigma"] = flex.sqrt( reflections["intensity.prf.variance"]) variables_present.extend( ["intensity.prf.value", "intensity.prf.sigma"]) # Should always exist reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG variables_present.extend(["angle"]) if "partiality" in reflections: variables_present.extend(["partiality"]) for name in variables_present: if name in reflections: header += (extra_items[name][0], ) self._fmt += " " + extra_items[name][1] if "scale" in self.params.intensity: # Write dataset_statistics - first make a miller array crystal_symmetry = cctbxcrystal.symmetry( space_group=experiments[0].crystal.get_space_group(), unit_cell=experiments[0].crystal.get_unit_cell(), ) miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=reflections["miller_index"], anomalous_flag=False, ) i_obs = miller.array(miller_set, data=reflections["intensity.scale.value"]) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas(reflections["intensity.scale.sigma"]) i_obs.set_info( miller.array_info(source="DIALS", source_type="reflection_tables")) result = dataset_statistics( i_obs=i_obs, crystal_symmetry=crystal_symmetry, use_internal_variance=False, eliminate_sys_absent=False, assert_is_not_unique_set_under_symmetry=False, ) merged_block = iotbx.cif.model.block() merged_block["_reflns.pdbx_ordinal"] = 1 merged_block["_reflns.pdbx_diffrn_id"] = 1 merged_block["_reflns.entry_id"] = "DIALS" merged_data = result.as_cif_block() merged_block.update(merged_data) cif_block.update(merged_block) # Write the crystal information # if v5, that's all so return if self.params.mmcif.pdb_version == "v5": return cif_block # continue if v5_next cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_unmerged_cell.ordinal", "_pdbx_diffrn_unmerged_cell.crystal_id", "_pdbx_diffrn_unmerged_cell.wavelength", "_pdbx_diffrn_unmerged_cell.cell_length_a", "_pdbx_diffrn_unmerged_cell.cell_length_b", "_pdbx_diffrn_unmerged_cell.cell_length_c", "_pdbx_diffrn_unmerged_cell.cell_angle_alpha", "_pdbx_diffrn_unmerged_cell.cell_angle_beta", "_pdbx_diffrn_unmerged_cell.cell_angle_gamma", "_pdbx_diffrn_unmerged_cell.Bravais_lattice", )) crystals = experiments.crystals() crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)} for i, exp in enumerate(experiments): crystal = exp.crystal crystal_id = crystal_to_id[crystal] wavelength = exp.beam.get_wavelength() a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters() latt_type = str( bravais_types.bravais_lattice(group=crystal.get_space_group())) cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha, beta, gamma, latt_type)) cif_block.add_loop(cif_loop) # Write the scan information cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_scan.scan_id", "_pdbx_diffrn_scan.crystal_id", "_pdbx_diffrn_scan.image_id_begin", "_pdbx_diffrn_scan.image_id_end", "_pdbx_diffrn_scan.scan_angle_begin", "_pdbx_diffrn_scan.scan_angle_end", )) expid_to_scan_id = { exp.identifier: i + 1 for i, exp in enumerate(experiments) } for i, exp in enumerate(experiments): scan = exp.scan crystal_id = crystal_to_id[exp.crystal] image_range = scan.get_image_range() osc_range = scan.get_oscillation_range(deg=True) cif_loop.add_row(( i + 1, crystal_id, image_range[0], image_range[1], osc_range[0], osc_range[1], )) cif_block.add_loop(cif_loop) _, _, _, _, z0, z1 = reflections["bbox"].parts() h, k, l = [ hkl.iround() for hkl in reflections["miller_index"].as_vec3_double().parts() ] # make scan id consistent with header as defined above scan_id = flex.int(reflections.size(), 0) for id_ in reflections.experiment_identifiers().keys(): expid = reflections.experiment_identifiers()[id_] sel = reflections["id"] == id_ scan_id.set_selected(sel, expid_to_scan_id[expid]) loop_values = [ flex.size_t_range(1, len(reflections) + 1), scan_id, z0, z1, h, k, l, ] + [reflections[name] for name in variables_present] cif_loop = iotbx.cif.model.loop(data=dict(zip(header, loop_values))) cif_block.add_loop(cif_loop) return cif_block
def exercise_miller_array_as_cns_hkl(): s = StringIO() crystal_symmetry = crystal.symmetry() for anomalous_flag in [False, True]: miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=flex.miller_index([(1,2,3),(-3,5,-7)]), anomalous_flag=anomalous_flag) for data in [flex.bool((False,True)), flex.int((-3,4)), flex.double((10,13)), flex.complex_double((10,13)), flex.hendrickson_lattman(((0,1,2,3),(4,5,6,7)))]: miller_array = miller_set.array(data=data) miller_array.export_as_cns_hkl(file_object=s) if (isinstance(data, flex.double)): miller_array = miller_set.array(data=data, sigmas=data/10.) miller_array.export_as_cns_hkl(file_object=s) assert not show_diff(s.getvalue(), """\ NREFlections=2 ANOMalous=FALSe DECLare NAME=DATA DOMAin=RECIprocal TYPE=INTEger END INDEx 1 2 3 DATA= 0 INDEx -3 5 -7 DATA= 1 NREFlections=2 ANOMalous=FALSe DECLare NAME=DATA DOMAin=RECIprocal TYPE=INTEger END INDEx 1 2 3 DATA= -3 INDEx -3 5 -7 DATA= 4 NREFlections=2 ANOMalous=FALSe DECLare NAME=DATA DOMAin=RECIprocal TYPE=REAL END INDEx 1 2 3 DATA= 10 INDEx -3 5 -7 DATA= 13 NREFlections=2 ANOMalous=FALSe DECLare NAME=FOBS DOMAin=RECIprocal TYPE=REAL END DECLare NAME=SIGMA DOMAin=RECIprocal TYPE=REAL END INDEx 1 2 3 FOBS= 10 SIGMA= 1 INDEx -3 5 -7 FOBS= 13 SIGMA= 1.3 NREFlections=2 ANOMalous=FALSe DECLare NAME=F DOMAin=RECIprocal TYPE=COMPLEX END INDEx 1 2 3 F= 10 0 INDEx -3 5 -7 F= 13 0 NREFlections=2 ANOMalous=FALSe DECLare NAME=PA DOMAin=RECIprocal TYPE=REAL END DECLare NAME=PB DOMAin=RECIprocal TYPE=REAL END DECLare NAME=PC DOMAin=RECIprocal TYPE=REAL END DECLare NAME=PD DOMAin=RECIprocal TYPE=REAL END GROUp TYPE=HL OBJEct=PA OBJEct=PB OBJEct=PC OBJEct=PD END INDEx 1 2 3 PA= 0 PB= 1 PC= 2 PD= 3 INDEx -3 5 -7 PA= 4 PB= 5 PC= 6 PD= 7 NREFlections=2 ANOMalous=TRUE DECLare NAME=DATA DOMAin=RECIprocal TYPE=INTEger END INDEx 1 2 3 DATA= 0 INDEx -3 5 -7 DATA= 1 NREFlections=2 ANOMalous=TRUE DECLare NAME=DATA DOMAin=RECIprocal TYPE=INTEger END INDEx 1 2 3 DATA= -3 INDEx -3 5 -7 DATA= 4 NREFlections=2 ANOMalous=TRUE DECLare NAME=DATA DOMAin=RECIprocal TYPE=REAL END INDEx 1 2 3 DATA= 10 INDEx -3 5 -7 DATA= 13 NREFlections=2 ANOMalous=TRUE DECLare NAME=FOBS DOMAin=RECIprocal TYPE=REAL END DECLare NAME=SIGMA DOMAin=RECIprocal TYPE=REAL END INDEx 1 2 3 FOBS= 10 SIGMA= 1 INDEx -3 5 -7 FOBS= 13 SIGMA= 1.3 NREFlections=2 ANOMalous=TRUE DECLare NAME=F DOMAin=RECIprocal TYPE=COMPLEX END INDEx 1 2 3 F= 10 0 INDEx -3 5 -7 F= 13 0 NREFlections=2 ANOMalous=TRUE DECLare NAME=PA DOMAin=RECIprocal TYPE=REAL END DECLare NAME=PB DOMAin=RECIprocal TYPE=REAL END DECLare NAME=PC DOMAin=RECIprocal TYPE=REAL END DECLare NAME=PD DOMAin=RECIprocal TYPE=REAL END GROUp TYPE=HL OBJEct=PA OBJEct=PB OBJEct=PC OBJEct=PD END INDEx 1 2 3 PA= 0 PB= 1 PC= 2 PD= 3 INDEx -3 5 -7 PA= 4 PB= 5 PC= 6 PD= 7 """)
def scaled_data_as_miller_array( reflection_table_list, experiments, best_unit_cell=None, anomalous_flag=False, wavelength=None, ): """Get a scaled miller array from an experiment and reflection table.""" if len(reflection_table_list) > 1: joint_table = flex.reflection_table() for reflection_table in reflection_table_list: # better to just create many miller arrays and join them? refl_for_joint_table = flex.reflection_table() for col in [ "miller_index", "intensity.scale.value", "inverse_scale_factor", "intensity.scale.variance", ]: refl_for_joint_table[col] = reflection_table[col] good_refl_sel = ~reflection_table.get_flags( reflection_table.flags.bad_for_scaling, all=False) refl_for_joint_table = refl_for_joint_table.select(good_refl_sel) joint_table.extend(refl_for_joint_table) else: reflection_table = reflection_table_list[0] good_refl_sel = ~reflection_table.get_flags( reflection_table.flags.bad_for_scaling, all=False) joint_table = reflection_table.select(good_refl_sel) # Filter out negative scale factors to avoid merging statistics errors. # These are not removed from the output data, as it is likely one would # want to do further analysis e.g. delta cc1/2 and rescaling, to exclude # certain data and get better scale factors for all reflections. pos_scales = joint_table["inverse_scale_factor"] > 0 if pos_scales.count(False) > 0: logger.info( """There are %s reflections with non-positive scale factors which will not be used for calculating merging statistics""", pos_scales.count(False), ) joint_table = joint_table.select(pos_scales) if best_unit_cell is None: best_unit_cell = determine_best_unit_cell(experiments) miller_set = miller.set( crystal_symmetry=crystal.symmetry( unit_cell=best_unit_cell, space_group=experiments[0].crystal.get_space_group(), assert_is_compatible_unit_cell=False, ), indices=joint_table["miller_index"], anomalous_flag=anomalous_flag, ) i_obs = miller.array( miller_set, data=joint_table["intensity.scale.value"] / joint_table["inverse_scale_factor"], ) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas( flex.sqrt(joint_table["intensity.scale.variance"]) / joint_table["inverse_scale_factor"]) if not wavelength: wavelength = np.mean( [expt.beam.get_wavelength() for expt in experiments]) i_obs.set_info( miller.array_info( source="DIALS", source_type="reflection_tables", wavelength=wavelength, )) return i_obs
def write(self, experiments, reflections): """ Write the experiments and reflections to file """ # if mmcif filename is auto, then choose scaled.cif or integrated.cif if self.params.mmcif.hklout in (None, Auto, "auto"): if ("intensity.scale.value" in reflections) and ("intensity.scale.variance" in reflections): filename = "scaled.cif" logger.info( "Data appears to be scaled, setting mmcif.hklout = 'scaled.cif'" ) else: filename = "integrated.cif" logger.info( "Data appears to be unscaled, setting mmcif.hklout = 'integrated.cif'" ) else: filename = self.params.mmcif.hklout # Select reflections selection = reflections.get_flags(reflections.flags.integrated, all=True) reflections = reflections.select(selection) # Filter out bad variances and other issues, but don't filter on ice rings # or alter partialities. # Assumes you want to apply the lp and dqe corrections to sum and prf # Do we want to combine partials? reflections = filter_reflection_table( reflections, self.params.intensity, combine_partials=False, partiality_threshold=0.0, d_min=self.params.mtz.d_min, ) # Get the cif block cif_block = iotbx.cif.model.block() # Audit trail dials_version = dials.util.version.dials_version() cif_block["_audit.creation_method"] = dials_version cif_block["_audit.creation_date"] = datetime.date.today().isoformat() cif_block["_computing.data_reduction"] = ( "%s (Winter, G. et al., 2018)" % dials_version) cif_block[ "_publ.section_references"] = "Winter, G. et al. (2018) Acta Cryst. D74, 85-97." if "scale" in self.params.intensity: cif_block[ "_publ.section_references"] += "\nBeilsten-Edmands, J. et al. (2020) Acta Cryst. D76, 385-399." # Hard coding X-ray cif_block["_pdbx_diffrn_data_section.id"] = "dials" cif_block["_pdbx_diffrn_data_section.type_scattering"] = "x-ray" cif_block["_pdbx_diffrn_data_section.type_merged"] = "false" cif_block["_pdbx_diffrn_data_section.type_scaled"] = str( "scale" in self.params.intensity).lower() # FIXME finish metadata addition - detector and source details needed # http://mmcif.wwpdb.org/dictionaries/mmcif_pdbx_v50.dic/Categories/index.html # Add source information; # _diffrn_source.pdbx_wavelength_list = (list of wavelengths) # _diffrn_source.source = (general class of source e.g. synchrotron) # _diffrn_source.type = (specific beamline or instrument e.g DIAMOND BEAMLINE I04) wls = [] epochs = [] for exp in experiments: wls.append(round(exp.beam.get_wavelength(), 5)) epochs.append(exp.scan.get_epochs()[0]) unique_wls = set(wls) cif_block["_diffrn_source.pdbx_wavelength_list"] = ", ".join( str(w) for w in unique_wls) # Add detector information; # _diffrn_detector.detector = (general class e.g. PIXEL, PLATE etc) # _diffrn_detector.pdbx_collection_date = (Date of collection yyyy-mm-dd) # _diffrn_detector.type = (full name of detector e.g. DECTRIS PILATUS3 2M) # One date is required, so if multiple just use the first date. min_epoch = min(epochs) date_str = time.strftime("%Y-%m-%d", time.gmtime(min_epoch)) cif_block["_diffrn_detector.pdbx_collection_date"] = date_str # Write the crystal information cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_unmerged_cell.ordinal", "_pdbx_diffrn_unmerged_cell.crystal_id", "_pdbx_diffrn_unmerged_cell.wavelength", "_pdbx_diffrn_unmerged_cell.cell_length_a", "_pdbx_diffrn_unmerged_cell.cell_length_b", "_pdbx_diffrn_unmerged_cell.cell_length_c", "_pdbx_diffrn_unmerged_cell.cell_angle_alpha", "_pdbx_diffrn_unmerged_cell.cell_angle_beta", "_pdbx_diffrn_unmerged_cell.cell_angle_gamma", "_pdbx_diffrn_unmerged_cell.Bravais_lattice", )) crystals = experiments.crystals() crystal_to_id = {crystal: i + 1 for i, crystal in enumerate(crystals)} for i, exp in enumerate(experiments): crystal = exp.crystal crystal_id = crystal_to_id[crystal] wavelength = exp.beam.get_wavelength() a, b, c, alpha, beta, gamma = crystal.get_unit_cell().parameters() latt_type = str( bravais_types.bravais_lattice(group=crystal.get_space_group())) cif_loop.add_row((i + 1, crystal_id, wavelength, a, b, c, alpha, beta, gamma, latt_type)) cif_block.add_loop(cif_loop) # Write the scan information cif_loop = iotbx.cif.model.loop(header=( "_pdbx_diffrn_scan.scan_id", "_pdbx_diffrn_scan.crystal_id", "_pdbx_diffrn_scan.image_id_begin", "_pdbx_diffrn_scan.image_id_end", "_pdbx_diffrn_scan.scan_angle_begin", "_pdbx_diffrn_scan.scan_angle_end", )) for i, exp in enumerate(experiments): scan = exp.scan crystal_id = crystal_to_id[exp.crystal] image_range = scan.get_image_range() osc_range = scan.get_oscillation_range(deg=True) cif_loop.add_row(( i + 1, crystal_id, image_range[0], image_range[1], osc_range[0], osc_range[1], )) cif_block.add_loop(cif_loop) # Make a dict of unit_cell parameters unit_cell_parameters = {} if crystal.num_scan_points > 1: for i in range(crystal.num_scan_points): a, b, c, alpha, beta, gamma = crystal.get_unit_cell_at_scan_point( i).parameters() unit_cell_parameters[i] = (a, b, c, alpha, beta, gamma) else: unit_cell_parameters[0] = (a, b, c, alpha, beta, gamma) # Write reflection data # Required columns header = ( "_pdbx_diffrn_unmerged_refln.reflection_id", "_pdbx_diffrn_unmerged_refln.scan_id", "_pdbx_diffrn_unmerged_refln.image_id_begin", "_pdbx_diffrn_unmerged_refln.image_id_end", "_pdbx_diffrn_unmerged_refln.index_h", "_pdbx_diffrn_unmerged_refln.index_k", "_pdbx_diffrn_unmerged_refln.index_l", ) extra_items = { "scales": ("_pdbx_diffrn_unmerged_refln.scale_value", "%5.3f"), "intensity.scale.value": ( "_pdbx_diffrn_unmerged_refln.intensity_meas", "%8.3f", ), "intensity.scale.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_sigma", "%8.3f", ), "intensity.sum.value": ( "_pdbx_diffrn_unmerged_refln.intensity_sum", "%8.3f", ), "intensity.sum.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_sum_sigma", "%8.3f", ), "intensity.prf.value": ( "_pdbx_diffrn_unmerged_refln.intensity_prf", "%8.3f", ), "intensity.prf.sigma": ( "_pdbx_diffrn_unmerged_refln.intensity_prf_sigma", "%8.3f", ), "angle": ("_pdbx_diffrn_unmerged_refln.scan_angle_reflection", "%7.4f"), "partiality": ("_pdbx_diffrn_unmerged_refln.partiality", "%7.4f"), } fmt = "%6i %2i %5i %5i %-2i %-2i %-2i" variables_present = [] if "scale" in self.params.intensity: reflections["scales"] = 1.0 / reflections["inverse_scale_factor"] reflections["intensity.scale.sigma"] = flex.sqrt( reflections["intensity.scale.variance"]) variables_present.extend( ["scales", "intensity.scale.value", "intensity.scale.sigma"]) if "sum" in self.params.intensity: reflections["intensity.sum.sigma"] = flex.sqrt( reflections["intensity.sum.variance"]) variables_present.extend( ["intensity.sum.value", "intensity.sum.sigma"]) if "profile" in self.params.intensity: reflections["intensity.prf.sigma"] = flex.sqrt( reflections["intensity.prf.variance"]) variables_present.extend( ["intensity.prf.value", "intensity.prf.sigma"]) # Should always exist reflections["angle"] = reflections["xyzcal.mm"].parts()[2] * RAD2DEG variables_present.extend(["angle"]) if "partiality" in reflections: variables_present.extend(["partiality"]) for name in variables_present: if name in reflections: header += (extra_items[name][0], ) fmt += " " + extra_items[name][1] loop_format_strings = {"_pdbx_diffrn_unmerged_refln": fmt} if "scale" in self.params.intensity: # Write dataset_statistics - first make a miller array crystal_symmetry = cctbxcrystal.symmetry( space_group=experiments[0].crystal.get_space_group(), unit_cell=experiments[0].crystal.get_unit_cell(), ) miller_set = miller.set( crystal_symmetry=crystal_symmetry, indices=reflections["miller_index"], anomalous_flag=False, ) i_obs = miller.array(miller_set, data=reflections["intensity.scale.value"]) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas(reflections["intensity.scale.sigma"]) i_obs.set_info( miller.array_info(source="DIALS", source_type="reflection_tables")) result = dataset_statistics( i_obs=i_obs, crystal_symmetry=crystal_symmetry, use_internal_variance=False, eliminate_sys_absent=False, ) cif_block.update(result.as_cif_block()) _, _, _, _, z0, z1 = reflections["bbox"].parts() h, k, l = [ hkl.iround() for hkl in reflections["miller_index"].as_vec3_double().parts() ] loop_values = [ flex.size_t_range(1, len(reflections) + 1), reflections["id"] + 1, z0, z1, h, k, l, ] + [reflections[name] for name in variables_present] cif_loop = iotbx.cif.model.loop(data=dict(zip(header, loop_values))) cif_block.add_loop(cif_loop) # Add the block self._cif["dials"] = cif_block # Print to file if self.params.mmcif.compress and not filename.endswith( "." + self.params.mmcif.compress): filename += "." + self.params.mmcif.compress if self.params.mmcif.compress == "gz": open_fn = gzip.open elif self.params.mmcif.compress == "bz2": open_fn = bz2.open elif self.params.mmcif.compress == "xz": open_fn = lzma.open else: open_fn = open with open_fn(filename, "wt") as fh: self._cif.show(out=fh, loop_format_strings=loop_format_strings) # Log logger.info("Wrote reflections to %s" % filename)
def rho_stats( xray_structure, d_min, resolution_factor, electron_sum_radius, zero_out_f000): n_real = [] n_half_plus = [] n_half_minus = [] s2 = d_min * resolution_factor * 2 for l in xray_structure.unit_cell().parameters()[:3]: nh = ifloor(l / s2) n_real.append(2*nh+1) n_half_plus.append(nh) n_half_minus.append(-nh) n_real = tuple(n_real) n_real_product = matrix.col(n_real).product() crystal_gridding = maptbx.crystal_gridding( unit_cell=xray_structure.unit_cell(), space_group_info=xray_structure.space_group_info(), pre_determined_n_real=n_real) miller_indices = flex.miller_index() miller_indices.reserve(n_real_product) for h in flex.nested_loop(n_half_minus, n_half_plus, open_range=False): miller_indices.append(h) assert miller_indices.size() == n_real_product # miller_set = miller.set( crystal_symmetry=xray_structure, anomalous_flag=True, indices=miller_indices).sort(by_value="resolution") assert miller_set.indices()[0] == (0,0,0) f_calc = miller_set.structure_factors_from_scatterers( xray_structure=xray_structure, algorithm="direct", cos_sin_table=False).f_calc() if (zero_out_f000): f_calc.data()[0] = 0j # unit_cell_volume = xray_structure.unit_cell().volume() voxel_volume = unit_cell_volume / n_real_product number_of_miller_indices = [] rho_max = [] electron_sums_around_atoms = [] densities_along_x = [] for f in [f_calc, f_calc.resolution_filter(d_min=d_min)]: assert f.indices()[0] == (0,0,0) number_of_miller_indices.append(f.indices().size()) fft_map = miller.fft_map( crystal_gridding=crystal_gridding, fourier_coefficients=f) assert fft_map.n_real() == n_real rho = fft_map.real_map_unpadded() / unit_cell_volume assert approx_equal(voxel_volume*flex.sum(rho), f_calc.data()[0]) if (xray_structure.scatterers().size() == 1): assert flex.max_index(rho) == 0 rho_max.append(rho[0]) else: rho_max.append(flex.max(rho)) site_cart = xray_structure.sites_cart()[0] gias = maptbx.grid_indices_around_sites( unit_cell=xray_structure.unit_cell(), fft_n_real=n_real, fft_m_real=n_real, sites_cart=flex.vec3_double([site_cart]), site_radii=flex.double([electron_sum_radius])) electron_sums_around_atoms.append( flex.sum(rho.as_1d().select(gias))*voxel_volume) # a = xray_structure.unit_cell().parameters()[0] nx = n_real[0] nxh = nx//2 x = [] y = [] for ix in xrange(-nxh,nxh+1): x.append(a*ix/nx) y.append(rho[(ix%nx,0,0)]) densities_along_x.append((x,y)) # print \ "%3.1f %4.2f %-12s %5d %5d | %6.3f %6.3f | %6.3f %6.3f | %4.2f %5.1f" % ( d_min, resolution_factor, n_real, number_of_miller_indices[0], number_of_miller_indices[1], electron_sums_around_atoms[0], electron_sums_around_atoms[1], rho_max[0], rho_max[1], f_calc.data()[0].real, u_as_b(xray_structure.scatterers()[0].u_iso)) # return densities_along_x