def exercise_map_to_asu(sg_symbol): sg_type = sgtbx.space_group_type(sg_symbol) index_abs_range = (4,4,4) for anomalous_flag in (False,True): m = miller.index_generator( sg_type, anomalous_flag, index_abs_range).to_array() a = flex.double() p = flex.double() c = flex.hendrickson_lattman() for i in range(m.size()): a.append(random.random()) p.append(random.random() * 2) c.append([random.random() for j in range(4)]) f = flex.polar(a, p) p = [p, p*(180/math.pi)] m_random = flex.miller_index() p_random = [flex.double(), flex.double()] c_random = flex.hendrickson_lattman() f_random = flex.complex_double() for i,h_asym in enumerate(m): h_eq = miller.sym_equiv_indices(sg_type.group(), h_asym) i_eq = random.randrange(h_eq.multiplicity(anomalous_flag)) h_i = h_eq(i_eq) m_random.append(h_i.h()) for deg in (False,True): p_random[deg].append(h_i.phase_eq(p[deg][i], deg)) f_random.append(h_i.complex_eq(f[i])) c_random.append(h_i.hendrickson_lattman_eq(c[i])) m_random_copy = m_random.deep_copy() miller.map_to_asu(sg_type, anomalous_flag, m_random_copy) for i,h_asym in enumerate(m): assert h_asym == m_random_copy[i] m_random_copy = m_random.deep_copy() miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, f_random) for i,h_asym in enumerate(m): assert h_asym == m_random_copy[i] for i,f_asym in enumerate(f): assert abs(f_asym - f_random[i]) < 1.e-6 m_random_copy = m_random.deep_copy() a_random = a.deep_copy() miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, a_random) for i,h_asym in enumerate(m): assert h_asym == m_random_copy[i] for i,a_asym in enumerate(a): assert a_asym == a_random[i] for deg in (False,True): m_random_copy = m_random.deep_copy() miller.map_to_asu( sg_type, anomalous_flag, m_random_copy, p_random[deg], deg) for i,h_asym in enumerate(m): assert h_asym == m_random_copy[i] for i,p_asym in enumerate(p[deg]): assert scitbx.math.phase_error(p_asym, p_random[deg][i], deg) < 1.e-5 m_random_copy = m_random.deep_copy() miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, c_random) for i,h_asym in enumerate(m): assert h_asym == m_random_copy[i] for i,c_asym in enumerate(c): for j in range(4): assert abs(c_asym[j] - c_random[i][j]) < 1.e-5
def get_hkl_xyz_isigi(mtz_file): m = mtz.object(mtz_file) r = get_phi_range(m) sg = m.space_group() xdet_col = m.get_column('XDET') ydet_col = m.get_column('YDET') rot_col = m.get_column('ROT') i_col = m.get_column('I') sigi_col = m.get_column('SIGI') mi_s = m.extract_miller_indices() map_to_asu(sg.type(), False, mi_s) xdet_s = xdet_col.extract_values(not_a_number_substitute=0.0) ydet_s = ydet_col.extract_values(not_a_number_substitute=0.0) rot_s = rot_col.extract_values(not_a_number_substitute=0.0) i_s = i_col.extract_values(not_a_number_substitute=0.0) sigi_s = sigi_col.extract_values(not_a_number_substitute=0.0) hkl_xyz_isigi = [] # N.B. swapping X, Y here for j in range(mi_s.size()): hkl_xyz_isigi.append( ((mi_s[j][0], mi_s[j][1], mi_s[j][2]), (ydet_s[j], xdet_s[j], rot_s[j] / r), (i_s[j], sigi_s[j]))) return hkl_xyz_isigi
def get_hkl_xyz_isigi(mtz_file): m = mtz.object(mtz_file) r = get_phi_range(m) sg = m.space_group() xdet_col = m.get_column('XDET') ydet_col = m.get_column('YDET') rot_col = m.get_column('ROT') i_col = m.get_column('I') sigi_col = m.get_column('SIGI') mi_s = m.extract_miller_indices() map_to_asu(sg.type(), False, mi_s) xdet_s = xdet_col.extract_values(not_a_number_substitute = 0.0) ydet_s = ydet_col.extract_values(not_a_number_substitute = 0.0) rot_s = rot_col.extract_values(not_a_number_substitute = 0.0) i_s = i_col.extract_values(not_a_number_substitute = 0.0) sigi_s = sigi_col.extract_values(not_a_number_substitute = 0.0) hkl_xyz_isigi = [] # N.B. swapping X, Y here for j in range(mi_s.size()): hkl_xyz_isigi.append(((mi_s[j][0], mi_s[j][1], mi_s[j][2]), (ydet_s[j], xdet_s[j], rot_s[j] / r), (i_s[j], sigi_s[j]))) return hkl_xyz_isigi
def exercise_map_to_asu(sg_symbol): sg_type = sgtbx.space_group_type(sg_symbol) index_abs_range = (4,4,4) for anomalous_flag in (False,True): m = miller.index_generator( sg_type, anomalous_flag, index_abs_range).to_array() a = flex.double() p = flex.double() c = flex.hendrickson_lattman() for i in xrange(m.size()): a.append(random.random()) p.append(random.random() * 2) c.append([random.random() for j in xrange(4)]) f = flex.polar(a, p) p = [p, p*(180/math.pi)] m_random = flex.miller_index() p_random = [flex.double(), flex.double()] c_random = flex.hendrickson_lattman() f_random = flex.complex_double() for i,h_asym in enumerate(m): h_eq = miller.sym_equiv_indices(sg_type.group(), h_asym) i_eq = random.randrange(h_eq.multiplicity(anomalous_flag)) h_i = h_eq(i_eq) m_random.append(h_i.h()) for deg in (False,True): p_random[deg].append(h_i.phase_eq(p[deg][i], deg)) f_random.append(h_i.complex_eq(f[i])) c_random.append(h_i.hendrickson_lattman_eq(c[i])) m_random_copy = m_random.deep_copy() miller.map_to_asu(sg_type, anomalous_flag, m_random_copy) for i,h_asym in enumerate(m): assert h_asym == m_random_copy[i] m_random_copy = m_random.deep_copy() miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, f_random) for i,h_asym in enumerate(m): assert h_asym == m_random_copy[i] for i,f_asym in enumerate(f): assert abs(f_asym - f_random[i]) < 1.e-6 m_random_copy = m_random.deep_copy() a_random = a.deep_copy() miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, a_random) for i,h_asym in enumerate(m): assert h_asym == m_random_copy[i] for i,a_asym in enumerate(a): assert a_asym == a_random[i] for deg in (False,True): m_random_copy = m_random.deep_copy() miller.map_to_asu( sg_type, anomalous_flag, m_random_copy, p_random[deg], deg) for i,h_asym in enumerate(m): assert h_asym == m_random_copy[i] for i,p_asym in enumerate(p[deg]): assert scitbx.math.phase_error(p_asym, p_random[deg][i], deg) < 1.e-5 m_random_copy = m_random.deep_copy() miller.map_to_asu(sg_type, anomalous_flag, m_random_copy, c_random) for i,h_asym in enumerate(m): assert h_asym == m_random_copy[i] for i,c_asym in enumerate(c): for j in xrange(4): assert abs(c_asym[j] - c_random[i][j]) < 1.e-5
def _compute_rij_wij(self, use_cache=True, use_super=False): if use_super: # for testing return super()._compute_rij_wij(use_cache=use_cache) def _compute_one_row(args): """ Call the compute_one_row method of CC, which is a compute_rij_wij_detail instance. Args is a tuple that we unpack because easy_mp.parallel_map can only pass a single argument. """ CC, i, NN = args rij_row, rij_col, rij_data, wij_row, wij_col, wij_data = [ list(x) for x in CC.compute_one_row(self._lattices.size, i) ] rij = sparse.coo_matrix((rij_data, (rij_row, rij_col)), shape=(NN, NN)) wij = sparse.coo_matrix((wij_data, (wij_row, wij_col)), shape=(NN, NN)) return rij, wij n_lattices = self._lattices.size n_sym_ops = len(self.sym_ops) NN = n_lattices * n_sym_ops lower_i = flex.int() upper_i = flex.int() for lidx in range(self._lattices.size): LL, UU = self._lattice_lower_upper_index(lidx) lower_i.append(int(LL)) if UU is None: UU = self._data.data().size() upper_i.append(int(UU)) indices = {} space_group_type = self._data.space_group().type() from xfel.merging import compute_rij_wij_detail CC = compute_rij_wij_detail(lower_i, upper_i, self._data.data(), self._min_pairs) for sym_op in self.sym_ops: cb_op = sgtbx.change_of_basis_op(sym_op) indices_reindexed = cb_op.apply(self._data.indices()) miller.map_to_asu(space_group_type, False, indices_reindexed) indices[cb_op.as_xyz()] = indices_reindexed CC.set_indices(cb_op, indices_reindexed) assert self._nproc == 1 results = map(_compute_one_row, [(CC, i, NN) for i in range(n_lattices)]) rij_matrix = None wij_matrix = None for (rij, wij) in results: if rij_matrix is None: rij_matrix = rij wij_matrix = wij else: rij_matrix += rij wij_matrix += wij self.rij_matrix = rij_matrix.todense().astype(np.float64) self.wij_matrix = wij_matrix.todense().astype(np.float64) return self.rij_matrix, self.wij_matrix
def reindex(self, reindex_operation): """Reindex the reflections by the given reindexing operation.""" R = rt_mx(reindex_operation).inverse() # first construct mapping table - native, anomalous map_native = {} hkls = flex.miller_index() for hkl in self._merged_reflections: Fhkl = R * hkl Rhkl = nint(Fhkl[0]), nint(Fhkl[1]), nint(Fhkl[2]) hkls.append(Rhkl) map_to_asu(self._mf.get_space_group().type(), False, hkls) for j, hkl in enumerate(self._merged_reflections): map_native[hkl] = hkls[j] map_anomalous = {} hkls = flex.miller_index() for hkl in self._merged_reflections_anomalous: Fhkl = R * hkl Rhkl = nint(Fhkl[0]), nint(Fhkl[1]), nint(Fhkl[2]) hkls.append(Rhkl) map_to_asu(self._mf.get_space_group().type(), True, hkls) for j, hkl in enumerate(self._merged_reflections_anomalous): map_anomalous[hkl] = hkls[j] # then remap the actual measurements merged_reflections = {} for hkl in self._merged_reflections: Rhkl = map_native[hkl] merged_reflections[Rhkl] = self._merged_reflections[hkl] self._merged_reflections = merged_reflections merged_reflections = {} for hkl in self._merged_reflections_anomalous: Rhkl = map_anomalous[hkl] merged_reflections[Rhkl] = self._merged_reflections_anomalous[hkl] self._merged_reflections_anomalous = merged_reflections unmerged_reflections = {} for hkl in self._unmerged_reflections: Rhkl = map_native[hkl] unmerged_reflections[Rhkl] = self._unmerged_reflections[hkl] self._unmerged_reflections = unmerged_reflections
def load_xds_or_dials(i, file_in, scout): print "Reading %5d %s" % (i, file_in if input_type == "xds_ascii" else file_in[1]) if input_type == "xds_ascii": tmp = get_data_from_xac(params, file_in) else: tmp = get_data_from_dials(params, file_in) miller.map_to_asu(sgtype, params.anomalous_flag, tmp.indices()) k, b = None, None if scale_ref is not None: k, b, cc = scale_data(tmp.indices(), tmp.data(), scale_ref, params.scaling.parameter, params.scaling.calc_cc) scout.append("%s %.4e %.4e %.4f\n" % (file_in if input_type == "xds_ascii" else file_in[1], k, b, cc)) #tmp.iobs *= k #tmp.sigma_iobs *= k #if b == b: # nan if not calculated # d_star_sq = tmp.symm.unit_cell().d_star_sq(tmp.indices) # tmp.iobs *= flex.exp(-b*d_star_sq) # tmp.sigma_iobs *= flex.exp(-b*d_star_sq) return i, tmp, k, b
def reduce_reflections_to_asu(space_group_number, indices): """Reduce reflection indices to asymmetric unit.""" from cctbx.sgtbx import space_group, space_group_symbols from cctbx.array_family import flex from cctbx.miller import map_to_asu sg = space_group(space_group_symbols(space_group_number).hall()) miller = flex.miller_index(indices) map_to_asu(sg.type(), False, miller) return [hkl for hkl in miller]
def reduce_reflections_to_asu(space_group_number, indices): '''Reduce reflection indices to asymmetric unit.''' from cctbx.sgtbx import space_group, space_group_symbols from cctbx.array_family import flex from cctbx.miller import map_to_asu sg = space_group(space_group_symbols(space_group_number).hall()) miller = flex.miller_index(indices) map_to_asu(sg.type(), False, miller) return [hkl for hkl in miller]
def exercise_change_basis_in_place(): from cctbx import sgtbx space_group_info = sgtbx.space_group_info(symbol='P4') unit_cell = space_group_info.any_compatible_unit_cell(volume=1000) miller_set = crystal.symmetry( unit_cell=unit_cell, space_group_info=space_group_info).build_miller_set( d_min=1, anomalous_flag=True) miller_set = miller_set.expand_to_p1().customized_copy( space_group_info=miller_set.space_group_info()) m = mtz.object() m.set_title('This is a title') m.set_space_group_info(miller_set.space_group_info()) x = m.add_crystal('XTAL', 'CCTBX', miller_set.unit_cell().parameters()) d = x.add_dataset('TEST', 1) indices = miller_set.indices() original_indices = indices.deep_copy() # map the miller indices to one hemisphere (i.e. just I+) miller.map_to_asu(m.space_group().type(), False, indices) h, k, l = [i.iround() for i in indices.as_vec3_double().parts()] m.adjust_column_array_sizes(len(h)) m.set_n_reflections(len(h)) # assign H, K, L d.add_column('H', 'H').set_values(h.as_double().as_float()) d.add_column('K', 'H').set_values(k.as_double().as_float()) d.add_column('L', 'H').set_values(l.as_double().as_float()) d.add_column('M_ISYM', 'Y').set_values(flex.float(len(indices))) b = m.add_batch() b.set_cell(flex.float(unit_cell.parameters())) b.set_umat( flex.float( (-0.9542511701583862, -0.1780465543270111, -0.2402169108390808, -0.13100790977478027, 0.9711279273033142, -0.19936780631542206, 0.26877808570861816, -0.1587766408920288, -0.9500254392623901))) m.change_basis_in_place(sgtbx.change_of_basis_op('-h,k,-l')) assert approx_equal( b.umat(), (0.9542511701583862, 0.1780465543270111, 0.2402169108390808, -0.13100790977478027, 0.9711279273033142, -0.19936780631542206, -0.26877808570861816, 0.1587766408920288, 0.9500254392623901))
def compare(a, b, sgtype): ind_a = a.indices() ind_b = b.indices() from cctbx.miller import map_to_asu map_to_asu(sgtype, False, ind_a) map_to_asu(sgtype, False, ind_b) _a, _b = a.common_sets(b, assert_is_similar_symmetry=False) if _a.size() < 5: return 0.0, _a.size() return cc(_a.data(), _b.data()), _a.size()
def read_xds_integrate(xds_integrate_file): # N.B. in here need to reduce indices to asymmetric unit sg_num = 0 r = 0.0 for record in open(xds_integrate_file): if not '!' in record[:1]: break if 'SPACE_GROUP_NUMBER' in record: sg_num = int(record.split()[-1]) if 'OSCILLATION_RANGE' in record: r = float(record.split()[-1]) if not sg_num: raise RuntimeError('spacegroup missing') if not r: raise RuntimeError('rotation missing') sg = space_group(space_group_symbols(sg_num).hall()) observations = [] hkls = flex.miller_index() xyzs = [] isigmas = [] for record in open(xds_integrate_file): if '!' in record[:1]: continue values = record.split() hkls.append([int(h) for h in values[:3]]) xyzs.append((float(values[5]), float(values[6]), float(values[7]))) isigmas.append([float(x) for x in values[3:5]]) map_to_asu(sg.type(), False, hkls) for hkl, xyz, isigma in zip(hkls, xyzs, isigmas): observations.append((hkl, xyz, isigma)) return observations
def read_xds_integrate(xds_integrate_file): # N.B. in here need to reduce indices to asymmetric unit sg_num = 0 r = 0.0 for record in open(xds_integrate_file): if not '!' in record[:1]: break if 'SPACE_GROUP_NUMBER' in record: sg_num = int(record.split()[-1]) if 'OSCILLATION_RANGE' in record: r = float(record.split()[-1]) if not sg_num: raise RuntimeError, 'spacegroup missing' if not r: raise RuntimeError, 'rotation missing' sg = space_group(space_group_symbols(sg_num).hall()) observations = [] hkls = flex.miller_index() xyzs = [] isigmas = [] for record in open(xds_integrate_file): if '!' in record[:1]: continue values = record.split() hkls.append(map(int, values[:3])) xyzs.append((float(values[5]), float(values[6]), float(values[7]))) isigmas.append(map(float, values[3:5])) map_to_asu(sg.type(), False, hkls) for hkl, xyz, isigma in zip(hkls, xyzs, isigmas): observations.append((hkl, xyz, isigma)) return observations
def add_asu_miller_indices_column(self, reflections): '''Add a "symmetry-reduced hkl" column to the reflection table''' if reflections.size() == 0: return import copy # Build target symmetry. The exact experiment unit cell values don't matter for converting HKLs to asu HKLs. target_unit_cell = self.params.scaling.unit_cell target_space_group_info = self.params.scaling.space_group target_symmetry = symmetry(unit_cell=target_unit_cell, space_group_info=target_space_group_info) target_space_group = target_symmetry.space_group() # generate and add an asu hkl column if 'miller_index_asymmetric' not in reflections: reflections['miller_index_asymmetric'] = copy.deepcopy(reflections['miller_index']) miller.map_to_asu(target_space_group.type(), not self.params.merging.merge_anomalous, reflections['miller_index_asymmetric'])
def add_asu_miller_indices_column(self, experiments, reflections): '''Add a "symmetry-reduced hkl" column to the reflection table''' if reflections.size() == 0: return from cctbx import miller from cctbx.crystal import symmetry import copy # build target space group target_unit_cell = self.params.scaling.unit_cell target_space_group_info = self.params.scaling.space_group target_symmetry = symmetry(unit_cell=target_unit_cell, space_group_info=target_space_group_info) target_space_group = target_symmetry.space_group() # generate and add an asu hkl column reflections['miller_index_asymmetric'] = copy.deepcopy(reflections['miller_index']) miller.map_to_asu(target_space_group.type(), not self.params.merging.merge_anomalous, reflections['miller_index_asymmetric'])
def load_xds(i, xac, scout): print "Reading %5d %s" %(i, xac) tmp = get_data_from_xac(params, xac) miller.map_to_asu(sgtype, params.anomalous_flag, tmp.indices) k, b = None, None if scale_ref is not None: k, b, cc = scale_data(tmp.indices, tmp.iobs, scale_ref, params.scaling.parameter, params.scaling.calc_cc) scout.append("%s %.4e %.4e %.4f\n" % (xac, k, b, cc)) #tmp.iobs *= k #tmp.sigma_iobs *= k #if b == b: # nan if not calculated # d_star_sq = tmp.symm.unit_cell().d_star_sq(tmp.indices) # tmp.iobs *= flex.exp(-b*d_star_sq) # tmp.sigma_iobs *= flex.exp(-b*d_star_sq) return i, tmp, k, b
def reindex_refl_by_coset(refl,data,symms,uuids,co,anomalous_flag = False, verbose=True): if verbose: cache_miller = refl["miller_index"].deep_copy() cache_asu = refl["miller_index_asymmetric"].deep_copy() for icoset,partition in enumerate(co.partitions): if icoset==0: continue # no change of basis cb_op = change_of_basis_op(partition[0]) mi_new = cb_op.apply(refl["miller_index"]) mi_asu_new = mi_new.deep_copy() miller.map_to_asu(symms[0].space_group().info().type(), anomalous_flag, mi_asu_new) # now select only those expts assigned to that coset good_refls = flex.bool(len(refl), False) all_expt_id = list(data["experiment"]) all_coset = list(data["coset"]) # would like to understand how to use pandas rather than Python list for iexpt in range(len(symms)): iexpt_id = uuids[iexpt] this_coset = all_coset[ all_expt_id.index(iexpt_id) ] if this_coset == icoset: good_refls |= refl["exp_id"] == iexpt_id re_millers = mi_new.select(good_refls) re_asu = mi_asu_new.select(good_refls) refl["miller_index"].set_selected(good_refls, re_millers) refl["miller_index_asymmetric"].set_selected(good_refls, re_asu) if verbose: for x in range(len(refl)): print ("%3d"%x,refl["exp_id"][x], all_coset[ all_expt_id.index(refl["exp_id"][x]) ], "%10s"%(change_of_basis_op(co.partitions[ all_coset[ all_expt_id.index(refl["exp_id"][x]) ] ][0]).as_hkl()), "(%4d%4d%4d)"%(cache_miller[x]), "(%4d%4d%4d)"%(cache_asu[x]), "(%4d%4d%4d)"%(refl["miller_index"][x]), "(%4d%4d%4d)"%(refl["miller_index_asymmetric"][x])) return refl
def map_hkl_list(Hi_lst, anomalous_flag=True, symbol="P43212"): sg_type = sgtbx.space_group_info(symbol=symbol).type() Hi_flex = flex.miller_index(tuple(map(tuple, Hi_lst))) miller.map_to_asu(sg_type, anomalous_flag, Hi_flex) return list(Hi_flex)
fast = flex.miller_index() data = flex.double() for r in reflections: # only copy across the ones which should be present if sg.is_sys_absent([h, k, l]): continue o, h, k, l = r data.append(o) fast.append([h, k, l]) map_to_asu(sg.type(), False, fast, data) # now unpack to original data structures for j in range(data.size()): d = data[j] hkl = fast[j] new_reflections.append((d, hkl[0], hkl[1], hkl[2])) # now do something with these symmetry reduced reflections... outfile = '%s.sym' % os.path.split(hkl_file)[-1][:-4] print 'Writing reflections to %s' % outfile fout = open(outfile, 'w')
def export_xds_ascii(integrated_data, experiment_list, hklout, summation=False, include_partials=False, keep_partials=False, var_model=(1, 0)): '''Export data from integrated_data corresponding to experiment_list to an XDS_ASCII.HKL formatted text file.''' from dials.array_family import flex # for the moment assume (and assert) that we will convert data from exactly # one lattice... assert (len(experiment_list) == 1) # select reflections that are assigned to an experiment (i.e. non-negative id) integrated_data = integrated_data.select(integrated_data['id'] >= 0) assert max(integrated_data['id']) == 0 if not summation: assert ('intensity.prf.value' in integrated_data) if 'intensity.prf.variance' in integrated_data: selection = integrated_data.get_flags(integrated_data.flags.integrated, all=True) else: selection = integrated_data.get_flags( integrated_data.flags.integrated_sum) integrated_data = integrated_data.select(selection) selection = integrated_data['intensity.sum.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d reflections with negative variance' % \ selection.count(True)) if 'intensity.prf.variance' in integrated_data: selection = integrated_data['intensity.prf.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d profile reflections with negative variance' % \ selection.count(True)) if include_partials: integrated_data = sum_partial_reflections(integrated_data) integrated_data = scale_partial_reflections(integrated_data) if 'partiality' in integrated_data: selection = integrated_data['partiality'] < 0.99 if selection.count(True) > 0 and not keep_partials: integrated_data.del_selected(selection) logger.info('Removing %d incomplete reflections' % \ selection.count(True)) experiment = experiment_list[0] # sort data before output nref = len(integrated_data['miller_index']) indices = flex.size_t_range(nref) import copy unique = copy.deepcopy(integrated_data['miller_index']) from cctbx.miller import map_to_asu map_to_asu(experiment.crystal.get_space_group().type(), False, unique) perm = sorted(indices, key=lambda k: unique[k]) integrated_data = integrated_data.select(flex.size_t(perm)) from scitbx import matrix from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame assert (not experiment.goniometer is None) unit_cell = experiment.crystal.get_unit_cell() from scitbx.array_family import flex from math import sqrt assert (not experiment.scan is None) image_range = experiment.scan.get_image_range() phi_start, phi_range = experiment.scan.get_image_oscillation( image_range[0]) # gather the required information for the reflection file nref = len(integrated_data['miller_index']) zdet = flex.double(integrated_data['xyzcal.px'].parts()[2]) miller_index = integrated_data['miller_index'] I = None sigI = None # export including scale factors if 'lp' in integrated_data: lp = integrated_data['lp'] else: lp = flex.double(nref, 1.0) if 'dqe' in integrated_data: dqe = integrated_data['dqe'] else: dqe = flex.double(nref, 1.0) scl = lp / dqe # profile correlation if 'profile.correlation' in integrated_data: prof_corr = 100.0 * integrated_data['profile.correlation'] else: prof_corr = flex.double(nref, 100.0) # partiality if 'partiality' in integrated_data: partiality = 100 * integrated_data['partiality'] else: prof_corr = flex.double(nref, 100.0) if summation: I = integrated_data['intensity.sum.value'] * scl V = integrated_data['intensity.sum.variance'] * scl * scl assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) else: I = integrated_data['intensity.prf.value'] * scl V = integrated_data['intensity.prf.variance'] * scl * scl assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) fout = open(hklout, 'w') # first write the header - in the "standard" coordinate frame... panel = experiment.detector[0] fast = panel.get_fast_axis() slow = panel.get_slow_axis() Rd = align_reference_frame(fast, (1, 0, 0), slow, (0, 1, 0)) print 'Coordinate change:' print '%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n' % Rd.elems fast = Rd * fast slow = Rd * slow qx, qy = panel.get_pixel_size() nx, ny = panel.get_image_size() distance = matrix.col(Rd * panel.get_origin()).dot( matrix.col(Rd * panel.get_normal())) org = Rd * (matrix.col(panel.get_origin()) - distance * matrix.col(panel.get_normal())) orgx = -org.dot(fast) / qx orgy = -org.dot(slow) / qy UB = Rd * matrix.sqr(experiment.crystal.get_A()) real_space_ABC = UB.inverse().elems axis = Rd * experiment.goniometer.get_rotation_axis() beam = Rd * experiment.beam.get_s0() cell_fmt = '%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f' axis_fmt = '%9.3f %9.3f %9.3f' fout.write('\n'.join([ '!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL\'S_LAW=TRUE', '!Generated by dials.export', '!DATA_RANGE= %d %d' % image_range, '!ROTATION_AXIS= %9.6f %9.6f %9.6f' % axis.elems, '!OSCILLATION_RANGE= %f' % phi_range, '!STARTING_ANGLE= %f' % phi_start, '!STARTING_FRAME= %d' % image_range[0], '!SPACE_GROUP_NUMBER= %d' % experiment.crystal.get_space_group().type().number(), '!UNIT_CELL_CONSTANTS= %s' % (cell_fmt % unit_cell.parameters()), '!UNIT_CELL_A-AXIS= %s' % (axis_fmt % real_space_ABC[0:3]), '!UNIT_CELL_B-AXIS= %s' % (axis_fmt % real_space_ABC[3:6]), '!UNIT_CELL_C-AXIS= %s' % (axis_fmt % real_space_ABC[6:9]), '!X-RAY_WAVELENGTH= %f' % experiment.beam.get_wavelength(), '!INCIDENT_BEAM_DIRECTION= %f %f %f' % beam.elems, '!NX= %d NY= %d QX= %f QY= %f' % (nx, ny, qx, qy), '!ORGX= %9.2f ORGY= %9.2f' % (orgx, orgy), '!DETECTOR_DISTANCE= %8.3f' % distance, '!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f' % fast.elems, '!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f' % slow.elems, '!VARIANCE_MODEL= %7.3e %7.3e' % var_model, '!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12', '!ITEM_H=1', '!ITEM_K=2', '!ITEM_L=3', '!ITEM_IOBS=4', '!ITEM_SIGMA(IOBS)=5', '!ITEM_XD=6', '!ITEM_YD=7', '!ITEM_ZD=8', '!ITEM_RLP=9', '!ITEM_PEAK=10', '!ITEM_CORR=11', '!ITEM_PSI=12', '!END_OF_HEADER', '' ])) # then write the data records s0 = Rd * matrix.col(experiment.beam.get_s0()) for j in range(nref): x, y, z = integrated_data['xyzcal.px'][j] phi = phi_start + z * phi_range h, k, l = miller_index[j] X = (UB * (h, k, l)).rotate(axis, phi, deg=True) s = s0 + X g = s.cross(s0).normalize() f = (s - s0).normalize() # find component of beam perpendicular to f, e e = -(s + s0).normalize() if h == k and k == l: u = (h, -h, 0) else: u = (k - l, l - h, h - k) q = (matrix.col(u).transpose() * UB.inverse()).normalize().transpose().rotate(axis, phi, deg=True) psi = q.angle(g, deg=True) if q.dot(e) < 0: psi *= -1 fout.write('%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n' % (h, k, l, I[j], sigI[j], x, y, z, scl[j], partiality[j], prof_corr[j], psi)) fout.write('!END_OF_DATA\n') fout.close() logger.info('Output %d reflections to %s' % (nref, hklout))
def reindex(self, reindex_operation): '''Reindex the reflections by the given reindexing operation.''' R = rt_mx(reindex_operation).inverse() # first construct mapping table - native, anomalous map_native = { } hkls = flex.miller_index() for hkl in self._merged_reflections: Fhkl = R * hkl Rhkl = nint(Fhkl[0]), nint(Fhkl[1]), nint(Fhkl[2]) hkls.append(Rhkl) map_to_asu(self._mf.get_space_group().type(), False, hkls) for j, hkl in enumerate(self._merged_reflections): map_native[hkl] = hkls[j] map_anomalous = { } hkls = flex.miller_index() for hkl in self._merged_reflections_anomalous: Fhkl = R * hkl Rhkl = nint(Fhkl[0]), nint(Fhkl[1]), nint(Fhkl[2]) hkls.append(Rhkl) map_to_asu(self._mf.get_space_group().type(), True, hkls) for j, hkl in enumerate(self._merged_reflections_anomalous): map_anomalous[hkl] = hkls[j] # then remap the actual measurements merged_reflections = { } for hkl in self._merged_reflections: Rhkl = map_native[hkl] merged_reflections[Rhkl] = self._merged_reflections[hkl] self._merged_reflections = merged_reflections merged_reflections = { } for hkl in self._merged_reflections_anomalous: Rhkl = map_anomalous[hkl] merged_reflections[Rhkl] = self._merged_reflections_anomalous[hkl] self._merged_reflections_anomalous = merged_reflections unmerged_reflections = { } for hkl in self._unmerged_reflections: Rhkl = map_native[hkl] unmerged_reflections[Rhkl] = self._unmerged_reflections[hkl] self._unmerged_reflections = unmerged_reflections return
def __init__( self, rtable, elist, calculate_variances=False, keep_singles=False, uncertainty="sigma", outfile=None, ): """ Generate z-scores and a normal probability plot from a DIALS reflection_table and a dxtbx ExperimentList, containing the observations and the corresponding experiments, respectively. :param rtable: A reflection table object, containing at least the columns * ``miller_index`` * ``intensity.sum.value`` * ``intensity.sum.variance`` * ``xyzobs.px.value`` :type rtable: dials.array_family_flex_ext.reflection_table :param elist: A corresponding experiment list. :type elist: dxtbx.model.ExperimentList :param calculate_variances: Choose whether to calculate weighted aggregate variances. Doing so incurs a performance penalty. Defaults to False. :type calculate_variances: bool :param keep_singles: Choose whether to keep multiplicity-1 reflections. Defaults to False. :type keep_singles: bool :param uncertainty: Measure of spread to use in normalising the z-scores, i.e. z = (I - <I>) / uncertainty. Possible values for uncertainty: * 'sigma': Use measured sigma values; * 'stddev': Use sample standard deviations calculated as square-root of unbiased weighted sample variances of symmetry-equivalent reflection intensities; Defaults to 'sigma'. :type uncertainty: str :param outfile: Filename root for output PNG plots. Defaults to None. :type: outfile: str """ if not isinstance(rtable, flex.reflection_table) or not isinstance( elist, ExperimentList): raise TypeError( "Must be called with a reflection table and an experiment list." ) rtable = rtable.copy() # Discard unindexed reflections (only necessary because of # https://github.com/dials/dials/issues/615 — # TODO remove the line below when issue #615 is fixed). rtable.del_selected(rtable["id"] == -1) rtable["miller_index.asu"] = rtable["miller_index"] # Divide reflections by the space group to which they have been indexed. self.rtables = { expt.crystal.get_space_group().make_tidy(): flex.reflection_table() for expt in elist } for expt, sel in rtable.iterate_experiments_and_indices(elist): sg = expt.crystal.get_space_group().make_tidy() self.rtables[sg].extend(rtable.select(sel)) # Map Miller indices to asymmetric unit. for space_group, rtable in self.rtables.items(): # TODO Handle anomalous flag sensibly. Currently assumes not anomalous. miller.map_to_asu(space_group.type(), False, rtable["miller_index.asu"]) # Calculate normal probability plot data. self._multiplicity_mean_error_stddev( calculate_variances=calculate_variances, keep_singles=keep_singles) self._make_z(uncertainty) self._probplot_data() self.rtable = flex.reflection_table() for rtable in self.rtables.values(): self.rtable.extend(rtable) if not outfile: outfile = "" self.outfile = outfile
def _compute_rij_wij(self, use_cache=True): """Compute the rij_wij matrix. Rij is a symmetric matrix of size (n x m, n x m), where n is the number of datasets and m is the number of symmetry operations. It is composed of (m, m) blocks of size (n, n), where each block contains the correlation coefficients between cb_op_k applied to datasets 1..N with cb_op_kk applied to datasets 1.. N. If `use_cache=True`, then an optimisation is made to reflect the fact some elements of the matrix are equivalent, i.e.: CC[(a, cb_op_k), (b, cb_op_kk)] == CC[(a,), (b, cb_op_k.inverse() * cb_op_kk)] """ n_lattices = len(self._lattices) n_sym_ops = len(self.sym_ops) # Pre-calculate miller indices after application of each cb_op. Only calculate # this once per cb_op instead of on-the-fly every time we need it. indices = {} epsilons = {} space_group_type = self._data.space_group().type() for cb_op in self.sym_ops: cb_op = sgtbx.change_of_basis_op(cb_op) indices_reindexed = cb_op.apply(self._data.indices()) miller.map_to_asu(space_group_type, False, indices_reindexed) cb_op_str = cb_op.as_xyz() indices[cb_op_str] = np.array([ h.iround().as_numpy_array() for h in indices_reindexed.as_vec3_double().parts() ]).transpose() epsilons[cb_op_str] = self._patterson_group.epsilon( indices_reindexed).as_numpy_array() intensities = self._data.data().as_numpy_array() # Map indices to an array of flat 1d indices which can later be used for # matching pairs of indices offset = -np.min(np.concatenate(list(indices.values())), axis=0) dims = np.max(np.concatenate(list(indices.values())), axis=0) + offset + 1 for cb_op, hkl in indices.items(): indices[cb_op] = np.ravel_multi_index((hkl + offset).T, dims) # Create an empty 2D array of shape (m * n, L), where m is the number of sym # ops, n is the number of lattices, and L is the number of unique miller indices all_intensities = np.empty((n_sym_ops * n_lattices, np.prod(dims))) # Populate all_intensities with intensity values, filling absent intensities # with np.nan all_intensities.fill(np.nan) slices = np.append(self._lattices, intensities.size) slices = list(map(slice, slices[:-1], slices[1:])) for i, (mil_ind, eps) in enumerate(zip(indices.values(), epsilons.values())): for j, selection in enumerate(slices): # map (i, j) to a column in all_intensities column = np.ravel_multi_index((i, j), (n_sym_ops, n_lattices)) epsilon_equals_one = eps[selection] == 1 valid_mil_ind = mil_ind[selection][epsilon_equals_one] valid_intensities = intensities[selection][epsilon_equals_one] all_intensities[column, valid_mil_ind] = valid_intensities # Ideally we would use `np.ma.corrcoef` here, but it is broken, so use # pd.DataFrame.corr() instead (see numpy/numpy#15601) rij = (pd.DataFrame(all_intensities).T.dropna(how="all").corr( min_periods=self._min_pairs).values) # Set any NaN correlation coefficients to zero np.nan_to_num(rij, copy=False) # Cosym does not make use of the on-diagonal correlation coefficients np.fill_diagonal(rij, 0) if self._weights: wij = np.zeros_like(rij) right_up = np.triu_indices_from(wij, k=1) # For each correlation coefficient, set the weight equal to the size of # the sample used to calculate that coefficient pairwise_combos = itertools.combinations( np.isfinite(all_intensities), 2) sample_size = lambda x, y: np.count_nonzero(x & y) wij[right_up] = list( itertools.starmap(sample_size, pairwise_combos)) if self._weights == "standard_error": # Set each weights as the reciprocal of the standard error on the # corresponding correlation coefficient # http://www.sjsu.edu/faculty/gerstman/StatPrimer/correlation.pdf with np.errstate(divide="ignore", invalid="ignore"): reciprocal_se = np.sqrt( (wij[right_up] - 2) / (1 - np.square(rij[right_up]))) wij[right_up] = np.where(wij[right_up] > 2, reciprocal_se, 0) # Symmetrise the wij matrix wij += wij.T else: wij = None return rij, wij
# for hkl in detectable_rays['miller_index']: # seen_hkl_multiplicity[hkl] += 1 expt = experiments[0] spacegroup = expt.crystal.get_space_group() unit_cell = expt.crystal.get_unit_cell() possible_hkl = self.list_possible_reflections(spacegroup, unit_cell, dmin, dmax) possible_hkl_flex = flex.miller_index(possible_hkl) hkl_to_id = { hkl: id for (id, hkl) in enumerate(possible_hkl) } id_to_hkl = [ hkl for (id, hkl) in enumerate(possible_hkl) ] # find mapping of reciprocal space onto reciprocal asymmetric unit and its inverse from cctbx.miller import map_to_asu asu_hkl_flex = flex.miller_index(possible_hkl) map_to_asu(spacegroup.type(), False, asu_hkl_flex) # TODO: Treat anomalous signal? map_hkl_to_symmhkl = {r: rs for (r, rs) in zip(possible_hkl, list(asu_hkl_flex))} map_symmhkl_to_hkl = {} for k, v in map_hkl_to_symmhkl.iteritems(): map_symmhkl_to_hkl[v] = map_symmhkl_to_hkl.get(v, []) map_symmhkl_to_hkl[v].append(k) unique_asu_indices = set(asu_hkl_flex) completeness_limit = len(unique_asu_indices) # scoring function: hkl_id -> shared_bin_id scoring_unique_reflection = { hkl: unique_id for (unique_id, hkl) in enumerate(possible_hkl) } scoring_unique_reflection = { hkl_to_id[h]: id for (h, id) in scoring_unique_reflection.iteritems() } scoring_symmetry_related_reflection = { hkl_to_id[hkl]: hkl_to_id[asu] for (hkl, asu) in map_hkl_to_symmhkl.iteritems() }
def exercise_unmerged(space_group_info): import random from cctbx import sgtbx # shuffle the space_group = sgtbx.space_group('P 1', no_expand=True) perm = list(range(len(space_group_info.group()))) random.shuffle(perm) for i in perm: space_group.expand_smx(space_group_info.group()[i]) unit_cell = space_group_info.any_compatible_unit_cell(volume=1000) for cb_op in (None, space_group.info().change_of_basis_op_to_primitive_setting(), unit_cell.change_of_basis_op_to_niggli_cell()): miller_set = crystal.symmetry( unit_cell=unit_cell, space_group=space_group).build_miller_set(d_min=1, anomalous_flag=True) miller_set = miller_set.expand_to_p1().customized_copy( space_group_info=miller_set.space_group_info()) if cb_op is not None: miller_set = miller_set.change_basis(cb_op) m = mtz.object() m.set_title('This is a title') m.set_space_group_info(miller_set.space_group_info()) x = m.add_crystal('XTAL', 'CCTBX', miller_set.unit_cell().parameters()) d = x.add_dataset('TEST', 1) indices = miller_set.indices() original_indices = indices.deep_copy() # map the miller indices to one hemisphere (i.e. just I+) miller.map_to_asu(m.space_group().type(), False, indices) h, k, l = [i.iround() for i in indices.as_vec3_double().parts()] m.adjust_column_array_sizes(len(h)) m.set_n_reflections(len(h)) # assign H, K, L d.add_column('H', 'H').set_values(h.as_double().as_float()) d.add_column('K', 'H').set_values(k.as_double().as_float()) d.add_column('L', 'H').set_values(l.as_double().as_float()) d.add_column('M_ISYM', 'Y').set_values(flex.float(len(indices))) m.replace_original_index_miller_indices(original_indices) assert (m.extract_original_index_miller_indices() == original_indices ).count(False) == 0 # check the indices in the mtz file are actually in the asu extracted_indices = m.extract_miller_indices() miller.map_to_asu(m.space_group().type(), False, extracted_indices) assert ( extracted_indices == m.extract_miller_indices()).count(False) == 0 # test change_basis_in_place if appropriate for current space group import cctbx.sgtbx.cosets cb_op_to_niggli_cell \ = miller_set.change_of_basis_op_to_niggli_cell() minimum_cell_symmetry = crystal.symmetry.change_basis( miller_set.crystal_symmetry(), cb_op=cb_op_to_niggli_cell) lattice_group = sgtbx.lattice_symmetry.group( minimum_cell_symmetry.unit_cell(), max_delta=3.0) lattice_group.expand_inv(sgtbx.tr_vec((0, 0, 0))) lattice_group.make_tidy() cosets = sgtbx.cosets.left_decomposition_point_groups_only( g=lattice_group, h=minimum_cell_symmetry.reflection_intensity_symmetry( anomalous_flag=True).space_group( ).build_derived_acentric_group().make_tidy()) possible_twin_laws = cosets.best_partition_representatives( cb_op=cb_op_to_niggli_cell.inverse(), omit_first_partition=True, omit_negative_determinants=True) for twin_law in possible_twin_laws: cb_op = sgtbx.change_of_basis_op(twin_law.as_xyz()) #print cb_op.as_hkl() # forward direction m.change_basis_in_place(cb_op) assert (m.extract_original_index_miller_indices() == cb_op.apply( original_indices)).count(False) == 0 ## check the indices in the mtz file are actually in the asu #extracted_indices = m.extract_miller_indices() #miller.map_to_asu(m.space_group().type(), False, extracted_indices) #assert (extracted_indices == m.extract_miller_indices()).count(False) == 0 # and back again m.change_basis_in_place(cb_op.inverse()) assert (m.extract_original_index_miller_indices() == original_indices).count(False) == 0
def export_xds_ascii(integrated_data, experiment_list, hklout, summation=False, include_partials=False, keep_partials=False, var_model=(1,0)): '''Export data from integrated_data corresponding to experiment_list to an XDS_ASCII.HKL formatted text file.''' from dials.array_family import flex import math # for the moment assume (and assert) that we will convert data from exactly # one lattice... assert(len(experiment_list) == 1) # select reflections that are assigned to an experiment (i.e. non-negative id) integrated_data = integrated_data.select(integrated_data['id'] >= 0) assert max(integrated_data['id']) == 0 if not summation: assert('intensity.prf.value' in integrated_data) if 'intensity.prf.variance' in integrated_data: selection = integrated_data.get_flags( integrated_data.flags.integrated, all=True) else: selection = integrated_data.get_flags( integrated_data.flags.integrated_sum) integrated_data = integrated_data.select(selection) selection = integrated_data['intensity.sum.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d reflections with negative variance' % \ selection.count(True)) if 'intensity.prf.variance' in integrated_data: selection = integrated_data['intensity.prf.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d profile reflections with negative variance' % \ selection.count(True)) if include_partials: integrated_data = sum_partial_reflections(integrated_data) integrated_data = scale_partial_reflections(integrated_data) if 'partiality' in integrated_data: selection = integrated_data['partiality'] < 0.99 if selection.count(True) > 0 and not keep_partials: integrated_data.del_selected(selection) logger.info('Removing %d incomplete reflections' % \ selection.count(True)) experiment = experiment_list[0] # sort data before output nref = len(integrated_data['miller_index']) indices = flex.size_t_range(nref) import copy unique = copy.deepcopy(integrated_data['miller_index']) from cctbx.miller import map_to_asu map_to_asu(experiment.crystal.get_space_group().type(), False, unique) perm = sorted(indices, key=lambda k: unique[k]) integrated_data = integrated_data.select(flex.size_t(perm)) from scitbx import matrix from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame assert (not experiment.goniometer is None) unit_cell = experiment.crystal.get_unit_cell() from scitbx.array_family import flex from math import floor, sqrt assert(not experiment.scan is None) image_range = experiment.scan.get_image_range() phi_start, phi_range = experiment.scan.get_image_oscillation(image_range[0]) # gather the required information for the reflection file nref = len(integrated_data['miller_index']) zdet = flex.double(integrated_data['xyzcal.px'].parts()[2]) miller_index = integrated_data['miller_index'] I = None sigI = None # export including scale factors if 'lp' in integrated_data: lp = integrated_data['lp'] else: lp = flex.double(nref, 1.0) if 'dqe' in integrated_data: dqe = integrated_data['dqe'] else: dqe = flex.double(nref, 1.0) scl = lp / dqe # profile correlation if 'profile.correlation' in integrated_data: prof_corr = 100.0 * integrated_data['profile.correlation'] else: prof_corr = flex.double(nref, 100.0) # partiality if 'partiality' in integrated_data: partiality = 100 * integrated_data['partiality'] else: prof_corr = flex.double(nref, 100.0) if summation: I = integrated_data['intensity.sum.value'] * scl V = integrated_data['intensity.sum.variance'] * scl * scl assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) else: I = integrated_data['intensity.prf.value'] * scl V = integrated_data['intensity.prf.variance'] * scl * scl assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) fout = open(hklout, 'w') # first write the header - in the "standard" coordinate frame... panel = experiment.detector[0] fast = panel.get_fast_axis() slow = panel.get_slow_axis() Rd = align_reference_frame(fast, (1,0,0), slow, (0,1,0)) print 'Coordinate change:' print '%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n' % Rd.elems fast = Rd * fast slow = Rd * slow qx, qy = panel.get_pixel_size() nx, ny = panel.get_image_size() distance = matrix.col(Rd * panel.get_origin()).dot( matrix.col(Rd * panel.get_normal())) org = Rd * (matrix.col(panel.get_origin()) - distance * matrix.col( panel.get_normal())) orgx = - org.dot(fast) / qx orgy = - org.dot(slow) / qy UB = Rd * matrix.sqr(experiment.crystal.get_A()) real_space_ABC = UB.inverse().elems axis = Rd * experiment.goniometer.get_rotation_axis() beam = Rd * experiment.beam.get_s0() cell_fmt = '%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f' axis_fmt = '%9.3f %9.3f %9.3f' fout.write('\n'.join([ '!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL\'S_LAW=TRUE', '!Generated by dials.export', '!DATA_RANGE= %d %d' % image_range, '!ROTATION_AXIS= %9.6f %9.6f %9.6f' % axis.elems, '!OSCILLATION_RANGE= %f' % phi_range, '!STARTING_ANGLE= %f' % phi_start, '!STARTING_FRAME= %d' % image_range[0], '!SPACE_GROUP_NUMBER= %d' % experiment.crystal.get_space_group().type().number(), '!UNIT_CELL_CONSTANTS= %s' % (cell_fmt % unit_cell.parameters()), '!UNIT_CELL_A-AXIS= %s' % (axis_fmt % real_space_ABC[0:3]), '!UNIT_CELL_B-AXIS= %s' % (axis_fmt % real_space_ABC[3:6]), '!UNIT_CELL_C-AXIS= %s' % (axis_fmt % real_space_ABC[6:9]), '!X-RAY_WAVELENGTH= %f' % experiment.beam.get_wavelength(), '!INCIDENT_BEAM_DIRECTION= %f %f %f' % beam.elems, '!NX= %d NY= %d QX= %f QY= %f' % (nx, ny, qx, qy), '!ORGX= %9.2f ORGY= %9.2f' % (orgx, orgy), '!DETECTOR_DISTANCE= %8.3f' % distance, '!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f' % fast.elems, '!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f' % slow.elems, '!VARIANCE_MODEL= %7.3e %7.3e' % var_model, '!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12', '!ITEM_H=1', '!ITEM_K=2', '!ITEM_L=3', '!ITEM_IOBS=4', '!ITEM_SIGMA(IOBS)=5', '!ITEM_XD=6', '!ITEM_YD=7', '!ITEM_ZD=8', '!ITEM_RLP=9', '!ITEM_PEAK=10', '!ITEM_CORR=11', '!ITEM_PSI=12', '!END_OF_HEADER', ''])) # then write the data records s0 = Rd * matrix.col(experiment.beam.get_s0()) for j in range(nref): x, y, z = integrated_data['xyzcal.px'][j] phi = phi_start + z * phi_range h, k, l = miller_index[j] X = (UB * (h, k, l)).rotate(axis, phi, deg=True) s = s0 + X g = s.cross(s0).normalize() f = (s - s0).normalize() # find component of beam perpendicular to f, e e = - (s + s0).normalize() if h == k and k == l: u = (h, -h, 0) else: u = (k - l, l - h, h - k) q = (matrix.col(u).transpose() * UB.inverse()).normalize( ).transpose().rotate(axis, phi, deg=True) psi = q.angle(g, deg=True) if q.dot(e) < 0: psi *= -1 fout.write('%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n' % (h, k, l, I[j], sigI[j], x, y, z, scl[j], partiality[j], prof_corr[j], psi)) fout.write('!END_OF_DATA\n') fout.close() logger.info('Output %d reflections to %s' % (nref, hklout)) return
def exercise_unmerged(space_group_info): import random from cctbx import sgtbx # shuffle the space_group = sgtbx.space_group("P 1", no_expand=True) perm = list(range(len(space_group_info.group()))) random.shuffle(perm) for i in perm: space_group.expand_smx(space_group_info.group()[i]) unit_cell = space_group_info.any_compatible_unit_cell(volume=1000) for cb_op in ( None, space_group.info().change_of_basis_op_to_primitive_setting(), unit_cell.change_of_basis_op_to_niggli_cell(), ): miller_set = crystal.symmetry(unit_cell=unit_cell, space_group=space_group).build_miller_set( d_min=1, anomalous_flag=True ) miller_set = miller_set.expand_to_p1().customized_copy(space_group_info=miller_set.space_group_info()) if cb_op is not None: miller_set = miller_set.change_basis(cb_op) m = mtz.object() m.set_title("This is a title") m.set_space_group_info(miller_set.space_group_info()) x = m.add_crystal("XTAL", "CCTBX", miller_set.unit_cell().parameters()) d = x.add_dataset("TEST", 1) indices = miller_set.indices() original_indices = indices.deep_copy() # map the miller indices to one hemisphere (i.e. just I+) miller.map_to_asu(m.space_group().type(), False, indices) h, k, l = [i.iround() for i in indices.as_vec3_double().parts()] m.adjust_column_array_sizes(len(h)) m.set_n_reflections(len(h)) # assign H, K, L d.add_column("H", "H").set_values(h.as_double().as_float()) d.add_column("K", "H").set_values(k.as_double().as_float()) d.add_column("L", "H").set_values(l.as_double().as_float()) d.add_column("M_ISYM", "Y").set_values(flex.float(len(indices))) m.replace_original_index_miller_indices(original_indices) assert (m.extract_original_index_miller_indices() == original_indices).count(False) == 0 # check the indices in the mtz file are actually in the asu extracted_indices = m.extract_miller_indices() miller.map_to_asu(m.space_group().type(), False, extracted_indices) assert (extracted_indices == m.extract_miller_indices()).count(False) == 0 # test change_basis_in_place if appropriate for current space group import cctbx.sgtbx.cosets cb_op_to_niggli_cell = miller_set.change_of_basis_op_to_niggli_cell() minimum_cell_symmetry = crystal.symmetry.change_basis(miller_set.crystal_symmetry(), cb_op=cb_op_to_niggli_cell) lattice_group = sgtbx.lattice_symmetry.group(minimum_cell_symmetry.unit_cell(), max_delta=3.0) lattice_group.expand_inv(sgtbx.tr_vec((0, 0, 0))) lattice_group.make_tidy() cosets = sgtbx.cosets.left_decomposition_point_groups_only( g=lattice_group, h=minimum_cell_symmetry.reflection_intensity_symmetry(anomalous_flag=True) .space_group() .build_derived_acentric_group() .make_tidy(), ) possible_twin_laws = cosets.best_partition_representatives( cb_op=cb_op_to_niggli_cell.inverse(), omit_first_partition=True, omit_negative_determinants=True ) for twin_law in possible_twin_laws: cb_op = sgtbx.change_of_basis_op(twin_law.as_xyz()) # print cb_op.as_hkl() # forward direction m.change_basis_in_place(cb_op) assert (m.extract_original_index_miller_indices() == cb_op.apply(original_indices)).count(False) == 0 ## check the indices in the mtz file are actually in the asu # extracted_indices = m.extract_miller_indices() # miller.map_to_asu(m.space_group().type(), False, extracted_indices) # assert (extracted_indices == m.extract_miller_indices()).count(False) == 0 # and back again m.change_basis_in_place(cb_op.inverse()) assert (m.extract_original_index_miller_indices() == original_indices).count(False) == 0
def _compute_rij_wij(self, use_cache=True): """Compute the rij_wij matrix. Rij is a symmetric matrix of size (n x m, n x m), where n is the number of datasets and m is the number of symmetry operations. It is composed of (m, m) blocks of size (n, n), where each block contains the correlation coefficients between cb_op_k applied to datasets 1..N with cb_op_kk applied to datasets 1.. N. If `use_cache=True`, then an optimisation is made to reflect the fact some elements of the matrix are equivalent, i.e.: CC[(a, cb_op_k), (b, cb_op_kk)] == CC[(a,), (b, cb_op_k.inverse() * cb_op_kk)] """ n_lattices = len(self._lattices) # Pre-calculate miller indices after application of each cb_op. Only calculate # this once per cb_op instead of on-the-fly every time we need it. indices = {} space_group_type = self._data.space_group().type() for cb_op in self.sym_ops: cb_op = sgtbx.change_of_basis_op(cb_op) indices_reindexed = cb_op.apply(self._data.indices()) miller.map_to_asu(space_group_type, False, indices_reindexed) indices[cb_op.as_xyz()] = indices_reindexed def _compute_rij_matrix_one_row_block(i): rij_cache = {} n_sym_ops = len(self.sym_ops) NN = n_lattices * n_sym_ops rij_row = [] rij_col = [] rij_data = [] if self._weights is not None: wij_row = [] wij_col = [] wij_data = [] else: wij = None i_lower, i_upper = self._lattice_lower_upper_index(i) intensities_i = self._data.data()[i_lower:i_upper] for j in range(n_lattices): j_lower, j_upper = self._lattice_lower_upper_index(j) intensities_j = self._data.data()[j_lower:j_upper] for k, cb_op_k in enumerate(self.sym_ops): cb_op_k = sgtbx.change_of_basis_op(cb_op_k) indices_i = indices[cb_op_k.as_xyz()][i_lower:i_upper] for kk, cb_op_kk in enumerate(self.sym_ops): if i == j and k == kk: # don't include correlation of dataset with itself continue cb_op_kk = sgtbx.change_of_basis_op(cb_op_kk) ik = i + (n_lattices * k) jk = j + (n_lattices * kk) key = (i, j, str(cb_op_k.inverse() * cb_op_kk)) if use_cache and key in rij_cache: cc, n = rij_cache[key] else: indices_j = indices[cb_op_kk.as_xyz()][j_lower:j_upper] matches = miller.match_indices(indices_i, indices_j) pairs = matches.pairs() isel_i = pairs.column(0) isel_j = pairs.column(1) isel_i = isel_i.select( self._patterson_group.epsilon(indices_i.select(isel_i)) == 1 ) isel_j = isel_j.select( self._patterson_group.epsilon(indices_j.select(isel_j)) == 1 ) corr = flex.linear_correlation( intensities_i.select(isel_i), intensities_j.select(isel_j), ) if corr.is_well_defined(): cc = corr.coefficient() n = corr.n() rij_cache[key] = (cc, n) else: cc = None n = None if ( n is None or cc is None or (self._min_pairs is not None and n < self._min_pairs) ): continue if self._weights == "count": wij_row.extend([ik, jk]) wij_col.extend([jk, ik]) wij_data.extend([n, n]) elif self._weights == "standard_error": assert n > 2 # http://www.sjsu.edu/faculty/gerstman/StatPrimer/correlation.pdf se = math.sqrt((1 - cc ** 2) / (n - 2)) wij = 1 / se wij_row.extend([ik, jk]) wij_col.extend([jk, ik]) wij_data.extend([wij, wij]) rij_row.append(ik) rij_col.append(jk) rij_data.append(cc) rij = sparse.coo_matrix((rij_data, (rij_row, rij_col)), shape=(NN, NN)) if self._weights is not None: wij = sparse.coo_matrix((wij_data, (wij_row, wij_col)), shape=(NN, NN)) return rij, wij args = [(i,) for i in range(n_lattices)] results = easy_mp.parallel_map( _compute_rij_matrix_one_row_block, args, processes=self._nproc, iterable_type=easy_mp.posiargs, method="multiprocessing", ) rij_matrix = None wij_matrix = None for i, (rij, wij) in enumerate(results): if rij_matrix is None: rij_matrix = rij else: rij_matrix += rij if wij is not None: if wij_matrix is None: wij_matrix = wij else: wij_matrix += wij rij_matrix = rij_matrix.todense().astype(np.float64) if wij_matrix is not None: wij_matrix = wij_matrix.todense().astype(np.float64) return rij_matrix, wij_matrix
def StrategyEvaluator(self, experiments, evaluation_function_factory, dmin, dmax): # TODO: Don't take experiments, rather take one set of experiment components and a strategy equivalent list def calculate_miller_rings(detector): import time t = time.clock() mrings = {} for scan_axis in ["phi", "omega"]: mrings[scan_axis] = {} for other_axis in range(0, 360, 10): gonio = goniometer_factory.make_kappa_goniometer( alpha=54.7356, # fixed magic angle kappa=0, # fixed 0 kappa phi=0 if scan_axis == "phi" else other_axis, omega=0 if scan_axis == "omega" else other_axis, direction="-y", scan_axis=scan_axis, ) ring = tools.determine_miller_ring_sectors( detector, gonio, s0, possible_hkl_flex, crystal_A) mrings[scan_axis][other_axis] = ring print(time.clock() - t) print(list(mrings.iterkeys())) print([list(x.iterkeys()) for x in mrings.itervalues()]) t = time.clock() sc = scorings all_scorings = { "%s:%s:%s" % (scan, str(x1), str(x2)): tools.geometric_scoring( [hkl_to_id[hkl] for hkl in mrings[scan][x1][x2]], sc)["total"] for scan in mrings.iterkeys() for x1 in mrings[scan].iterkeys() for x2 in range(0, 36) } # print list(all_scorings.itervalues()) print( len(list(all_scorings.itervalues())), min(all_scorings.itervalues()), max(all_scorings.itervalues()), sum(all_scorings.itervalues()) / len(list(all_scorings.itervalues())), ) print(time.clock() - t) sys.exit(0) def calculate_observations(detector, goniometer, oscillation): calculate_miller_rings(detector) import time t = time.clock() r = tools.determine_miller_ring_sectors(detector, goniometer, s0, possible_hkl_flex, crystal_A) sc = scorings print( map( lambda x: tools.geometric_scoring( [hkl_to_id[hkl] for hkl in x], sc)["total"], r, )) sc = tools.geometric_scoring([hkl_to_id[hkl] for hkl in r[4]], scorings)["scorings"] sc = tools.geometric_scoring([hkl_to_id[hkl] for hkl in r[5]], sc)["scorings"] sc = tools.geometric_scoring([hkl_to_id[hkl] for hkl in r[6]], sc)["scorings"] print( map( lambda x: tools.geometric_scoring( [hkl_to_id[hkl] for hkl in x], sc)["total"], r, )) print(time.clock() - t) print(map(len, r)) sys.exit(0) # TODO: To test 2theta rotation code use detector.py detector factory # TODO: and create two different two_theta detectors and try to rotate one onto the other # ----: Reflection only counted once within oscillation, even when multiple times in diffraction condition # ----: Is it?! Unsure. # SOLVED: half-true. Not problematic here. def export_mtz(observed_hkls, experiment, filename): if experiment.goniometer: axis = experiment.goniometer.get_rotation_axis() else: axis = 0.0, 0.0, 0.0 s0 = experiment.beam.get_s0() wavelength = experiment.beam.get_wavelength() from scitbx import matrix panel = experiment.detector[0] pixel_size = panel.get_pixel_size() cb_op_to_ref = (experiment.crystal.get_space_group().info(). change_of_basis_op_to_reference_setting()) experiment.crystal = experiment.crystal.change_basis(cb_op_to_ref) from iotbx import mtz from scitbx.array_family import flex import itertools m = mtz.object() m.set_title("from dials.scratch.mg.strategy_i19") m.set_space_group_info(experiment.crystal.get_space_group().info()) nrefcount = sum(observed_hkls.itervalues()) nref = max(observed_hkls.itervalues()) for batch in range(1, nref + 1): o = m.add_batch().set_num(batch).set_nbsetid(1).set_ncryst(1) o.set_time1(0.0).set_time2(0.0).set_title("Batch %d" % batch) o.set_ndet(1).set_theta(flex.float((0.0, 0.0))).set_lbmflg(0) o.set_alambd(wavelength).set_delamb(0.0).set_delcor(0.0) o.set_divhd(0.0).set_divvd(0.0) o.set_so(flex.float(s0)).set_source(flex.float((0, 0, -1))) o.set_bbfac(0.0).set_bscale(1.0) o.set_sdbfac(0.0).set_sdbscale(0.0).set_nbscal(0) _unit_cell = experiment.crystal.get_unit_cell() _U = experiment.crystal.get_U() o.set_cell(flex.float(_unit_cell.parameters())) o.set_lbcell(flex.int((-1, -1, -1, -1, -1, -1))) o.set_umat(flex.float(_U.transpose().elems)) mosaic = experiment.crystal.get_mosaicity() o.set_crydat( flex.float([ mosaic, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ])) o.set_lcrflg(0) o.set_datum(flex.float((0.0, 0.0, 0.0))) # detector size, distance o.set_detlm( flex.float([ 0.0, panel.get_image_size()[0], 0.0, panel.get_image_size()[1], 0, 0, 0, 0, ])) o.set_dx(flex.float([panel.get_directed_distance(), 0.0])) # goniometer axes and names, and scan axis number, and number of axes, missets o.set_e1(flex.float(axis)) o.set_e2(flex.float((0.0, 0.0, 0.0))) o.set_e3(flex.float((0.0, 0.0, 0.0))) o.set_gonlab(flex.std_string(("AXIS", "", ""))) o.set_jsaxs(1) o.set_ngonax(1) o.set_phixyz(flex.float((0.0, 0.0, 0.0, 0.0, 0.0, 0.0))) phi_start, phi_range = 0.0, 0.0 o.set_phistt(phi_start) o.set_phirange(phi_range) o.set_phiend(phi_start + phi_range) o.set_scanax(flex.float(axis)) # number of misorientation angles o.set_misflg(0) # crystal axis closest to rotation axis (why do I want this?) o.set_jumpax(0) # type of data - 1; 2D, 2; 3D, 3; Laue o.set_ldtype(2) # now create the actual data structures - first keep a track of the columns # H K L M/ISYM BATCH I SIGI IPR SIGIPR FRACTIONCALC XDET YDET ROT WIDTH # LP MPART FLAG BGPKRATIOS from cctbx.array_family import flex as cflex # implicit import # now go for it and make an MTZ file... x = m.add_crystal("XTAL", "DIALS", unit_cell.parameters()) d = x.add_dataset("FROMDIALS", wavelength) # now add column information... type_table = { "IPR": "J", "BGPKRATIOS": "R", "WIDTH": "R", "I": "J", "H": "H", "K": "H", "MPART": "I", "L": "H", "BATCH": "B", "M_ISYM": "Y", "SIGI": "Q", "FLAG": "I", "XDET": "R", "LP": "R", "YDET": "R", "SIGIPR": "Q", "FRACTIONCALC": "R", "ROT": "R", } m.adjust_column_array_sizes(nrefcount) m.set_n_reflections(nrefcount) # assign H, K, L, M_ISYM space for column in "H", "K", "L", "M_ISYM": d.add_column(column, type_table[column]).set_values( flex.float(nrefcount, 0.0)) batchnums = (_ for (x, n) in observed_hkls.iteritems() for _ in range(1, n + 1)) d.add_column("BATCH", type_table["BATCH"]).set_values(flex.float(batchnums)) d.add_column("FRACTIONCALC", type_table["FRACTIONCALC"]).set_values( flex.float(nrefcount, 3.0)) m.replace_original_index_miller_indices( cb_op_to_ref.apply( cflex.miller_index([ _ for (x, n) in observed_hkls.iteritems() for _ in itertools.repeat(x, n) ]))) m.write(filename) return m def evaluate_concrete_strategy(hkls, evaluation_function): print("%5d reflections fall on detector during sweep" % len(hkls)) # print "%5d unique reflections fall on detector during sweep (without symmetry relations)" % len(set(list(hkls))) # for r in detectable_rays.rows(): # hkl = r['miller_index'] # if (abs(hkl[0]) == 11) and (abs(hkl[1]) == 5) and (abs(hkl[2]) < 5): # print "%12s -> %10s observed at angle %.2f on image %.1f" % (hkl, map_hkl_to_symmhkl[hkl], r['phi'], # expt.scan.get_image_index_from_angle(r['phi'] - (2 * 3.1415926535), deg=False)) symmetry_mapped_hkls = set( [map_hkl_to_symmhkl[hkl] for hkl in hkls]) print( "%5d unique reflections fall on detector during sweep (including symmetry relations)" % len(symmetry_mapped_hkls)) completeness = 100 * len(symmetry_mapped_hkls) / completeness_limit multiplicity = len(hkls) / len(symmetry_mapped_hkls) score = evaluation_function(hkls) print( "Estimated sweep completeness: %5.1f %% sweep multiplicity : %.1f sweep score: %.1f" % (completeness, multiplicity, score)) return { "completeness": completeness, "multiplicity": multiplicity, "score": score, } # # count the number of observations per reflection to construct an evaluation function # seen_hkl_multiplicity = {x: 0 for x in possible_hkl} # for hkl in detectable_rays['miller_index']: # seen_hkl_multiplicity[hkl] += 1 expt = experiments[0] spacegroup = expt.crystal.get_space_group() unit_cell = expt.crystal.get_unit_cell() possible_hkl = self.list_possible_reflections(spacegroup, unit_cell, dmin, dmax) possible_hkl_flex = flex.miller_index(possible_hkl) hkl_to_id = {hkl: id for (id, hkl) in enumerate(possible_hkl)} id_to_hkl = [hkl for (id, hkl) in enumerate(possible_hkl)] # find mapping of reciprocal space onto reciprocal asymmetric unit and its inverse from cctbx.miller import map_to_asu asu_hkl_flex = flex.miller_index(possible_hkl) map_to_asu(spacegroup.type(), False, asu_hkl_flex) # TODO: Treat anomalous signal? map_hkl_to_symmhkl = { r: rs for (r, rs) in zip(possible_hkl, list(asu_hkl_flex)) } map_symmhkl_to_hkl = {} for k, v in map_hkl_to_symmhkl.iteritems(): map_symmhkl_to_hkl[v] = map_symmhkl_to_hkl.get(v, []) map_symmhkl_to_hkl[v].append(k) unique_asu_indices = set(asu_hkl_flex) completeness_limit = len(unique_asu_indices) # scoring function: hkl_id -> shared_bin_id scoring_unique_reflection = { hkl: unique_id for (unique_id, hkl) in enumerate(possible_hkl) } scoring_unique_reflection = { hkl_to_id[h]: id for (h, id) in scoring_unique_reflection.iteritems() } scoring_symmetry_related_reflection = { hkl_to_id[hkl]: hkl_to_id[asu] for (hkl, asu) in map_hkl_to_symmhkl.iteritems() } scorings = { "unique": (scoring_unique_reflection, [0] * len(possible_hkl)), "symmetry": (scoring_symmetry_related_reflection, [0] * len(possible_hkl)), } print( "%5d unique reflections possible in %s (ignoring anomalous signal)" % (completeness_limit, spacegroup.type().lookup_symbol())) print() # Determine the detectable reflections given some data collection sweep s0 = expt.beam.get_s0() rotation_axis = expt.goniometer.get_rotation_axis() # Obtain the A matrix with all diffractometer angles set to 0. crystal_R = matrix.sqr(expt.goniometer.get_fixed_rotation()) crystal_A = crystal_R.inverse() * expt.crystal.get_A() def run_strategies(strategies): # starting from scratch observed_hkls = {} total_score = 0.0 degrees = 0 from itertools import izip, count for (run, strategy) in izip(count(1), strategies): print() print("Sweep %d:" % run) evaluation_function = evaluation_function_factory( possible_hkl, observed_hkls, map_hkl_to_symmhkl, map_symmhkl_to_hkl) goniometer = goniometer_factory.make_kappa_goniometer( alpha=54.7, kappa=strategy["kappa"], phi=strategy["phi"], omega=strategy["omega"], direction="-y", scan_axis=strategy["scan_axis"], ) degrees += strategy["scan"] from math import radians if strategy["scan_axis"] == "omega": oscillation_range = ( radians(strategy["omega"]), radians(strategy["omega"] + strategy["scan"]), ) else: oscillation_range = ( radians(strategy["phi"]), radians(strategy["phi"] + strategy["scan"]), ) # repeat sweep: expt.goniometer proposed_hkls = calculate_observations(expt.detector, goniometer, oscillation_range) strategy_results = evaluate_concrete_strategy( proposed_hkls, evaluation_function) combined_observations = self.add_seen_multiplicity( observed_hkls, proposed_hkls) num_observed_hkls = len( set([ map_hkl_to_symmhkl[hkl] for hkl in combined_observations.keys() ])) count_hkl_observations = sum(combined_observations.values()) completeness = 100 * num_observed_hkls / completeness_limit multiplicity = count_hkl_observations / num_observed_hkls total_score += strategy_results["score"] print( "Estimated total completeness: %5.1f %% total multiplicity : %.1f total score: %.1f" % (completeness, multiplicity, total_score)) # keep the repeat sweep and continue observed_hkls = combined_observations export_mtz(observed_hkls, expt, "mtzout.mtz") return { "completeness": completeness, "multiplicity": multiplicity, "score": total_score, "degrees": degrees, } return run_strategies
def _export_experiment(filename, integrated_data, experiment, params, var_model=(1, 0)): # type: (str, flex.reflection_table, dxtbx.model.Experiment, libtbx.phil.scope_extract, Tuple) """Export a single experiment to an XDS_ASCII.HKL format file. Args: filename: The file to write to integrated_data: The reflection table, pre-selected to one experiment experiment: The experiment list entry to export params: The PHIL configuration object var_model: """ # export for xds_ascii should only be for non-scaled reflections assert any(i in integrated_data for i in ["intensity.sum.value", "intensity.prf.value"]) # Handle requesting profile intensities (default via auto) but no column if "profile" in params.intensity and "intensity.prf.value" not in integrated_data: raise Sorry( "Requested profile intensity data but only summed present. Use intensity=sum." ) integrated_data = filter_reflection_table( integrated_data, intensity_choice=params.intensity, partiality_threshold=params.mtz.partiality_threshold, combine_partials=params.mtz.combine_partials, min_isigi=params.mtz.min_isigi, filter_ice_rings=params.mtz.filter_ice_rings, d_min=params.mtz.d_min, ) # calculate the scl = lp/dqe correction for outputting but don't apply it as # it has already been applied in filter_reflection_table ( integrated_data, scl, ) = FilteringReductionMethods.calculate_lp_qe_correction_and_filter( integrated_data) # sort data before output nref = len(integrated_data["miller_index"]) indices = flex.size_t_range(nref) unique = copy.deepcopy(integrated_data["miller_index"]) map_to_asu(experiment.crystal.get_space_group().type(), False, unique) perm = sorted(indices, key=lambda k: unique[k]) integrated_data = integrated_data.select(flex.size_t(perm)) if experiment.goniometer is None: print( "Warning: No goniometer. Experimentally exporting with (1 0 0) axis" ) unit_cell = experiment.crystal.get_unit_cell() if experiment.scan is None: print( "Warning: No Scan. Experimentally exporting no-oscillation values") image_range = (1, 1) phi_start, phi_range = 0.0, 0.0 else: image_range = experiment.scan.get_image_range() phi_start, phi_range = experiment.scan.get_image_oscillation( image_range[0]) # gather the required information for the reflection file nref = len(integrated_data["miller_index"]) miller_index = integrated_data["miller_index"] # profile correlation if "profile.correlation" in integrated_data: prof_corr = 100.0 * integrated_data["profile.correlation"] else: prof_corr = flex.double(nref, 100.0) # partiality if "partiality" in integrated_data: partiality = 100 * integrated_data["partiality"] else: prof_corr = flex.double(nref, 100.0) if "intensity.sum.value" in integrated_data: I = integrated_data["intensity.sum.value"] V = integrated_data["intensity.sum.variance"] assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) else: I = integrated_data["intensity.prf.value"] V = integrated_data["intensity.prf.variance"] assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) fout = open(filename, "w") # first write the header - in the "standard" coordinate frame... panel = experiment.detector[0] fast = panel.get_fast_axis() slow = panel.get_slow_axis() Rd = align_reference_frame(fast, (1, 0, 0), slow, (0, 1, 0)) print("Coordinate change:") print("%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n" % Rd.elems) fast = Rd * fast slow = Rd * slow qx, qy = panel.get_pixel_size() nx, ny = panel.get_image_size() distance = matrix.col(Rd * panel.get_origin()).dot( matrix.col(Rd * panel.get_normal())) org = Rd * (matrix.col(panel.get_origin()) - distance * matrix.col(panel.get_normal())) orgx = -org.dot(fast) / qx orgy = -org.dot(slow) / qy UB = Rd * matrix.sqr(experiment.crystal.get_A()) real_space_ABC = UB.inverse().elems if experiment.goniometer is not None: axis = Rd * experiment.goniometer.get_rotation_axis() else: axis = Rd * (1, 0, 0) beam = Rd * experiment.beam.get_s0() cell_fmt = "%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f" axis_fmt = "%9.3f %9.3f %9.3f" fout.write("\n".join([ "!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL'S_LAW=TRUE", "!Generated by dials.export", "!DATA_RANGE= %d %d" % image_range, "!ROTATION_AXIS= %9.6f %9.6f %9.6f" % axis.elems, "!OSCILLATION_RANGE= %f" % phi_range, "!STARTING_ANGLE= %f" % phi_start, "!STARTING_FRAME= %d" % image_range[0], "!SPACE_GROUP_NUMBER= %d" % experiment.crystal.get_space_group().type().number(), "!UNIT_CELL_CONSTANTS= %s" % (cell_fmt % unit_cell.parameters()), "!UNIT_CELL_A-AXIS= %s" % (axis_fmt % real_space_ABC[0:3]), "!UNIT_CELL_B-AXIS= %s" % (axis_fmt % real_space_ABC[3:6]), "!UNIT_CELL_C-AXIS= %s" % (axis_fmt % real_space_ABC[6:9]), "!X-RAY_WAVELENGTH= %f" % experiment.beam.get_wavelength(), "!INCIDENT_BEAM_DIRECTION= %f %f %f" % beam.elems, "!NX= %d NY= %d QX= %f QY= %f" % (nx, ny, qx, qy), "!ORGX= %9.2f ORGY= %9.2f" % (orgx, orgy), "!DETECTOR_DISTANCE= %8.3f" % distance, "!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f" % fast.elems, "!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f" % slow.elems, "!VARIANCE_MODEL= %7.3e %7.3e" % var_model, "!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12", "!ITEM_H=1", "!ITEM_K=2", "!ITEM_L=3", "!ITEM_IOBS=4", "!ITEM_SIGMA(IOBS)=5", "!ITEM_XD=6", "!ITEM_YD=7", "!ITEM_ZD=8", "!ITEM_RLP=9", "!ITEM_PEAK=10", "!ITEM_CORR=11", "!ITEM_PSI=12", "!END_OF_HEADER", "", ])) # then write the data records s0 = Rd * matrix.col(experiment.beam.get_s0()) for j in range(nref): x, y, z = integrated_data["xyzcal.px"][j] phi = phi_start + z * phi_range h, k, l = miller_index[j] X = (UB * (h, k, l)).rotate(axis, phi, deg=True) s = s0 + X g = s.cross(s0).normalize() # find component of beam perpendicular to f, e e = -(s + s0).normalize() if h == k and k == l: u = (h, -h, 0) else: u = (k - l, l - h, h - k) q = ((matrix.col(u).transpose() * UB.inverse()).normalize().transpose().rotate(axis, phi, deg=True)) psi = q.angle(g, deg=True) if q.dot(e) < 0: psi *= -1 fout.write("%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n" % ( h, k, l, I[j], sigI[j], x, y, z, scl[j], partiality[j], prof_corr[j], psi, )) fout.write("!END_OF_DATA\n") fout.close() logger.info("Output %d reflections to %s" % (nref, filename))
def run(args): phil = iotbx.phil.process_command_line( args = args, master_string = master_phil) work_params = phil.work.extract() if ("--help" in args) : libtbx.phil.parse(master_phil.show()) return if ((work_params.d_min is None) or (work_params.data is None) or ((work_params.model is None) and work_params.scaling.algorithm != "mark1")): raise Usage("cxi.merge " "d_min=4.0 " "data=~/scratch/r0220/006/strong/ " "model=3bz1_3bz2_core.pdb") if ((work_params.rescale_with_average_cell) and (not work_params.set_average_unit_cell)) : raise Usage("If rescale_with_average_cell=True, you must also specify "+ "set_average_unit_cell=True.") miller_set = symmetry( unit_cell = work_params.target_unit_cell, space_group_info = work_params.target_space_group ).build_miller_set( anomalous_flag = not work_params.merge_anomalous, d_min = work_params.d_min) from xfel.cxi.merging.general_fcalc import random_structure i_model = random_structure(work_params) # ---- Augment this code with any special procedures for x scaling scaler = xscaling_manager( miller_set = miller_set, i_model = i_model, params = work_params) scaler.read_all() sg = miller_set.space_group() pg = sg.build_derived_laue_group() rational_ops = [] for symop in pg: rational_ops.append((matrix.sqr(symop.r().transpose().as_rational()), symop.r().as_hkl())) # miller_set.show_summary() uc = work_params.target_unit_cell hkl_asu = scaler.observations["hkl_id"] imageno = scaler.observations["frame_id"] intensi = scaler.observations["i"] sigma_i = scaler.observations["sigi"] lookup = scaler.millers["merged_asu_hkl"] origH = scaler.observations["H"] origK = scaler.observations["K"] origL = scaler.observations["L"] from cctbx.miller import map_to_asu sgtype = miller_set.space_group_info().type() aflag = miller_set.anomalous_flag() from cctbx.array_family import flex # FIXME in here perform the mapping to ASU for both the original and other # index as an array-wise manipulation to make things a bunch faster... # however this also uses a big chunk of RAM... FIXME also in here use # cb_op.apply(indices) to get the indices reindexed... original_indices = flex.miller_index() for x in xrange(len(scaler.observations["hkl_id"])): original_indices.append(lookup[hkl_asu[x]]) from cctbx.sgtbx import change_of_basis_op I23 = change_of_basis_op('k, -h, l') other_indices = I23.apply(original_indices) map_to_asu(sgtype, aflag, original_indices) map_to_asu(sgtype, aflag, other_indices) # FIXME would be useful in here to have a less expensive way of finding the # symmetry operation which gave the map to the ASU - perhaps best way is to # make a new C++ map_to_asu which records this. # FIXME in here recover the original frame structure of the data to # logical frame objetcs - N.B. the frame will need to be augmented to test # alternative indexings # construct table of start / end indices for frames: now using Python # range indexing starts = [0] ends = [] for x in xrange(1, len(scaler.observations["hkl_id"])): if imageno[x] != imageno[x - 1]: ends.append(x) starts.append(x) ends.append(len(scaler.observations["hkl_id"])) keep_start = [] keep_end = [] for j, se in enumerate(zip(starts, ends)): print 'processing frame %d: %d to %d' % (j, se[0], se[1]) s, e = se isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s) dmin = 100.0 for x in xrange(s, e): d = uc.d(lookup[hkl_asu[x]]) if d < dmin: dmin = d if isig > 6.0 and dmin < 3.2: keep_start.append(s) keep_end.append(e) starts = keep_start ends = keep_end print 'Keeping %d frames' % len(starts) # then start running the comparison code frames = [] for s, e in zip(starts, ends): # FIXME need this from remap to ASU misym = [0 for x in range(s, e)] indices = [original_indices[x] for x in range(s, e)] other = [other_indices[x] for x in range(s, e)] intensities = intensi[s:e] sigmas = sigma_i[s:e] frames.append(Frame(uc, indices, other, intensities, sigmas)) reference = FrameFromReferenceMTZ() fout = open('cc_reference.log', 'w') for j, f in enumerate(frames): _cc = reference.cc(f) _oo = reference.cc_other(f) print '%d %d %d %d %f %d %f' % (j, starts[j], ends[j], _cc[0], _cc[1], _oo[0], _oo[1]) fout.write('%d %d %d %d %f %d %f\n' % (j, starts[j], ends[j], _cc[0], _cc[1], _oo[0], _oo[1])) fout.close() return
mtz_dataset.add_miller_array(miller_array=i_obs.select(~remove_sel), column_root_label="ICUT") mtz_dataset.add_miller_array(miller_array=i_obs.select(remove_sel), column_root_label="IREMOVED") if f_obs is not None: mtz_dataset.add_miller_array(miller_array=f_obs.select(~remove_sel), column_root_label="FCUT") mtz_dataset.add_miller_array(miller_array=f_obs.select(remove_sel), column_root_label="FREMOVED") mtz_dataset.mtz_object().write(file_name=params.hklout) if params.xds_ascii is not None: # XXX Need to check unit cell compatiblity from yamtbx.dataproc.xds.xds_ascii import XDS_ASCII from cctbx import miller xa = XDS_ASCII(params.xds_ascii, sys.stdout) miller.map_to_asu(xa.symm.space_group_info().type(), False, xa.indices) removed_indices = i_obs.indices().select(remove_sel) out = open("removed_positions.dat", "w") for hkl, x, y, z, i, sigi in zip(xa.indices, xa.xd, xa.yd, xa.zd, xa.iobs, xa.sigma_iobs): if sigi <= 0: print "sigi<=0", x, y, z, i, sigi continue if hkl in removed_indices: print >>out, x, y, z, i, sigi if params.hklref is not None: #from eval_Rfree_factors_with_common_reflections import get_flag from cctbx.array_family import flex calc_r = lambda f_obs, f_model: flex.sum(flex.abs(f_obs.data() - f_model.data())) / flex.sum(f_obs.data()) hklref_arrays = iotbx.mtz.object(params.hklref).as_miller_arrays()
def run(args): phil = iotbx.phil.process_command_line( args = args, master_string = master_phil) work_params = phil.work.extract() if ("--help" in args) : libtbx.phil.parse(master_phil.show()) return if ((work_params.d_min is None) or (work_params.data is None) or ((work_params.model is None) and work_params.scaling.algorithm != "mark1")): raise Usage("cxi.merge " "d_min=4.0 " "data=~/scratch/r0220/006/strong/ " "model=3bz1_3bz2_core.pdb") if ((work_params.rescale_with_average_cell) and (not work_params.set_average_unit_cell)) : raise Usage("If rescale_with_average_cell=True, you must also specify "+ "set_average_unit_cell=True.") miller_set = symmetry( unit_cell = work_params.target_unit_cell, space_group_info = work_params.target_space_group ).build_miller_set( anomalous_flag = not work_params.merge_anomalous, d_min = work_params.d_min) from xfel.cxi.merging.general_fcalc import random_structure i_model = random_structure(work_params) # ---- Augment this code with any special procedures for x scaling scaler = xscaling_manager( miller_set = miller_set, i_model = i_model, params = work_params) scaler.read_all() sg = miller_set.space_group() pg = sg.build_derived_laue_group() rational_ops = [] for symop in pg: rational_ops.append((matrix.sqr(symop.r().transpose().as_rational()), symop.r().as_hkl())) # miller_set.show_summary() uc = work_params.target_unit_cell hkl_asu = scaler.observations["hkl_id"] imageno = scaler.observations["frame_id"] intensi = scaler.observations["i"] sigma_i = scaler.observations["sigi"] lookup = scaler.millers["merged_asu_hkl"] origH = scaler.observations["H"] origK = scaler.observations["K"] origL = scaler.observations["L"] from cctbx.miller import map_to_asu sgtype = miller_set.space_group_info().type() aflag = miller_set.anomalous_flag() from cctbx.array_family import flex # FIXME in here perform the mapping to ASU for both the original and other # index as an array-wise manipulation to make things a bunch faster... # however this also uses a big chunk of RAM... FIXME also in here use # cb_op.apply(indices) to get the indices reindexed... original_indices = flex.miller_index() for x in xrange(len(scaler.observations["hkl_id"])): original_indices.append(lookup[hkl_asu[x]]) from cctbx.sgtbx import change_of_basis_op I23 = change_of_basis_op('k, -h, l') other_indices = I23.apply(original_indices) map_to_asu(sgtype, aflag, original_indices) map_to_asu(sgtype, aflag, other_indices) # FIXME would be useful in here to have a less expensive way of finding the # symmetry operation which gave the map to the ASU - perhaps best way is to # make a new C++ map_to_asu which records this. # FIXME in here recover the original frame structure of the data to # logical frame objetcs - N.B. the frame will need to be augmented to test # alternative indexings # construct table of start / end indices for frames: now using Python # range indexing starts = [0] ends = [] for x in xrange(1, len(scaler.observations["hkl_id"])): if imageno[x] != imageno[x - 1]: ends.append(x) starts.append(x) ends.append(len(scaler.observations["hkl_id"])) keep_start = [] keep_end = [] for j, se in enumerate(zip(starts, ends)): print 'processing frame %d: %d to %d' % (j, se[0], se[1]) s, e = se isig = sum(i / s for i, s in zip(intensi[s:e], sigma_i[s:e])) / (e - s) dmin = 100.0 for x in xrange(s, e): d = uc.d(lookup[hkl_asu[x]]) if d < dmin: dmin = d if isig > 6.0 and dmin < 3.2: keep_start.append(s) keep_end.append(e) starts = keep_start ends = keep_end print 'Keeping %d frames' % len(starts) # then start running the comparison code frames = [] for s, e in zip(starts, ends): # FIXME need this from remap to ASU misym = [0 for x in range(s, e)] indices = [original_indices[x] for x in range(s, e)] other = [other_indices[x] for x in range(s, e)] intensities = intensi[s:e] sigmas = sigma_i[s:e] frames.append(Frame(uc, indices, other, intensities, sigmas)) cycle = 0 total_nref = sum([len(f.get_indices()) for f in frames]) # pre-scale the data - first determine average ln(k), B; then apply kbs = [f.kb() for f in frames] mn_k = sum([kb[0] for kb in kbs]) / len(kbs) mn_B = sum([kb[1] for kb in kbs]) / len(kbs) for f in frames: f.scale_to_kb(mn_k, mn_B) while True: print 'Analysing %d frames' % len(frames) print 'Cycle %d' % cycle cycle += 1 print 'Power spectrum' fn = frame_numbers(frames) for j in sorted(fn): print '%4d %4d' % (j, fn[j]) nref_cycle = sum([len(f.get_indices()) for f in frames]) assert(nref_cycle == total_nref) # first work on the original indices import numpy common_reflections = numpy.zeros((len(frames), len(frames)), dtype = numpy.short) obs = { } # for other hand add -j for j, f in enumerate(frames): indices = set(f.get_indices()) for i in indices: _i = tuple(i) if not _i in obs: obs[_i] = [] obs[_i].append(j) for hkl in obs: obs[hkl].sort() for j, f1 in enumerate(obs[hkl][:-1]): for f2 in obs[hkl][j + 1:]: if f1 * f2 > 0: common_reflections[(abs(f1), abs(f2))] += 1 cmn_rfl_list = [] for f1 in range(len(frames)): for f2 in range(f1 + 1, len(frames)): if common_reflections[(f1, f2)] > 10: cmn_rfl_list.append((common_reflections[(f1, f2)], f1, f2)) cmn_rfl_list.sort() cmn_rfl_list.reverse() joins = [] used = [] for n, f1, f2 in cmn_rfl_list: if f1 in used or f2 in used: continue _cc = frames[f1].cc(frames[f2]) # really only need to worry about f2 which will get merged... # merging multiple files together should be OK provided they are # correctly sorted (though the order should not matter anyhow?) # anyhow they are sorted anyway... ah as f2 > f1 then just sorting # the list by f2 will make sure the data cascase correctly. # p-value small (3% ish) for cc > 0.6 for > 10 observations - # necessary as will be correlated due to Wilson curves though # with B factor < 10 this is less of an issue if _cc[0] > 10 and _cc[1] > 0.6: print '%4d %.3f' % _cc, f1, f2 joins.append((f2, f1)) used.append(f2) if not joins: print 'No pairs found' break joins.sort() joins.reverse() for j2, j1 in joins: rmerge = frames[j1].merge(frames[j2]) if rmerge: print 'R: %4d %4d %6.3f' % (j1, j2, rmerge) else: print 'R: %4d %4d ------' % (j1, j2) all_joins = [j for j in joins] # then do the same for the alternative indices other_reflections = numpy.zeros((len(frames), len(frames)), dtype = numpy.short) obs = { } # for other hand add -j for j, f in enumerate(frames): indices = set(f.get_indices()) for i in indices: _i = tuple(i) if not _i in obs: obs[_i] = [] obs[_i].append(j) indices = set(f.get_other()) for i in indices: _i = tuple(i) if not _i in obs: obs[_i] = [] obs[_i].append(-j) for hkl in obs: obs[hkl].sort() for j, f1 in enumerate(obs[hkl][:-1]): for f2 in obs[hkl][j + 1:]: if f1 * f2 < 0: other_reflections[(abs(f1), abs(f2))] += 1 oth_rfl_list = [] for f1 in range(len(frames)): for f2 in range(f1 + 1, len(frames)): if other_reflections[(f1, f2)] > 10: oth_rfl_list.append((other_reflections[(f1, f2)], f1, f2)) joins = [] oth_rfl_list.sort() oth_rfl_list.reverse() for n, f1, f2 in oth_rfl_list: if f1 in used or f2 in used: continue _cc = frames[f1].cc_other(frames[f2]) # really only need to worry about f2 which will get merged... # merging multiple files together should be OK provided they are # correctly sorted (though the order should not matter anyhow?) # anyhow they are sorted anyway... ah as f2 > f1 then just sorting # the list by f2 will make sure the data cascase correctly. # p-value small (3% ish) for cc > 0.6 for > 10 observations - # necessary as will be correlated due to Wilson curves though # with B factor < 10 this is less of an issue if _cc[0] > 10 and _cc[1] > 0.6: print '%4d %.3f' % _cc, f1, f2 joins.append((f2, f1)) used.append(f2) all_joins += joins if not all_joins: break joins.sort() joins.reverse() for j2, j1 in joins: frames[j2].reindex() rmerge = frames[j1].merge(frames[j2]) if rmerge: print 'R: %4d %4d %6.3f' % (j1, j2, rmerge) else: print 'R: %4d %4d ------' % (j1, j2) continue frames.sort() print 'Biggest few: #frames; #unique refl' j = -1 while frames[j].get_frames() > 1: print frames[j].get_frames(), frames[j].get_unique_indices() frames[j].output_as_scalepack(sg, 'scalepack-%d.sca' % j) j -= 1 return
def _compute_rij_wij(self, use_cache=True): """Compute the rij_wij matrix.""" n_lattices = self._lattices.size() n_sym_ops = len(self._sym_ops) NN = n_lattices * n_sym_ops self.rij_matrix = flex.double(flex.grid(NN, NN), 0.0) if self._weights is None: self.wij_matrix = None else: self.wij_matrix = flex.double(flex.grid(NN, NN), 0.0) indices = {} space_group_type = self._data.space_group().type() for cb_op in self._sym_ops: cb_op = sgtbx.change_of_basis_op(cb_op) indices_reindexed = cb_op.apply(self._data.indices()) miller.map_to_asu(space_group_type, False, indices_reindexed) indices[cb_op.as_xyz()] = indices_reindexed def _compute_rij_matrix_one_row_block(i): rij_cache = {} n_sym_ops = len(self._sym_ops) NN = n_lattices * n_sym_ops from scipy import sparse rij_row = [] rij_col = [] rij_data = [] if self._weights is not None: wij_row = [] wij_col = [] wij_data = [] else: wij = None i_lower, i_upper = self._lattice_lower_upper_index(i) intensities_i = self._data.data()[i_lower:i_upper] for j in range(n_lattices): j_lower, j_upper = self._lattice_lower_upper_index(j) intensities_j = self._data.data()[j_lower:j_upper] for k, cb_op_k in enumerate(self._sym_ops): cb_op_k = sgtbx.change_of_basis_op(cb_op_k) indices_i = indices[cb_op_k.as_xyz()][i_lower:i_upper] for kk, cb_op_kk in enumerate(self._sym_ops): if i == j and k == kk: # don't include correlation of dataset with itself continue cb_op_kk = sgtbx.change_of_basis_op(cb_op_kk) ik = i + (n_lattices * k) jk = j + (n_lattices * kk) key = (i, j, str(cb_op_k.inverse() * cb_op_kk)) if use_cache and key in rij_cache: cc, n = rij_cache[key] else: indices_j = indices[ cb_op_kk.as_xyz()][j_lower:j_upper] matches = miller.match_indices( indices_i, indices_j) pairs = matches.pairs() isel_i = pairs.column(0) isel_j = pairs.column(1) isel_i = isel_i.select( self._patterson_group.epsilon( indices_i.select(isel_i)) == 1) isel_j = isel_j.select( self._patterson_group.epsilon( indices_j.select(isel_j)) == 1) corr = flex.linear_correlation( intensities_i.select(isel_i), intensities_j.select(isel_j), ) if corr.is_well_defined(): cc = corr.coefficient() n = corr.n() rij_cache[key] = (cc, n) else: cc = None n = None if n < self._min_pairs: continue if cc is not None and n is not None: if self._weights == "count": wij_row.extend([ik, jk]) wij_col.extend([jk, ik]) wij_data.extend([n, n]) elif self._weights == "standard_error": assert n > 2 # http://www.sjsu.edu/faculty/gerstman/StatPrimer/correlation.pdf se = math.sqrt((1 - cc**2) / (n - 2)) wij = 1 / se wij_row.extend([ik, jk]) wij_col.extend([jk, ik]) wij_data.extend([wij, wij]) rij_row.append(ik) rij_col.append(jk) rij_data.append(cc) rij = sparse.coo_matrix((rij_data, (rij_row, rij_col)), shape=(NN, NN)) if self._weights is not None: wij = sparse.coo_matrix((wij_data, (wij_row, wij_col)), shape=(NN, NN)) return rij, wij from libtbx import easy_mp args = [(i, ) for i in range(n_lattices)] results = easy_mp.parallel_map( _compute_rij_matrix_one_row_block, args, processes=self._nproc, iterable_type=easy_mp.posiargs, method="multiprocessing", ) rij_matrix = None wij_matrix = None for i, (rij, wij) in enumerate(results): if rij_matrix is None: rij_matrix = rij else: rij_matrix += rij if wij is not None: if wij_matrix is None: wij_matrix = wij else: wij_matrix += wij self.rij_matrix = flex.double(rij_matrix.todense()) if wij_matrix is not None: import numpy as np self.wij_matrix = flex.double(wij_matrix.todense().astype( np.float64)) return self.rij_matrix, self.wij_matrix
def export_xds_ascii(integrated_data, experiment_list, params, var_model=(1, 0)): """Export data from integrated_data corresponding to experiment_list to an XDS_ASCII.HKL formatted text file.""" from dials.array_family import flex # for the moment assume (and assert) that we will convert data from exactly # one lattice... assert len(experiment_list) == 1 # select reflections that are assigned to an experiment (i.e. non-negative id) integrated_data = integrated_data.select(integrated_data["id"] >= 0) assert max(integrated_data["id"]) == 0 # export for xds_ascii should only be for non-scaled reflections assert any([ i in integrated_data for i in ["intensity.sum.value", "intensity.prf.value"] ]) integrated_data = filter_reflection_table( integrated_data, intensity_choice=params.intensity, partiality_threshold=params.mtz.partiality_threshold, combine_partials=params.mtz.combine_partials, min_isigi=params.mtz.min_isigi, filter_ice_rings=params.mtz.filter_ice_rings, d_min=params.mtz.d_min, ) # calculate the scl = lp/dqe correction for outputting but don't apply it as # it has already been applied in filter_reflection_table integrated_data, scl = FilteringReductionMethods.calculate_lp_qe_correction_and_filter( integrated_data) experiment = experiment_list[0] # sort data before output nref = len(integrated_data["miller_index"]) indices = flex.size_t_range(nref) import copy unique = copy.deepcopy(integrated_data["miller_index"]) from cctbx.miller import map_to_asu map_to_asu(experiment.crystal.get_space_group().type(), False, unique) perm = sorted(indices, key=lambda k: unique[k]) integrated_data = integrated_data.select(flex.size_t(perm)) from scitbx import matrix from rstbx.cftbx.coordinate_frame_helpers import align_reference_frame assert not experiment.goniometer is None unit_cell = experiment.crystal.get_unit_cell() from scitbx.array_family import flex assert not experiment.scan is None image_range = experiment.scan.get_image_range() phi_start, phi_range = experiment.scan.get_image_oscillation( image_range[0]) # gather the required information for the reflection file nref = len(integrated_data["miller_index"]) zdet = flex.double(integrated_data["xyzcal.px"].parts()[2]) miller_index = integrated_data["miller_index"] # profile correlation if "profile.correlation" in integrated_data: prof_corr = 100.0 * integrated_data["profile.correlation"] else: prof_corr = flex.double(nref, 100.0) # partiality if "partiality" in integrated_data: partiality = 100 * integrated_data["partiality"] else: prof_corr = flex.double(nref, 100.0) if "intensity.sum.value" in integrated_data: I = integrated_data["intensity.sum.value"] V = integrated_data["intensity.sum.variance"] assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) else: I = integrated_data["intensity.prf.value"] V = integrated_data["intensity.prf.variance"] assert V.all_gt(0) V = var_model[0] * (V + var_model[1] * I * I) sigI = flex.sqrt(V) fout = open(params.xds_ascii.hklout, "w") # first write the header - in the "standard" coordinate frame... panel = experiment.detector[0] fast = panel.get_fast_axis() slow = panel.get_slow_axis() Rd = align_reference_frame(fast, (1, 0, 0), slow, (0, 1, 0)) print("Coordinate change:") print("%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n%5.2f %5.2f %5.2f\n" % Rd.elems) fast = Rd * fast slow = Rd * slow qx, qy = panel.get_pixel_size() nx, ny = panel.get_image_size() distance = matrix.col(Rd * panel.get_origin()).dot( matrix.col(Rd * panel.get_normal())) org = Rd * (matrix.col(panel.get_origin()) - distance * matrix.col(panel.get_normal())) orgx = -org.dot(fast) / qx orgy = -org.dot(slow) / qy UB = Rd * matrix.sqr(experiment.crystal.get_A()) real_space_ABC = UB.inverse().elems axis = Rd * experiment.goniometer.get_rotation_axis() beam = Rd * experiment.beam.get_s0() cell_fmt = "%9.3f %9.3f %9.3f %7.3f %7.3f %7.3f" axis_fmt = "%9.3f %9.3f %9.3f" fout.write("\n".join([ "!FORMAT=XDS_ASCII MERGE=FALSE FRIEDEL'S_LAW=TRUE", "!Generated by dials.export", "!DATA_RANGE= %d %d" % image_range, "!ROTATION_AXIS= %9.6f %9.6f %9.6f" % axis.elems, "!OSCILLATION_RANGE= %f" % phi_range, "!STARTING_ANGLE= %f" % phi_start, "!STARTING_FRAME= %d" % image_range[0], "!SPACE_GROUP_NUMBER= %d" % experiment.crystal.get_space_group().type().number(), "!UNIT_CELL_CONSTANTS= %s" % (cell_fmt % unit_cell.parameters()), "!UNIT_CELL_A-AXIS= %s" % (axis_fmt % real_space_ABC[0:3]), "!UNIT_CELL_B-AXIS= %s" % (axis_fmt % real_space_ABC[3:6]), "!UNIT_CELL_C-AXIS= %s" % (axis_fmt % real_space_ABC[6:9]), "!X-RAY_WAVELENGTH= %f" % experiment.beam.get_wavelength(), "!INCIDENT_BEAM_DIRECTION= %f %f %f" % beam.elems, "!NX= %d NY= %d QX= %f QY= %f" % (nx, ny, qx, qy), "!ORGX= %9.2f ORGY= %9.2f" % (orgx, orgy), "!DETECTOR_DISTANCE= %8.3f" % distance, "!DIRECTION_OF_DETECTOR_X-AXIS= %9.5f %9.5f %9.5f" % fast.elems, "!DIRECTION_OF_DETECTOR_Y-AXIS= %9.5f %9.5f %9.5f" % slow.elems, "!VARIANCE_MODEL= %7.3e %7.3e" % var_model, "!NUMBER_OF_ITEMS_IN_EACH_DATA_RECORD=12", "!ITEM_H=1", "!ITEM_K=2", "!ITEM_L=3", "!ITEM_IOBS=4", "!ITEM_SIGMA(IOBS)=5", "!ITEM_XD=6", "!ITEM_YD=7", "!ITEM_ZD=8", "!ITEM_RLP=9", "!ITEM_PEAK=10", "!ITEM_CORR=11", "!ITEM_PSI=12", "!END_OF_HEADER", "", ])) # then write the data records s0 = Rd * matrix.col(experiment.beam.get_s0()) for j in range(nref): x, y, z = integrated_data["xyzcal.px"][j] phi = phi_start + z * phi_range h, k, l = miller_index[j] X = (UB * (h, k, l)).rotate(axis, phi, deg=True) s = s0 + X g = s.cross(s0).normalize() f = (s - s0).normalize() # find component of beam perpendicular to f, e e = -(s + s0).normalize() if h == k and k == l: u = (h, -h, 0) else: u = (k - l, l - h, h - k) q = ((matrix.col(u).transpose() * UB.inverse()).normalize().transpose().rotate(axis, phi, deg=True)) psi = q.angle(g, deg=True) if q.dot(e) < 0: psi *= -1 fout.write("%d %d %d %f %f %f %f %f %f %.1f %.1f %f\n" % ( h, k, l, I[j], sigI[j], x, y, z, scl[j], partiality[j], prof_corr[j], psi, )) fout.write("!END_OF_DATA\n") fout.close() logger.info("Output %d reflections to %s" % (nref, params.xds_ascii.hklout))
def exercise_asu(): sg_type = sgtbx.space_group_type("P 41") asu = sgtbx.reciprocal_space_asu(sg_type) miller_indices = flex.miller_index(((1,2,3), (3,5,0))) for h in miller_indices: h_eq = miller.sym_equiv_indices(sg_type.group(), h) for i_eq in xrange(h_eq.multiplicity(False)): h_i = h_eq(i_eq) for anomalous_flag in (False,True): a = miller.asym_index(sg_type.group(), asu, h_i.h()) assert a.h() == h o = a.one_column(anomalous_flag) assert o.i_column() == 0 t = a.two_column(anomalous_flag) assert t.h() == h assert (o.h() != h) == (t.i_column() == 1) assert not anomalous_flag or (t.i_column() != 0) == h_i.friedel_flag() assert anomalous_flag or t.i_column() == 0 miller.map_to_asu(sg_type, False, miller_indices) data = flex.double((0,0)) miller.map_to_asu(sg_type, False, miller_indices, data) miller.map_to_asu(sg_type, False, miller_indices, data, False) miller.map_to_asu(sg_type, False, miller_indices, data, True) data = flex.complex_double((0,0)) miller.map_to_asu(sg_type, False, miller_indices, data) data = flex.hendrickson_lattman(((1,2,3,4),(2,3,4,5))) miller.map_to_asu(sg_type, False, miller_indices, data) for sg_symbol in ("P 41", "P 31 1 2"): exercise_map_to_asu(sg_symbol) # sg_type = sgtbx.space_group_type("P 2") miller_indices = flex.miller_index(((1,2,3), (-1,-2,-3))) assert not miller.is_unique_set_under_symmetry( space_group_type=sg_type, anomalous_flag=False, miller_indices=miller_indices) assert miller.is_unique_set_under_symmetry( space_group_type=sg_type, anomalous_flag=True, miller_indices=miller_indices) assert list(miller.unique_under_symmetry_selection( space_group_type=sg_type, anomalous_flag=False, miller_indices=miller_indices)) == [0] assert list(miller.unique_under_symmetry_selection( space_group_type=sg_type, anomalous_flag=True, miller_indices=miller_indices)) == [0,1]
def exercise_asu(): sg_type = sgtbx.space_group_type("P 41") asu = sgtbx.reciprocal_space_asu(sg_type) miller_indices = flex.miller_index(((1,2,3), (3,5,0))) for h in miller_indices: h_eq = miller.sym_equiv_indices(sg_type.group(), h) for i_eq in range(h_eq.multiplicity(False)): h_i = h_eq(i_eq) for anomalous_flag in (False,True): a = miller.asym_index(sg_type.group(), asu, h_i.h()) assert a.h() == h o = a.one_column(anomalous_flag) assert o.i_column() == 0 t = a.two_column(anomalous_flag) assert t.h() == h assert (o.h() != h) == (t.i_column() == 1) assert not anomalous_flag or (t.i_column() != 0) == h_i.friedel_flag() assert anomalous_flag or t.i_column() == 0 miller.map_to_asu(sg_type, False, miller_indices) data = flex.double((0,0)) miller.map_to_asu(sg_type, False, miller_indices, data) miller.map_to_asu(sg_type, False, miller_indices, data, False) miller.map_to_asu(sg_type, False, miller_indices, data, True) data = flex.complex_double((0,0)) miller.map_to_asu(sg_type, False, miller_indices, data) data = flex.hendrickson_lattman(((1,2,3,4),(2,3,4,5))) miller.map_to_asu(sg_type, False, miller_indices, data) for sg_symbol in ("P 41", "P 31 1 2"): exercise_map_to_asu(sg_symbol) # sg_type = sgtbx.space_group_type("P 2") miller_indices = flex.miller_index(((1,2,3), (-1,-2,-3))) assert not miller.is_unique_set_under_symmetry( space_group_type=sg_type, anomalous_flag=False, miller_indices=miller_indices) assert miller.is_unique_set_under_symmetry( space_group_type=sg_type, anomalous_flag=True, miller_indices=miller_indices) assert list(miller.unique_under_symmetry_selection( space_group_type=sg_type, anomalous_flag=False, miller_indices=miller_indices)) == [0] assert list(miller.unique_under_symmetry_selection( space_group_type=sg_type, anomalous_flag=True, miller_indices=miller_indices)) == [0,1]