def intensities(self): ''' Compare the intensities. ''' from dials.array_family import flex # Sort by resolution d = self.refl1['d'] index = flex.size_t(reversed(sorted(range(len(d)), key=lambda x: d[x]))) self.refl1.reorder(index) self.refl2.reorder(index) # Get the intensities I1 = self.refl1['intensity.sum.value'] I2 = self.refl2['intensity.sum.value'] S1 = flex.sqrt(self.refl1['intensity.sum.variance']) S2 = flex.sqrt(self.refl2['intensity.sum.variance']) xyz1 = self.refl1['xyzcal.px'] xyz2 = self.refl2['xyzcal.px'] # Compute chunked statistics corr = [] R = [] scale = [] res = [] for i in range(len(self.refl1) // 1000): # Get the chunks of data a = i * 1000 b = (i+1) * 1000 II1 = I1[a:b] II2 = I2[a:b] res.append(d[a]) # Compute the mean and standard deviation per chunk mv1 = flex.mean_and_variance(II1) mv2 = flex.mean_and_variance(II2) m1 = mv1.mean() m2 = mv2.mean() s1 = mv1.unweighted_sample_standard_deviation() s2 = mv2.unweighted_sample_standard_deviation() # compute the correlation coefficient r = (1/(len(II1) - 1))*sum(((II1[j] - m1) / s1) * ((II2[j] - m2) / s2) for j in range(len(II1))) corr.append(r) # Compute the scale between the chunks s = sum(II1) / sum(II2) scale.append(s) # Compute R between the chunks r = sum(abs(abs(II1[j]) - abs(s * II2[j])) for j in range(len(II1))) \ / sum(abs(II1[j]) for j in range(len(II1))) R.append(r) from matplotlib import pylab pylab.plot(corr, label="CC") pylab.plot(R, label="R") pylab.plot(scale, label="K") pylab.legend() pylab.show()
def cctbx_i_over_sigi_ms_from_dials_data(reflections, cctbx_crystal_symmetry): from dials.array_family import flex from cctbx.miller import set as miller_set refl = reflections.select(reflections['intensity.sum.variance'] > 0) return miller_set(cctbx_crystal_symmetry, refl['miller_index']).array( data=refl['intensity.sum.value'], sigmas=flex.sqrt(refl['intensity.sum.variance']))
def _xl_unit_cell_derivatives(self, isel, parameterisation=None, reflections=None): # Get required data h = self._h.select(isel) B = self._B.select(isel) wl = self._wavelength.select(isel) # get derivatives of the B matrix wrt the parameters dB_dxluc_p = [None if der is None else flex.mat3_double(len(isel), der.elems) \ for der in parameterisation.get_ds_dp(use_none_as_null=True)] d2theta_dp = [] # loop through the parameters for der in dB_dxluc_p: if der is None: d2theta_dp.append(None) continue r0 = B * h dr0 = der * h r0len = r0.norms() dr0len = dr0.dot(r0) / r0len # 2theta = 2 * arcsin( |r0| / (2 * |s0| ) ) sintheta = 0.5 * r0len * wl fac = 1.0 / flex.sqrt(flex.double(len(wl), 1.0) - sintheta**2) val = fac * wl * dr0len d2theta_dp.append(val) return d2theta_dp
def get_pix_coords(wavelength, A, mill_arr, detector, delta_i=0.02): """ Code copied from sim.py courtesy of Aaron and Tara """ s0=col((0,0,-1/wavelength)) q=flex.vec3_double([A*col(idx) for idx in mill_arr.indices().as_vec3_double()]) s0_hat=flex.vec3_double([s0.normalize()]*len(q)) q_hat=q.each_normalize() #q_hat.cross(flex.vec3_double([s0_hat]*len(q_hat))) e1_hat = q_hat.cross(s0_hat) c0_hat = s0_hat.cross(e1_hat) q_len_sq = flex.double([col(v).length_sq() for v in q]) a_side=q_len_sq*wavelength/2 b_side=flex.sqrt(q_len_sq)-a_side**2 #flex.vec3_double([sqrt(q.length_sq()-a_side**2 for idx in mill_arr)]) r_vec=flex.vec3_double(-a_side*s0_hat+b_side*c0_hat) s1=r_vec+s0 EQ=q+s0 len_EQ=flex.double([col(v).length() for v in EQ]) ratio=len_EQ*wavelength indices = flex.miller_index() coords =flex.vec2_double() for i in range(len(s1)): if ratio[i] > 1 - delta_i and ratio[i] < 1 + delta_i: indices.append(mill_arr.indices()[i]) pix = detector[0].get_ray_intersection_px(s1[i]) if detector[0].is_coord_valid(pix): coords.append(pix) return coords, indices
def jacobian_callable(self,values): PB = self.get_partiality_array(values) EXP = flex.exp(-2.*values.BFACTOR*self.DSSQ) G_terms = (EXP * PB * self.ICALCVEC) B_terms = (values.G * EXP * PB * self.ICALCVEC)*(-2.*self.DSSQ) P_terms = (values.G * EXP * self.ICALCVEC) thetax = values.thetax; thetay = values.thetay; Rx = matrix.col((1,0,0)).axis_and_angle_as_r3_rotation_matrix(thetax) dRx_dthetax = matrix.col((1,0,0)).axis_and_angle_as_r3_derivative_wrt_angle(thetax) Ry = matrix.col((0,1,0)).axis_and_angle_as_r3_rotation_matrix(thetay) dRy_dthetay = matrix.col((0,1,0)).axis_and_angle_as_r3_derivative_wrt_angle(thetay) ref_ori = matrix.sqr(self.ORI.reciprocal_matrix()) miller_vec = self.MILLER.as_vec3_double() ds1_dthetax = flex.mat3_double(len(self.MILLER),Ry * dRx_dthetax * ref_ori) * miller_vec ds1_dthetay = flex.mat3_double(len(self.MILLER),dRy_dthetay * Rx * ref_ori) * miller_vec s1vec = self.get_s1_array(values) s1lenvec = flex.sqrt(s1vec.dot(s1vec)) dRh_dthetax = s1vec.dot(ds1_dthetax)/s1lenvec dRh_dthetay = s1vec.dot(ds1_dthetay)/s1lenvec rs = values.RS Rh = self.get_Rh_array(values) rs_sq = rs*rs denomin = (2. * Rh * Rh + rs_sq) dPB_dRh = -PB * 4. * Rh / denomin dPB_dthetax = dPB_dRh * dRh_dthetax dPB_dthetay = dPB_dRh * dRh_dthetay Px_terms = P_terms * dPB_dthetax; Py_terms = P_terms * dPB_dthetay dPB_drs = 4 * rs * Rh * Rh / (denomin * denomin) Prs_terms = P_terms * dPB_drs return [G_terms,B_terms,Prs_terms,Px_terms,Py_terms]
def spot_resolution_shells(imagesets, reflections, params): goniometer = imagesets[0].get_goniometer() from dials.algorithms.indexing import indexer from dials.array_family import flex mapped_reflections = flex.reflection_table() for i, imageset in enumerate(imagesets): if 'imageset_id' in reflections: sel = (reflections['imageset_id'] == i) else: sel = (reflections['id'] == i) if isinstance(reflections['id'], flex.size_t): reflections['id'] = reflections['id'].as_int() refl = indexer.indexer_base.map_spots_pixel_to_mm_rad( reflections.select(sel), imageset.get_detector(), imageset.get_scan()) indexer.indexer_base.map_centroids_to_reciprocal_space( refl, imageset.get_detector(), imageset.get_beam(), imageset.get_goniometer()) mapped_reflections.extend(refl) reflections = mapped_reflections two_theta_array = reflections['rlp'].norms() h0 = flex.weighted_histogram(two_theta_array ** 2, n_slots=params.shells) n = h0.slots() d = 1.0 / flex.sqrt(h0.slot_centers()) for j in range(params.shells): print '%d %f %d' % (j, d[j], n[j])
def plot_one_panel(self, ax, rlist): RMSD = flex.sqrt(rlist['background.mse']) MEAN = rlist['background.mean'] RMSD = RMSD / MEAN x, y, z = rlist['xyzcal.px'].parts() hex_ax = ax.hexbin( x.as_numpy_array(), y.as_numpy_array(), C=RMSD.as_numpy_array(), gridsize=self.gridsize, vmin=0, vmax=1 ) return hex_ax
def rmsd_vs_ios(self, rlist): ''' Analyse the correlations. ''' from os.path import join RMSD = flex.sqrt(rlist['background.mse']) MEAN = rlist['background.mean'] RMSD = RMSD / MEAN I = rlist['intensity.sum.value'] I_sig = flex.sqrt(rlist['intensity.sum.variance']) I_over_S = I / I_sig mask = I_over_S > 0.1 I_over_S = I_over_S.select(mask) RMSD = RMSD.select(mask) fig = pyplot.figure() pyplot.title("Background Model CVRMSD vs Log I/Sigma") cax = pyplot.hexbin(flex.log(I_over_S), RMSD, gridsize=100) cbar = pyplot.colorbar(cax) cax.axes.set_xlabel("Log I/Sigma") cax.axes.set_ylabel("Background Model CVRMSD") cbar.ax.set_ylabel("# reflections") fig.savefig(join(self.directory, "background_model_cvrmsd_vs_ios.png")) pyplot.close()
def i_over_s_hist(self, rlist): ''' Analyse the correlations. ''' from os.path import join I = rlist['intensity.sum.value'] I_sig = flex.sqrt(rlist['intensity.sum.variance']) I_over_S = I / I_sig fig = pyplot.figure() pyplot.title("Log I/Sigma histogram") pyplot.hist(flex.log(I_over_S), bins=20) pyplot.xlabel("Log I/Sigma") pyplot.ylabel("# reflections") fig.savefig(join(self.directory, "ioversigma_hist")) pyplot.close()
def centroid_diff_hist(self, rlist, threshold): ''' Analyse the correlations. ''' from os.path import join I = rlist['intensity.sum.value'] I_sig = flex.sqrt(rlist['intensity.sum.variance']) I_over_S = I / I_sig mask = I_over_S > threshold rlist = rlist.select(mask) assert(len(rlist) > 0) xc, yc, zc = rlist['xyzcal.px'].parts() xo, yo, zo = rlist['xyzobs.px.value'].parts() xd = xo - xc yd = yo - yc zd = zo - zc diff = flex.sqrt(xd*xd + yd*yd + zd*zd) fig = pyplot.figure() pyplot.title("Difference between observed and calculated") pyplot.hist(diff, bins=20) pyplot.xlabel("Difference in position") pyplot.ylabel("# reflections") fig.savefig(join(self.directory, "centroid_diff_hist.png")) pyplot.close()
def rmsd_hist(self, rlist): ''' Analyse the background RMSD. ''' from os.path import join RMSD = flex.sqrt(rlist['background.mse']) MEAN = rlist['background.mean'] RMSD = RMSD / MEAN fig = pyplot.figure() pyplot.title("Background Model mean histogram") pyplot.hist(RMSD, bins=20) pyplot.xlabel("mean") pyplot.ylabel("# reflections") fig.savefig(join(self.directory, "background_model_cvrmsd_hist")) pyplot.close()
def flex_ios(val, var): ''' Compute I/sigma or return zero for each element. ''' assert(len(val) == len(var)) result = flex.double(len(val),0) indices = flex.size_t(range(len(val))).select(var > 0) val = val.select(indices) var = var.select(indices) assert(var.all_gt(0)) result.set_selected(indices, val / flex.sqrt(var)) return result
def plot_one_panel(self, ax, rlist): I_sig = flex.sqrt(rlist['intensity.%s.variance' %intensity_type]) sel = I_sig > 0 rlist = rlist.select(sel) I_sig = I_sig.select(sel) I = rlist['intensity.%s.value' %intensity_type] I_over_S = I / I_sig x, y, z = rlist['xyzcal.px'].parts() hex_ax = ax.hexbin( x.as_numpy_array(), y.as_numpy_array(), C=flex.log(I_over_S), gridsize=self.gridsize, ) return hex_ax
def __init__(self, reflections, experiment, params=None): if params is None: params = om_scope.extract() # initial filter reflections = reflections.select(reflections.get_flags( reflections.flags.integrated)) # create new column containing the reduced Miller index xl = experiment.crystal symm = cctbx.crystal.symmetry(xl.get_unit_cell(), space_group=xl.get_space_group()) hkl_set = cctbx.miller.set(symm, reflections['miller_index']) asu_set = hkl_set.map_to_asu() reflections['asu_miller_index'] = asu_set.indices() # sort table by reduced hkl reflections.sort('asu_miller_index') if params.observations.integration_type == 'mix': raise Sorry('integration_type=mix is not supported yet') else: ikey = 'intensity.' + params.observations.integration_type + '.value' vkey = 'intensity.' + params.observations.integration_type + '.variance' # filters sel = reflections[vkey] > 0 reflections = reflections.select(sel) if params.observations.i_over_sigma_cutoff > 0: ios = reflections[ikey] / flex.sqrt(reflections[vkey]) sel = ios >= params.observations.i_over_sigma_cutoff reflections = reflections.select(sel) if params.observations.min_multiplicity > 0: sel = minimum_multiplicity_selection(reflections['asu_miller_index'], params.observations.min_multiplicity) reflections = reflections.select(sel) # extract columns of interest gp_idx = reflections['asu_miller_index'] intensity = reflections[ikey] weight = 1. / reflections[vkey] phi = reflections['xyzcal.mm'].parts()[2] scale = flex.double(len(reflections), 1.0) # set up reflection grouping object self._go = GroupedObservations(gp_idx, intensity, weight, phi, scale) return
def i_over_s_vs_z(self, rlist): ''' Plot I/Sigma vs Z. ''' from os.path import join I = rlist['intensity.sum.value'] I_sig = flex.sqrt(rlist['intensity.sum.variance']) I_over_S = I / I_sig x, y, z = rlist['xyzcal.px'].parts() fig = pyplot.figure() pyplot.title("Distribution of I/Sigma vs Z") cax = pyplot.hexbin(z, flex.log(I_over_S), gridsize=100) cax.axes.set_xlabel("z") cax.axes.set_ylabel("Log I/Sigma") cbar = pyplot.colorbar(cax) cbar.ax.set_ylabel("# reflections") fig.savefig(join(self.directory, "ioversigma_vs_z.png")) pyplot.close()
def rmsd_vs_z(self, rlist): ''' Plot I/Sigma vs Z. ''' from os.path import join RMSD = flex.sqrt(rlist['background.mse']) MEAN = rlist['background.mean'] RMSD = RMSD / MEAN x, y, z = rlist['xyzcal.px'].parts() fig = pyplot.figure() pyplot.title("Distribution of Background Model CVRMSD vs Z") cax = pyplot.hexbin(z, RMSD, gridsize=100) cax.axes.set_xlabel("z") cax.axes.set_ylabel("Background Model CVRMSD") cbar = pyplot.colorbar(cax) cbar.ax.set_ylabel("# reflections") fig.savefig(join(self.directory, "background_model_cvrmsd_vs_z.png")) pyplot.close()
def get_beam_gradients(self, reflections): ds0_dbeam_p = self.beam_parameterisation.get_ds_dp() p_names = self.beam_parameterisation.get_param_names() n = len(reflections) U = flex.mat3_double(n, self.U) B = flex.mat3_double(n, self.B) UB = U*B # q is the reciprocal lattice vector, in the lab frame h = reflections['miller_index'].as_vec3_double() q = (UB * h) qlen = q.norms() qlen2 = q.dot(q) q_s0 = q + self.s0 s1 = reflections['s1'] ss = qlen2 + 2 * q.dot(self.s0) + self.s0len2 assert (ss > 0.0).all_eq(True) s = flex.sqrt(ss) sss = s * ss inv_s = 1.0 / s inv_sss = 1.0 / sss # check equation 10 tmp = self.s0len * (q_s0) / s for a, b in zip(s1, tmp): assert approx_equal(a, b) ds1_dp = {} # loop through the parameters for name, der in zip(p_names, ds0_dbeam_p): # term1 term1 = self.us0.dot(der) * q_s0 + self.s0len * (der) term1 = term1 * inv_s # term2 term2 = self.s0len * q_s0 * q_s0.dot(der) term2 = term2 * inv_sss name = 'Beam1' + name # XXXX Hack to get matching keys ds1_dp[name] = {'ds1':(term1 - term2)} return ds1_dp
def get_crystal_unit_cell_gradients(self, reflections): # get derivatives of the B matrix wrt the parameters dB_dxluc_p = self.xl_unit_cell_parameterisation.get_ds_dp() p_names = self.xl_unit_cell_parameterisation.get_param_names() n = len(reflections) U = flex.mat3_double(n, self.U) B = flex.mat3_double(n, self.B) UB = U*B # q is the reciprocal lattice vector, in the lab frame h = reflections['miller_index'].as_vec3_double() q = (UB * h) qlen = q.norms() qlen2 = q.dot(q) q_s0 = q + self.s0 s1 = reflections['s1'] ss = qlen2 + 2 * q.dot(self.s0) + self.s0len2 assert (ss > 0.0).all_eq(True) s = flex.sqrt(ss) sss = s * ss inv_s = 1.0 / s inv_sss = 1.0 / sss ds1_dp = {} # loop through the parameters for name, der in zip(p_names, dB_dxluc_p): # calculate the derivative of q for this parameter dq = U * flex.mat3_double(n, der.elems) * h # term1 term1 = self.s0len * dq term1 = term1 * inv_s # term2 term2 = self.s0len * q_s0 * q_s0.dot(dq) term2 = term2 * inv_sss name = 'Crystal1' + name # XXXX Hack to get matching keys ds1_dp[name] = {'ds1':(term1 - term2)} return ds1_dp
def check_reference(self, reference): ''' Check the reference spots. ''' from dials.array_family import flex from dials.algorithms.image.centroid import centroid_image from math import sqrt # Get a load of stuff I_sim = reference['intensity.sim'] I_exp = reference['intensity.exp'] I_cal = reference['intensity.prf.value'] I_var = reference['intensity.prf.variance'] # Get the transformed shoeboxes profiles = reference['rs_shoebox'] n_sigma = 3 n_sigma2 = 5 grid_size = 4 step_size = n_sigma2 / (grid_size + 0.5) eps = 1e-7 for i in range(len(profiles)): data = profiles[i].data #dmax = flex.max(data) #data = 100 * data / dmax #p = data.as_numpy_array() #p = p.astype(numpy.int) #print p print flex.sum(data), I_exp[i], I_cal[i] #assert(abs(flex.sum(data) - I_exp[i]) < eps) centroid = centroid_image(data) m = centroid.mean() v = centroid.variance() s1 = tuple(sqrt(vv) for vv in v) s2 = tuple(ss * step_size for ss in s1) assert(all(abs(mm - (grid_size + 0.5)) < 0.25 for mm in m)) assert(all(abs(ss2 - n_sigma / n_sigma2) < 0.25 for ss2 in s2)) # Calculate Z Z = (I_cal - I_exp) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f" % (Z_mean, Z_var) from matplotlib import pylab pylab.hist((I_cal - I_exp) / I_exp) pylab.show()
def log_sum_i_sigi_vs_resolution(reflections, imageset, plot_filename=None): d_star_sq = flex.pow2(reflections['rlp'].norms()) hist = get_histogram(d_star_sq) intensities = reflections['intensity.sum.value'] variances = reflections['intensity.sum.variance'] sel = variances > 0 intensities = intensities.select(sel) variances = intensities.select(sel) i_over_sigi = intensities/flex.sqrt(variances) #log_i_over_sigi = flex.log(i_over_sigi) slots = [] for slot in hist.slot_infos(): sel = (d_star_sq > slot.low_cutoff) & (d_star_sq < slot.high_cutoff) if sel.count(True) > 0: slots.append(math.log(flex.sum(i_over_sigi.select(sel)))) else: slots.append(0) if plot_filename is not None: if pyplot is None: raise Sorry("matplotlib must be installed to generate a plot.") fig = pyplot.figure() ax = fig.add_subplot(1,1,1) #ax.bar(hist.slot_centers()-0.5*hist.slot_width(), hist.slots(), ax.scatter(hist.slot_centers()-0.5*hist.slot_width(), slots, s=20, color='blue', marker='o', alpha=0.5) ax.set_xlabel("d_star_sq") ax.set_ylabel("ln(sum(I/sigI))") ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis xticks = ax.get_xticks() xlim = ax.get_xlim() xticks_d = [ uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ] xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks] ax_.set_xticks(xticks) ax_.set_xlim(ax.get_xlim()) ax_.set_xlabel(r"Resolution ($\AA$)") ax_.set_xticklabels(["%.1f" %d for d in xticks_d]) #pyplot.show() pyplot.savefig(plot_filename) pyplot.close()
def _test_Imid_combinations(self): rows = [] results = {} for Imid in self.Imids: combined_intensities = flex.double([]) combined_sigmas = flex.double([]) combined_scales = flex.double([]) combined_indices = flex.miller_index([]) for dataset in self.datasets: Int, Var = _get_Is_from_Imidval(dataset, Imid) Int *= dataset["prescaling_correction"] sigma = flex.sqrt(Var) * dataset["prescaling_correction"] combined_intensities.extend(Int) combined_sigmas.extend(sigma) combined_scales.extend(dataset["inverse_scale_factor"]) combined_indices.extend(dataset["miller_index"]) # apply scale factor before determining merging stats miller_set = miller.set( crystal_symmetry=self.active_scalers[0].experiment.crystal. get_crystal_symmetry(), indices=combined_indices, anomalous_flag=False, ) i_obs = miller.array(miller_set, data=combined_intensities / combined_scales) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas(combined_sigmas / combined_scales) try: rmeas, cchalf = fast_merging_stats(array=i_obs) logger.debug("Imid: %s, Rmeas %s, cchalf %s", Imid, rmeas, cchalf) except RuntimeError: raise DialsMergingStatisticsError( "Unable to merge for intensity combination") # record the results results[Imid] = rmeas res_str = {0: "prf only", 1: "sum only"} if Imid not in res_str: res_str[Imid] = "Imid = " + str(round(Imid, 2)) rows.append( [res_str[Imid], str(round(cchalf, 5)), str(round(rmeas, 5))]) return rows, results
def _reflection_table_to_iobs(table, unit_cell, space_group): miller_set = miller.set( crystal_symmetry=crystal.symmetry( unit_cell=unit_cell, space_group=space_group, assert_is_compatible_unit_cell=False, ), indices=table["asu_miller_index"], anomalous_flag=False, ) i_obs = miller.array( miller_set, data=table["intensity"] / table["inverse_scale_factor"] ) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas(flex.sqrt(table["variance"]) / table["inverse_scale_factor"]) i_obs.set_info(miller.array_info(source="DIALS", source_type="reflection_tables")) return i_obs
def unpack_stddev(self): # the data-to_parameter ratio will control which method for returning e.s.d's data_to_parameter = float(self.N_raw_obs) / self.helper.x.size() self.helper.build_up() if data_to_parameter <= 4. and self.helper.x.size() < 500: # estimate standard deviations by singular value decomposition norm_mat_packed_upper = self.helper.get_normal_matrix() norm_mat_all_elems = self.packed_to_all(norm_mat_packed_upper) NM = sqr(norm_mat_all_elems) from scitbx.linalg.svd import inverse_via_svd svd_inverse,sigma = inverse_via_svd(NM.as_flex_double_matrix()) IA = sqr(svd_inverse) estimated_stddev = flex.double([math.sqrt(IA(i,i)) for i in xrange(self.helper.x.size())]) else: # estimate standard deviations by normal matrix curvatures diagonal_curvatures = self.helper.get_normal_matrix_diagonal() estimated_stddev = flex.sqrt(1./diagonal_curvatures) return self.fitted_as_annotated(estimated_stddev)
def example_array(reflections): """Generate a miller array for a test.""" exp_dict = { "__id__": "crystal", "real_space_a": [1.0, 0.0, 0.0], "real_space_b": [0.0, 1.0, 0.0], "real_space_c": [0.0, 0.0, 1.0], "space_group_hall_symbol": "-P 2yb", } crystal = Crystal.from_dict(exp_dict) ms = miller.set( crystal_symmetry=crystal.get_crystal_symmetry(), indices=reflections["miller_index"], anomalous_flag=False, ) ma = miller.array(ms, data=reflections["intensity"]) ma.set_sigmas(flex.sqrt(reflections["variance"])) return ma
def get_crystal_unit_cell_gradients(self, reflections): # get derivatives of the B matrix wrt the parameters dB_dxluc_p = self.xl_unit_cell_parameterisation.get_ds_dp() p_names = self.xl_unit_cell_parameterisation.get_param_names() n = len(reflections) U = flex.mat3_double(n, self.U) B = flex.mat3_double(n, self.B) UB = U * B # q is the reciprocal lattice vector, in the lab frame h = reflections["miller_index"].as_vec3_double() q = UB * h qlen2 = q.dot(q) q_s0 = q + self.s0 ss = qlen2 + 2 * q.dot(self.s0) + self.s0len2 assert (ss > 0.0).all_eq(True) s = flex.sqrt(ss) sss = s * ss inv_s = 1.0 / s inv_sss = 1.0 / sss ds1_dp = {} # loop through the parameters for name, der in zip(p_names, dB_dxluc_p): # calculate the derivative of q for this parameter dq = U * flex.mat3_double(n, der.elems) * h # term1 term1 = self.s0len * dq term1 = term1 * inv_s # term2 term2 = self.s0len * q_s0 * q_s0.dot(dq) term2 = term2 * inv_sss name = "Crystal1" + name # XXXX Hack to get matching keys ds1_dp[name] = {"ds1": (term1 - term2)} return ds1_dp
def _test_Imid_combinations(self): """Test the different combinations, returning the rows and results dict.""" rows = [] results = {} for Imid in self.Imids: Int, Var = _get_Is_from_Imidval(self.dataset, Imid) miller_set = miller.set( crystal_symmetry=self.experiment.crystal.get_crystal_symmetry( assert_is_compatible_unit_cell=False ), indices=self.dataset["miller_index"], anomalous_flag=False, ) i_obs = miller.array( miller_set, data=( Int * self.dataset["prescaling_correction"] / self.dataset["inverse_scale_factor"] ), ) i_obs.set_observation_type_xray_intensity() i_obs.set_sigmas( flex.sqrt(Var) * self.dataset["prescaling_correction"] / self.dataset["inverse_scale_factor"] ) try: rmeas, cchalf = fast_merging_stats(array=i_obs) logger.debug("Imid: %s, Rmeas %s, cchalf %s", Imid, rmeas, cchalf) except RuntimeError: raise DialsMergingStatisticsError( "Unable to merge for intensity combination" ) # record the results results[Imid] = rmeas res_str = {0: "prf only", 1: "sum only"} if Imid not in res_str: res_str[Imid] = "Imid = " + str(round(Imid, 2)) rows.append([res_str[Imid], str(round(cchalf, 5)), str(round(rmeas, 5))]) return rows, results
def mean_vs_ios(self, rlist): """ Analyse the correlations. """ MEAN = rlist["background.mean"] I = rlist["intensity.sum.value"] I_sig = flex.sqrt(rlist["intensity.sum.variance"]) I_over_S = I / I_sig mask = I_over_S > 0.1 I_over_S = I_over_S.select(mask) MEAN = MEAN.select(mask) fig = pyplot.figure() pyplot.title("Background Model mean vs Log I/Sigma") cax = pyplot.hexbin(flex.log(I_over_S), MEAN, gridsize=100) cbar = pyplot.colorbar(cax) cax.axes.set_xlabel("Log I/Sigma") cax.axes.set_ylabel("Background Model mean") cbar.ax.set_ylabel("# reflections") fig.savefig( os.path.join(self.directory, "background_model_mean_vs_ios.png")) pyplot.close()
def get_amplitudes(self, dials_model, refl_table, test_without_mpi=True): D = dials_model R = refl_table from cctbx.crystal import symmetry from cctbx.miller import array, set as miller_set uc = D.crystal.get_unit_cell() sg = D.crystal.get_space_group() MS = miller_set(symmetry(unit_cell=uc, space_group=sg), anomalous_flag=True, indices=R["miller_index"].select(R["spots_order"])) self.amplitudes = array(MS, data=flex.sqrt( R["spots_mockup_shoebox_sum"].select( R["spots_order"]))) from simtbx.gpu import gpu_energy_channels recommend_device = int(os.environ.get("CCTBX_RECOMMEND_DEVICE", 0)) self.gpu_channels_singleton = gpu_energy_channels( deviceId=recommend_device)
def unpack_stddev(self): # the data-to_parameter ratio will control which method for returning e.s.d's data_to_parameter = float(self.N_raw_obs) / self.helper.x.size() self.helper.build_up() if data_to_parameter <= 4. and self.helper.x.size() < 500: # estimate standard deviations by singular value decomposition norm_mat_packed_upper = self.helper.get_normal_matrix() norm_mat_all_elems = self.packed_to_all(norm_mat_packed_upper) NM = sqr(norm_mat_all_elems) from scitbx.linalg.svd import inverse_via_svd svd_inverse, sigma = inverse_via_svd(NM.as_flex_double_matrix()) IA = sqr(svd_inverse) estimated_stddev = flex.double( [math.sqrt(IA(i, i)) for i in range(self.helper.x.size())]) else: # estimate standard deviations by normal matrix curvatures diagonal_curvatures = self.helper.get_normal_matrix_diagonal() estimated_stddev = flex.sqrt(1. / diagonal_curvatures) return self.fitted_as_annotated(estimated_stddev)
def __init__(self, reflection_file, experiment_file): data = pickle.load(open(reflection_file, "rb")) self.data = data.select(data["intensity.sum.variance"] > 0) i = data["intensity.sum.value"] v = data["intensity.sum.variance"] s = flex.sqrt(v) self.i_s = i / s self.scale = 2 from dxtbx.model.experiment_list import ExperimentListFactory expt = ExperimentListFactory.from_json_file(experiment_file) panel = expt.detectors()[0][0] crystal = expt.crystals()[0] self.s0 = matrix.col(expt.beams()[0].get_s0()) wavelength = expt.beams()[0].get_wavelength() # make a list of observed q positions self.qobs = [] for j in range(data.size()): x, y, z = data["xyzobs.px.value"][j] p = matrix.col(panel.get_pixel_lab_coord((x, y))) q = p.normalize() / wavelength - self.s0 self.qobs.append(q) self.wavelength = wavelength self.panel = panel self.beam = expt.beams()[0] self.crystal = crystal # slurp data from $somewhere imageset = expt.imagesets()[0] self.raw_data = imageset.get_raw_data(0)[0] self.imageset = imageset return
def __init__(self, reflection_file, experiment_file): # make a pie data = pickle.load(open(reflection_file, "rb")) print("%d reflections" % data.size()) self.data = data.select(data["intensity.sum.variance"] > 0) i = data["intensity.sum.value"] v = data["intensity.sum.variance"] s = flex.sqrt(v) self.i_s = i / s from dxtbx.model.experiment_list import ExperimentListFactory expt = ExperimentListFactory.from_json_file(experiment_file) crystal = expt.crystals()[0] self.s0 = matrix.col(expt.beams()[0].get_s0()) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalUnitCellParameterisation, CrystalOrientationParameterisation, ) self.crystal = crystal self.cucp = CrystalUnitCellParameterisation(crystal) self.cop = CrystalOrientationParameterisation(crystal) # 0-point and deltas values = flex.double(self.cucp.get_param_vals() + self.cop.get_param_vals()) offset = flex.double([0.01 * v for v in self.cucp.get_param_vals()] + [0.1, 0.1, 0.1]) initial = crystal.get_unit_cell() initial_score = self.target(values) doohicky = simple_simplex(values, offset, self, 2000) best = doohicky.get_solution() print("Initial cell:", initial) print("Final cell: ", crystal.get_unit_cell()) print("Score change", initial_score, self.target(best, do_print=False)) self.best = best
def centroid_diff_z(self, rlist, threshold): """ Look at the centroid difference in x, y """ I = rlist["intensity.sum.value"] I_sig = flex.sqrt(rlist["intensity.sum.variance"]) I_over_S = I / I_sig mask = I_over_S > threshold rlist = rlist.select(mask) assert len(rlist) > 0 xc, yc, zc = rlist["xyzcal.px"].parts() xo, yo, zo = rlist["xyzobs.px.value"].parts() zd = zo - zc fig = pyplot.figure() pyplot.title("Difference between observed and calculated in Z") cax = pyplot.hexbin(zc, zd, gridsize=100) cax.axes.set_xlabel("z (images)") cax.axes.set_ylabel("Difference in z position") cbar = pyplot.colorbar(cax) cbar.ax.set_ylabel("# Reflections") fig.savefig(os.path.join(self.directory, "centroid_diff_z.png")) pyplot.close()
def get_correlation(cb_op=None): """ Helper function to get CC to the reference given an operator """ # Build a miller array for the experiment reflections exp_miller_indices = miller.set( target_symm, exp_reflections['miller_index_asymmetric'], True) exp_intensities = miller.array( exp_miller_indices, exp_reflections['intensity.sum.value'], flex.sqrt(exp_reflections['intensity.sum.variance'])) if cb_op: exp_intensities = exp_intensities.change_basis( cb_op).map_to_asu() # Extract an array of HKLs from the model to match the experiment HKLs matching_indices = miller.match_multi_indices( miller_indices_unique=model_intensities.indices(), miller_indices=exp_intensities.indices()) # Least squares scaling_result = scaler.fit_experiment_to_reference( model_intensities, exp_intensities, matching_indices) return scaling_result.correlation if scaling_result.correlation is not None else -1
def centroid_diff_z(self, rlist, threshold): ''' Look at the centroid difference in x, y ''' from os.path import join I = rlist['intensity.sum.value'] I_sig = flex.sqrt(rlist['intensity.sum.variance']) I_over_S = I / I_sig mask = I_over_S > threshold rlist = rlist.select(mask) assert(len(rlist) > 0) xc, yc, zc = rlist['xyzcal.px'].parts() xo, yo, zo = rlist['xyzobs.px.value'].parts() zd = zo - zc fig = pyplot.figure() pyplot.title("Difference between observed and calculated in Z") cax = pyplot.hexbin(zc, zd, gridsize=100) cax.axes.set_xlabel("z") cax.axes.set_ylabel("Difference in z position") cbar = pyplot.colorbar(cax) cbar.ax.set_ylabel("# Reflections") fig.savefig(join(self.directory, "centroid_diff_z.png")) pyplot.close()
def _find_nearest_neighbours_single(self, oxyz, pxyz): ''' Find the nearest predicted spot to the observed spot. :param observed: The observed reflections :param predicted: The predicted reflections :returns: (nearest neighbours, distance) ''' from annlib_ext import AnnAdaptor from scitbx.array_family import flex # Create the KD Tree ann = AnnAdaptor(pxyz.as_double().as_1d(), 3) # Query to find all the nearest neighbours ann.query(oxyz.as_double().as_1d()) # Return the nearest neighbours and distances return ann.nn, flex.sqrt(ann.distances)
def get_amplitudes(self, dials_model, refl_table, test_without_mpi=True): from LS49.adse13_187.cyto_batch import parse_input self.params, options = parse_input() D = dials_model R = refl_table from cctbx.crystal import symmetry from cctbx.miller import array, set as miller_set uc = D.crystal.get_unit_cell() sg = D.crystal.get_space_group() MS = miller_set(symmetry(unit_cell=uc, space_group=sg), anomalous_flag=True, indices=R["miller_index"].select(R["spots_order"])) self.amplitudes = array(MS, data=flex.sqrt( R["spots_mockup_shoebox_sum"].select( R["spots_order"]))) from simtbx.gpu import gpu_energy_channels self.gpu_channels_singleton = gpu_energy_channels( deviceId=0) # determine device by rank id later
def modify_errors(self, reflections): '''Formerly sdfac_auto, ha14 method applies sdfac to each-image data assuming negative intensities are normally distributed noise''' assert 0 == (reflections['intensity.sum.variance'] <= 0.0).count(True) I_over_sig = reflections['intensity.sum.value'] / flex.sqrt( reflections['intensity.sum.variance']) negative_I_over_sig = I_over_sig.select(I_over_sig < 0.) #assert that at least a few I/sigmas are less than zero negative_I_over_sig_count = I_over_sig.select(I_over_sig < 0.).size() if negative_I_over_sig_count > 2: # get a rough estimate for the SDFAC, assuming that negative measurements # represent false predictions and therefore normally distributed noise. no_signal = negative_I_over_sig for xns in range(len(no_signal)): no_signal.append(-no_signal[xns]) Stats = flex.mean_and_variance(no_signal) SDFAC = Stats.unweighted_sample_standard_deviation() else: SDFAC = 1. self.logger.log("The applied SDFAC is %7.4f" % SDFAC) reflections['intensity.sum.variance'] *= (SDFAC**2) return reflections
def jacobian_callable(self, values): PB = self.get_partiality_array(values) EXP = flex.exp(-2. * values.BFACTOR * self.DSSQ) G_terms = (EXP * PB * self.ICALCVEC) B_terms = (values.G * EXP * PB * self.ICALCVEC) * (-2. * self.DSSQ) P_terms = (values.G * EXP * self.ICALCVEC) thetax = values.thetax thetay = values.thetay Rx = matrix.col((1, 0, 0)).axis_and_angle_as_r3_rotation_matrix(thetax) dRx_dthetax = matrix.col( (1, 0, 0)).axis_and_angle_as_r3_derivative_wrt_angle(thetax) Ry = matrix.col((0, 1, 0)).axis_and_angle_as_r3_rotation_matrix(thetay) dRy_dthetay = matrix.col( (0, 1, 0)).axis_and_angle_as_r3_derivative_wrt_angle(thetay) ref_ori = matrix.sqr(self.ORI.reciprocal_matrix()) miller_vec = self.MILLER.as_vec3_double() ds1_dthetax = flex.mat3_double(len(self.MILLER), Ry * dRx_dthetax * ref_ori) * miller_vec ds1_dthetay = flex.mat3_double(len(self.MILLER), dRy_dthetay * Rx * ref_ori) * miller_vec s1vec = self.get_s1_array(values) s1lenvec = flex.sqrt(s1vec.dot(s1vec)) dRh_dthetax = s1vec.dot(ds1_dthetax) / s1lenvec dRh_dthetay = s1vec.dot(ds1_dthetay) / s1lenvec rs = values.RS Rh = self.get_Rh_array(values) rs_sq = rs * rs denomin = (2. * Rh * Rh + rs_sq) dPB_dRh = { "lorentzian": -PB * 4. * Rh / denomin, "gaussian": -PB * 4. * math.log(2) * Rh / rs_sq }[self.profile_shape] dPB_dthetax = dPB_dRh * dRh_dthetax dPB_dthetay = dPB_dRh * dRh_dthetay Px_terms = P_terms * dPB_dthetax Py_terms = P_terms * dPB_dthetay return [G_terms, B_terms, 0, Px_terms, Py_terms]
def wilson_outliers(reflections, ice_sel=None, p_cutoff=1e-2): # http://scripts.iucr.org/cgi-bin/paper?ba0032 if ice_sel is None: ice_sel = flex.bool(len(reflections), False) E_cutoff = math.sqrt(-math.log(p_cutoff)) intensities = reflections['intensity.sum.value'] variances = reflections['intensity.sum.variance'] Sigma_n = flex.mean(intensities.select(~ice_sel)) normalised_amplitudes = flex.sqrt(intensities)/math.sqrt(Sigma_n) outliers = normalised_amplitudes >= E_cutoff if outliers.count(True): # iterative outlier rejection inliers = ~outliers outliers.set_selected( inliers, wilson_outliers( reflections.select(inliers), ice_sel.select(inliers))) return outliers
def wilson_outliers(reflections, ice_sel=None, p_cutoff=1e-2): # http://scripts.iucr.org/cgi-bin/paper?ba0032 if ice_sel is None: ice_sel = flex.bool(len(reflections), False) E_cutoff = math.sqrt(-math.log(p_cutoff)) intensities = reflections["intensity.sum.value"] Sigma_n = flex.mean(intensities.select(~ice_sel)) normalised_amplitudes = flex.sqrt(intensities) / math.sqrt(Sigma_n) outliers = normalised_amplitudes >= E_cutoff if outliers.count(True): # iterative outlier rejection inliers = ~outliers outliers.set_selected( inliers, wilson_outliers(reflections.select(inliers), ice_sel.select(inliers)), ) return outliers
def _xl_unit_cell_derivatives(self, isel, parameterisation=None, reflections=None): # Get required data h = self._h.select(isel) B = self._B.select(isel) wl = self._wavelength.select(isel) # get derivatives of the B matrix wrt the parameters dB_dxluc_p = [ None if der is None else flex.mat3_double(len(isel), der.elems) for der in parameterisation.get_ds_dp(use_none_as_null=True) ] d2theta_dp = [] # loop through the parameters for der in dB_dxluc_p: if der is None: d2theta_dp.append(None) continue r0 = B * h dr0 = der * h r0len = r0.norms() dr0len = dr0.dot(r0) / r0len # 2theta = 2 * arcsin( |r0| / (2 * |s0| ) ) sintheta = 0.5 * r0len * wl fac = 1.0 / flex.sqrt( flex.double(len(wl), 1.0) - flex.pow2(sintheta)) val = fac * wl * dr0len d2theta_dp.append(val) return d2theta_dp
def correct(refl_sele, smart_sigmas=True): kapton_correction = image_kapton_correction( panel_size_px=panel_size_px, pixel_size_mm=pixel_size_mm, detector_dist_mm=detector_dist_mm, wavelength_ang=wavelength_ang, reflections_sele=refl_sele, params=self.params, expt=expt, refl=refl, smart_sigmas=smart_sigmas, logger=self.logger, ) k_corr, k_sigmas = kapton_correction() refl_sele["kapton_absorption_correction"] = k_corr if smart_sigmas: refl_sele["kapton_absorption_correction_sigmas"] = k_sigmas # apply corrections and propagate error # term1 = (sig(C)/C)^2 # term2 = (sig(Imeas)/Imeas)^2 # I' = C*I # sig^2(I') = (I')^2*(term1 + term2) integrated_data = refl_sele["intensity.sum.value"] integrated_variance = refl_sele["intensity.sum.variance"] integrated_sigma = flex.sqrt(integrated_variance) term1 = flex.pow(k_sigmas / k_corr, 2) term2 = flex.pow(integrated_sigma / integrated_data, 2) integrated_data *= k_corr integrated_variance = flex.pow(integrated_data, 2) * (term1 + term2) refl_sele["intensity.sum.value"] = integrated_data refl_sele["intensity.sum.variance"] = integrated_variance # order is purposeful: the two lines above require that integrated_data # has already been corrected! else: refl_sele["intensity.sum.value"] *= k_corr refl_sele["intensity.sum.variance"] *= flex.pow2(k_corr) return refl_sele
def ideal_reflection_corr_vs_ios(self, rlist, filename): ''' Analyse the correlations. ''' from os.path import join if 'correlation.ideal.profile' in rlist: corr = rlist['correlation.ideal.profile'] I = rlist['intensity.prf.value'] I_sig = flex.sqrt(rlist['intensity.prf.variance']) mask = I_sig > 0 I = I.select(mask) I_sig = I_sig.select(mask) corr = corr.select(mask) I_over_S = I / I_sig mask = I_over_S > 0.1 I_over_S = I_over_S.select(mask) corr = corr.select(mask) pyplot.title("Reflection correlations vs Log I/Sigma") cax = pyplot.hexbin(flex.log(I_over_S), corr, gridsize=100) cbar = pyplot.colorbar(cax) pyplot.xlabel("Log I/Sigma") pyplot.ylabel("Correlation with reference profile") cbar.ax.set_ylabel("# reflections") pyplot.savefig(join(self.directory, "ideal_%s_corr_vs_ios.png" % filename)) pyplot.close()
def reflection_corr_vs_ios(self, rlist, filename): """ Analyse the correlations. """ corr = rlist["profile.correlation"] I = rlist["intensity.prf.value"] I_sig = flex.sqrt(rlist["intensity.prf.variance"]) mask = I_sig > 0 I = I.select(mask) I_sig = I_sig.select(mask) corr = corr.select(mask) I_over_S = I / I_sig mask = I_over_S > 0.1 I_over_S = I_over_S.select(mask) corr = corr.select(mask) fig = pyplot.figure() pyplot.title("Reflection correlations vs Log I/Sigma") cax = pyplot.hexbin(flex.log(I_over_S), corr, gridsize=100) cbar = pyplot.colorbar(cax) cax.axes.set_xlabel("Log I/Sigma") cax.axes.set_ylabel("Correlation with reference profile") cbar.ax.set_ylabel("# reflections") fig.savefig( os.path.join(self.directory, "%s_corr_vs_ios.png" % filename)) pyplot.close()
def test_as_miller_array(): table = flex.reflection_table() table["intensity.1.value"] = flex.double([1.0, 2.0, 3.0]) table["intensity.1.variance"] = flex.double([0.25, 1.0, 4.0]) table["miller_index"] = flex.miller_index([(1, 0, 0), (2, 0, 0), (3, 0, 0)]) crystal = Crystal( real_space_a=(10, 0, 0), real_space_b=(0, 11, 0), real_space_c=(0, 0, 12), space_group=sgtbx.space_group_info("P 222").group(), ) experiment = Experiment(crystal=crystal) iobs = table.as_miller_array(experiment, intensity="1") assert list(iobs.data()) == list(table["intensity.1.value"]) assert list(iobs.sigmas()) == list(flex.sqrt(table["intensity.1.variance"])) with pytest.raises(KeyError): _ = table.as_miller_array(experiment, intensity="2") table["intensity.2.value"] = flex.double([1.0, 2.0, 3.0]) with pytest.raises(KeyError): _ = table.as_miller_array(experiment, intensity="2")
def match(a, b, max_separation=2, key="xyzobs.px.value", scale=(1, 1, 1)): """Match reflections from list a and list b, returning tuple of flex.size_t indices, optionally specifying the maximum distance and key to search on (which is assumed to be a 3-vector column). Can also apply relative scales to the vectors before matching in case e.g. very wide or very fine slicing.""" xyz_a = a[key] xyz_b = b[key] if scale != (1, 1, 1): x, y, z = xyz_a.parts() x *= scale[0] y *= scale[1] z *= scale[2] xyz_a = flex.vec3_double(x, y, z) x, y, z = xyz_b.parts() x *= scale[0] y *= scale[1] z *= scale[2] xyz_b = flex.vec3_double(x, y, z) a = xyz_a.as_double().as_1d() b = xyz_b.as_double().as_1d() ann = AnnAdaptorSelfInclude(a, 3) ann.query(b) mm = flex.size_t(range(xyz_b.size())) nn, distance = ann.nn, flex.sqrt(ann.distances) sel = distance <= max_separation mm = mm.select(sel) nn = nn.select(sel) distance = distance.select(sel) return nn, mm, distance
def setup_work_arrays(self, reflections): '''Select multiply-measured HKLs. Calculate and cache reflection deltas, deltas squared, and HKL means for every reflection''' self.deltas = flex.double() self.work_table = flex.reflection_table() delta_sq = flex.double() mean = flex.double() # mean = <I'_hj> biased_mean = flex.double() # biased_mean = <I_h>, so dont leave out any reflection var = flex.double() all_biased_mean = flex.double() for refls in reflection_table_utils.get_next_hkl_reflection_table(reflections): number_of_measurements = refls.size() if number_of_measurements == 0: # if the returned "refls" list is empty, it's the end of the input "reflections" list break refls_biased_mean = flex.double(len(refls), flex.mean(refls['intensity.sum.value'])) all_biased_mean.extend(refls_biased_mean) if number_of_measurements > self.params.merging.minimum_multiplicity: nn_factor_sqrt = math.sqrt((number_of_measurements - 1) / number_of_measurements) i_sum = flex.double(number_of_measurements, flex.sum(refls['intensity.sum.value'])) i_sum_minus_val = i_sum - refls['intensity.sum.value'] mean_without_val = i_sum_minus_val/(number_of_measurements-1) delta = nn_factor_sqrt * (refls['intensity.sum.value'] - mean_without_val) self.deltas.extend(delta/flex.sqrt(refls['intensity.sum.variance'])) # Please be careful about where to put the var delta_sq.extend(delta**2) mean.extend(mean_without_val) biased_mean.extend(refls_biased_mean) var.extend(refls['intensity.sum.variance']) self.work_table["delta_sq"] = delta_sq self.work_table["mean"] = mean self.work_table["biased_mean"] = biased_mean self.work_table["var"] = var reflections['biased_mean'] = all_biased_mean self.logger.log("Number of work reflections selected: %d"%self.deltas.size()) return reflections
def __call__(self, experiments, reflections): results = flex.reflection_table() table_header = ["","","","I","IsigI","N >","RMSD","Cutoff"] table_header2 = ["Bin","Resolution Range","Completeness","","","cutoff","(um)",""] for exp_id in xrange(len(experiments)): print("*"*80) print("Significance filtering experiment", exp_id) table_data = [] table_data.append(table_header) table_data.append(table_header2) experiment = experiments[exp_id] # Find the bins for this experiment crystal = experiment.crystal refls = reflections.select(reflections['id'] == exp_id) sym = symmetry(unit_cell = crystal.get_unit_cell(), space_group = crystal.get_space_group()) d = crystal.get_unit_cell().d(refls['miller_index']) mset = sym.miller_set(indices = refls['miller_index'], anomalous_flag=False) binner = mset.setup_binner(n_bins=self.params.n_bins) acceptable_resolution_bins = [] # Iterate through the bins, examining I/sigI at each bin for i in binner.range_used(): d_max, d_min = binner.bin_d_range(i) sel = (d <= d_max) & (d > d_min) sel &= refls['intensity.sum.value'] > 0 bin_refls = refls.select(sel) n_refls = len(bin_refls) avg_i = flex.mean(bin_refls['intensity.sum.value']) if n_refls > 0 else 0 avg_i_sigi = flex.mean(bin_refls['intensity.sum.value'] / flex.sqrt(bin_refls['intensity.sum.variance'])) if n_refls > 0 else 0 acceptable_resolution_bins.append(avg_i_sigi >= self.params.isigi_cutoff) bright_refls = bin_refls.select((bin_refls['intensity.sum.value']/flex.sqrt(bin_refls['intensity.sum.variance'])) >= self.params.isigi_cutoff) n_bright = len(bright_refls) rmsd_obs = 1000*math.sqrt((bright_refls['xyzcal.mm']-bright_refls['xyzobs.mm.value']).sum_sq()/n_bright) if n_bright > 0 else 0 table_row = [] table_row.append("%3d"%i) table_row.append("%-13s"%binner.bin_legend(i_bin=i,show_bin_number=False,show_bin_range=False, show_d_range=True, show_counts=False)) table_row.append("%13s"%binner.bin_legend(i_bin=i,show_bin_number=False,show_bin_range=False, show_d_range=False, show_counts=True)) table_row.append("%.1f"%(avg_i)) table_row.append("%.1f"%(avg_i_sigi)) table_row.append("%3d"%n_bright) table_row.append("%.1f"%(rmsd_obs)) table_data.append(table_row) # Throw out bins that go back above the cutoff after the first non-passing bin is found acceptable_resolution_bins = [acceptable_resolution_bins[i] for i in xrange(len(acceptable_resolution_bins)) if False not in acceptable_resolution_bins[:i+1]] for b, row in zip(acceptable_resolution_bins, table_data[2:]): if b: row.append("X") print(table_utils.format(table_data,has_header=2,justify='center',delim=" ")) # Save the results if any(acceptable_resolution_bins): best_index = acceptable_resolution_bins.count(True)-1 best_row = table_data[best_index+2] d_min = binner.bin_d_range(binner.range_used()[best_index])[1] print("best row:", " ".join(best_row)) if self.params.enable: results.extend(refls.select(d >= d_min)) else: print("Data didn't pass cutoff") if self.params.enable: return results else: return reflections
def apply_significance_filter(self, experiments, reflections): self.logger.log_step_time("SIGNIFICANCE_FILTER") # Apply an I/sigma filter ... accept resolution bins only if they # have significant signal; tends to screen out higher resolution observations # if the integration model doesn't quite fit unit_cell = self.params.scaling.unit_cell if unit_cell is None: try: unit_cell = self.params.statistics.average_unit_cell except AttributeError: pass target_symm = symmetry( unit_cell=unit_cell, space_group_info=self.params.scaling.space_group) new_experiments = ExperimentList() new_reflections = flex.reflection_table() for expt_id, experiment in enumerate(experiments): exp_reflections = reflections.select( reflections['exp_id'] == experiment.identifier) if not len(exp_reflections): continue N_obs_pre_filter = exp_reflections.size() N_bins_small_set = N_obs_pre_filter // self.params.select.significance_filter.min_ct N_bins_large_set = N_obs_pre_filter // self.params.select.significance_filter.max_ct # Ensure there is at least one bin. N_bins = max([ min([ self.params.select.significance_filter.n_bins, N_bins_small_set ]), N_bins_large_set, 1 ]) #print ("\nN_obs_pre_filter %d"%N_obs_pre_filter) #print >> out, "Total obs %d Choose n bins = %d"%(N_obs_pre_filter,N_bins) #if indices_to_edge is not None: # print >> out, "Total preds %d to edge of detector"%indices_to_edge.size() # Build a miller array for the experiment reflections exp_miller_indices = miller.set(target_symm, exp_reflections['miller_index'], True) exp_observations = miller.array( exp_miller_indices, exp_reflections['intensity.sum.value'], flex.sqrt(exp_reflections['intensity.sum.variance'])) assert exp_observations.size() == exp_reflections.size() out = StringIO() bin_results = show_observations(exp_observations, out=out, n_bins=N_bins) if self.params.output.log_level == 0: self.logger.log(out.getvalue()) acceptable_resolution_bins = [ bin.mean_I_sigI > self.params.select.significance_filter.sigma for bin in bin_results ] acceptable_nested_bin_sequences = [ i for i in range(len(acceptable_resolution_bins)) if False not in acceptable_resolution_bins[:i + 1] ] if len(acceptable_nested_bin_sequences) == 0: continue else: N_acceptable_bins = max(acceptable_nested_bin_sequences) + 1 imposed_res_filter = float(bin_results[N_acceptable_bins - 1].d_range.split()[2]) self.logger.log( "Experiment id %d, image index %d, resolution cutoff %f\n" % (expt_id, experiment.imageset.indices()[0], imposed_res_filter)) imposed_res_sel = exp_observations.resolution_filter_selection( d_min=imposed_res_filter) assert imposed_res_sel.size() == exp_reflections.size() new_exp_reflections = exp_reflections.select(imposed_res_sel) if new_exp_reflections.size() > 0: new_experiments.append(experiment) new_reflections.extend(new_exp_reflections) #self.logger.log("N acceptable bins %d"%N_acceptable_bins) #self.logger.log("Old n_obs: %d, new n_obs: %d"%(N_obs_pre_filter, exp_observations.size())) #if indices_to_edge is not None: # print >> out, "Total preds %d to edge of detector"%indices_to_edge.size() removed_reflections = len(reflections) - len(new_reflections) removed_experiments = len(experiments) - len(new_experiments) self.logger.log( "Reflections rejected because of significance filter: %d" % removed_reflections) self.logger.log( "Experiments rejected because of significance filter: %d" % removed_experiments) # MPI-reduce total counts comm = self.mpi_helper.comm MPI = self.mpi_helper.MPI total_removed_reflections = comm.reduce(removed_reflections, MPI.SUM, 0) total_removed_experiments = comm.reduce(removed_experiments, MPI.SUM, 0) # rank 0: log total counts if self.mpi_helper.rank == 0: self.logger.main_log( "Total reflections rejected because of significance filter: %d" % total_removed_reflections) self.logger.main_log( "Total experiments rejected because of significance filter: %d" % total_removed_experiments) self.logger.log_step_time("SIGNIFICANCE_FILTER", True) return new_experiments, new_reflections
def test_for_reflections(self, refl): from dials.algorithms.integration.sum import IntegrationAlgorithm from dials.array_family import flex from dials.algorithms.statistics import \ kolmogorov_smirnov_test_standard_normal # Get the calculated background and simulated background B_sim = refl['background.sim.a'].as_double() I_sim = refl['intensity.sim'].as_double() I_exp = refl['intensity.exp'] # Set the background as simulated shoebox = refl['shoebox'] for i in range(len(shoebox)): bg = shoebox[i].background ms = shoebox[i].mask for j in range(len(bg)): bg[j] = B_sim[i] # Integrate integration = IntegrationAlgorithm() integration(refl) I_cal = refl['intensity.sum.value'] I_var = refl['intensity.sum.variance'] # Only select variances greater than zero mask = I_var > 0 I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) # Calculate the z score perc = self.mv3n_tolerance_interval(3 * 3) Z = (I_cal - I_exp) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f" % (Z_mean, Z_var) # Do the kolmogorov smirnov test D, p = kolmogorov_smirnov_test_standard_normal(Z) print "KS: D: %f, p-value: %f" % (D, p) # FIXME Z score should be a standard normal distribution. When background is # the main component, we do indeed see that the z score is in a standard # normal distribution. When the intensity dominates, the variance of the Z # scores decreases indicating that for increasing intensity of the signal, # the variance is over estimated. assert (abs(Z_mean) <= 3 * Z_var) #from matplotlib import pylab #pylab.hist(Z, 20) #pylab.show() #Z_I = sorted(Z) ##n = int(0.05 * len(Z_I)) ##Z_I = Z_I[n:-n] ##mv = flex.mean_and_variance(flex.double(Z_I)) ##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation()) #edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))] #cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I] print 'OK'
def export_sadabs(integrated_data, experiment_list, params): """Export data from integrated_data corresponding to experiment_list to a file for input to SADABS. FIXME probably need to make a .p4p file as well...""" from dials.array_family import flex # for the moment assume (and assert) that we will convert data from exactly # one lattice... assert len(experiment_list) == 1 # select reflections that are assigned to an experiment (i.e. non-negative id) integrated_data = integrated_data.select(integrated_data["id"] >= 0) assert max(integrated_data["id"]) == 0 # export for sadabs should only be for non-scaled reflections assert any(i in integrated_data for i in ["intensity.sum.value", "intensity.prf.value"]) integrated_data = filter_reflection_table( integrated_data, intensity_choice=params.intensity, partiality_threshold=params.mtz.partiality_threshold, combine_partials=params.mtz.combine_partials, min_isigi=params.mtz.min_isigi, filter_ice_rings=params.mtz.filter_ice_rings, d_min=params.mtz.d_min, ) experiment = experiment_list[0] assert experiment.scan is not None # sort data before output nref = len(integrated_data["miller_index"]) indices = flex.size_t_range(nref) perm = sorted(indices, key=lambda k: integrated_data["miller_index"][k]) integrated_data = integrated_data.select(flex.size_t(perm)) assert experiment.goniometer is not None # Warn of unhelpful SADABS behaviour for certain multi-sequence data sets hkl_file_root, _ = os.path.splitext(params.sadabs.hklout) if not params.sadabs.run or re.search("_0+$", hkl_file_root): logger.warning( "It seems SADABS rejects multi-sequence data when the first " "filename ends " "'_0', '_00', etc., with a cryptic error message:\n" "\t'Inconsistent 2theta values in same scan'.\n" "You may need to begin the numbering of your SADABS HKL files from 1, " "rather than 0, and ensure the SADABS run/batch number is greater than 0." ) axis = matrix.col(experiment.goniometer.get_rotation_axis_datum()) beam = matrix.col(experiment.beam.get_sample_to_source_direction()) s0 = matrix.col(experiment.beam.get_s0()) F = matrix.sqr(experiment.goniometer.get_fixed_rotation()) S = matrix.sqr(experiment.goniometer.get_setting_rotation()) unit_cell = experiment.crystal.get_unit_cell() if params.debug: m_format = "%6.3f%6.3f%6.3f\n%6.3f%6.3f%6.3f\n%6.3f%6.3f%6.3f" c_format = "%.2f %.2f %.2f %.2f %.2f %.2f" logger.info( "Unit cell parameters from experiment: %s", c_format % unit_cell.parameters(), ) logger.info( "Symmetry: %s", experiment.crystal.get_space_group().type().lookup_symbol()) logger.info("Goniometer fixed matrix:\n%s", m_format % F.elems) logger.info("Goniometer setting matrix:\n%s", m_format % S.elems) logger.info("Goniometer scan axis:\n%6.3f%6.3f%6.3f", axis.elems) # detector scaling info assert len(experiment.detector) == 1 panel = experiment.detector[0] dims = panel.get_image_size() pixel = panel.get_pixel_size() fast_axis = matrix.col(panel.get_fast_axis()) slow_axis = matrix.col(panel.get_slow_axis()) normal = fast_axis.cross(slow_axis) detector2t = s0.angle(normal, deg=True) if params.debug: logger.info("Detector fast, slow axes:") logger.info("%6.3f%6.3f%6.3f", fast_axis.elems) logger.info("%6.3f%6.3f%6.3f", slow_axis.elems) logger.info("Detector two theta (degrees): %.2f", detector2t) scl_x = 512.0 / (dims[0] * pixel[0]) scl_y = 512.0 / (dims[1] * pixel[1]) image_range = experiment.scan.get_image_range() from cctbx.array_family import flex as cflex # implicit import # noqa: F401 from cctbx.miller import map_to_asu_isym # implicit import # noqa: F401 # gather the required information for the reflection file nref = len(integrated_data["miller_index"]) miller_index = integrated_data["miller_index"] if "intensity.sum.value" in integrated_data: I = integrated_data["intensity.sum.value"] V = integrated_data["intensity.sum.variance"] assert V.all_gt(0) sigI = flex.sqrt(V) else: I = integrated_data["intensity.prf.value"] V = integrated_data["intensity.prf.variance"] assert V.all_gt(0) sigI = flex.sqrt(V) # figure out scaling to make sure data fit into format 2F8.2 i.e. Imax < 1e5 Imax = flex.max(I) if params.debug: logger.info("Maximum intensity in file: %8.2f", Imax) if Imax > 99999.0: scale = 99999.0 / Imax I = I * scale sigI = sigI * scale phi_start, phi_range = experiment.scan.get_image_oscillation( image_range[0]) if params.sadabs.predict: logger.info("Using scan static predicted spot locations") from dials.algorithms.spot_prediction import ScanStaticReflectionPredictor predictor = ScanStaticReflectionPredictor(experiment) UB = experiment.crystal.get_A() predictor.for_reflection_table(integrated_data, UB) if not experiment.crystal.num_scan_points: logger.info("No scan varying model: use static") static = True else: static = False with open(params.sadabs.hklout, "w") as fout: for j in range(nref): h, k, l = miller_index[j] if params.sadabs.predict: x_mm, y_mm, z_rad = integrated_data["xyzcal.mm"][j] else: x_mm, y_mm, z_rad = integrated_data["xyzobs.mm.value"][j] z0 = integrated_data["xyzcal.px"][j][2] istol = int(round(10000 * unit_cell.stol((h, k, l)))) if params.sadabs.predict or static: # work from a scan static model & assume perfect goniometer # FIXME maybe should work back in the option to predict spot positions UB = matrix.sqr(experiment.crystal.get_A()) phi = phi_start + z0 * phi_range R = axis.axis_and_angle_as_r3_rotation_matrix(phi, deg=True) RUB = S * R * F * UB else: # properly compute RUB for every reflection UB = matrix.sqr( experiment.crystal.get_A_at_scan_point(int(round(z0)))) phi = phi_start + z0 * phi_range R = axis.axis_and_angle_as_r3_rotation_matrix(phi, deg=True) RUB = S * R * F * UB x = RUB * (h, k, l) s = (s0 + x).normalize() # can also compute s based on centre of mass of spot # s = (origin + x_mm * fast_axis + y_mm * slow_axis).normalize() astar = (RUB * (1, 0, 0)).normalize() bstar = (RUB * (0, 1, 0)).normalize() cstar = (RUB * (0, 0, 1)).normalize() ix = beam.dot(astar) iy = beam.dot(bstar) iz = beam.dot(cstar) dx = s.dot(astar) dy = s.dot(bstar) dz = s.dot(cstar) x = x_mm * scl_x y = y_mm * scl_y z = (z_rad * 180 / math.pi - phi_start) / phi_range fout.write("%4d%4d%4d%8.2f%8.2f%4d%8.5f%8.5f%8.5f%8.5f%8.5f%8.5f" % (h, k, l, I[j], sigI[j], params.sadabs.run, ix, dx, iy, dy, iz, dz)) fout.write("%7.2f%7.2f%8.2f%7.2f%5d\n" % (x, y, z, detector2t, istol)) fout.close() logger.info("Output %d reflections to %s", nref, params.sadabs.hklout)
def _multiplicity_mean_error_stddev(self, calculate_variances=False, keep_singles=False): """ Calculate aggregate properties of grouped symmetry-equivalent reflections. Populate the reflection table of observations with the following properties: * ``multiplicity`` — Multiplicity of observations of a given reflection in the asymmetric unit; :type: `dials.array_family_flex_ext.int` array * ``intensity.mean.value`` — Mean of symmetry-equivalent reflections, weighted by measurement error; :type: `dials.array_family_flex_ext.double` array * ``intensity.mean.std_error`` — Standard error on the weighted mean; :type: `dials.array_family_flex_ext.double` array * (optional) ``intensity.mean.variance`` — variance of symmetry-equivalent reflections, weighted by measurement error; :type: `dials.array_family_flex_ext.double` array :param calculate_variances: Elect whether to calculate the weighted variances. Defaults to False, to spare an expensive computation. :type calculate_variances: bool :param keep_singles: Choose whether to keep single-multiplicity reflections. :type keep_singles: bool """ for key, rtable in self.rtables.items(): # Sort the reflection table for speedier iteration. rtable.sort("miller_index.asu") # Record the positions of any multiplicity-1 reflections. if not keep_singles: singles = flex.size_t() # Record the multiplicities. multiplicity = flex.int() # For weighted averaging. weights = 1 / rtable["intensity.sum.variance"] sum_weights = flex.double() if calculate_variances: sum_square_weights = flex.double() # Calculate the weighted mean intensities. i_means = flex.double() # Calculate the standard deviations from unbiased weighted variances. variances = flex.double() # Iterate over the reflections, grouping by equivalent Miller index, # to calculate multiplicities, weighted mean intensities, etc.. # Some time can be saved by only calculating variances if necessary. # Initial values: prev_index = None count = 1 # The following will be set during loop iteration i_sum, sum_weight, sum_square_weight = None, None, None # One big loop through the entire reflection table: for j in range(rtable.size()): index = rtable["miller_index.asu"][j] weight = weights[j] # Aggregate within a symmetry-equivalent group of reflections: if index == prev_index: count += 1 i_sum += weight * rtable["intensity.sum.value"][j] sum_weight += weight if calculate_variances: sum_square_weight += weight * weight # Record the aggregated values for the group: elif prev_index: if count == 1 and not keep_singles: singles.append(j - 1) multiplicity.extend(flex.int(count, count)) i_means.extend(flex.double(count, i_sum / sum_weight)) sum_weights.extend(flex.double(count, sum_weight)) if calculate_variances: sum_square_weights.extend( flex.double(count, sum_square_weight)) # And reinitialise: prev_index = index count = 1 i_sum = weight * rtable["intensity.sum.value"][j] sum_weight = weight if calculate_variances: sum_square_weight = weight * weight # Handle the first row: else: prev_index = rtable["miller_index.asu"][j] i_sum = weight * rtable["intensity.sum.value"][j] sum_weight = weight if calculate_variances: sum_square_weight = weight * weight # Record the aggregated values for the last group: if count == 1 and not keep_singles: singles.append(rtable.size() - 1) multiplicity.extend(flex.int(count, count)) i_means.extend(flex.double(count, i_sum / sum_weight)) sum_weights.extend(flex.double(count, sum_weight)) if calculate_variances: sum_square_weights.extend(flex.double(count, sum_square_weight)) # Discard singletons: if not keep_singles: singles_del = flex.bool(rtable.size(), True) singles_del.set_selected(singles, False) multiplicity, weights, sum_weights, i_means = [ a.select(singles_del) for a in (multiplicity, weights, sum_weights, i_means) ] rtable.del_selected(singles) if calculate_variances: sum_square_weights = sum_square_weights.select(singles_del) # Record the multiplicities in the reflection table. rtable["multiplicity"] = multiplicity # Record the weighted mean intensities in the reflection table. rtable["intensity.mean.value"] = i_means # Record the standard errors on the means in the reflection table. rtable["intensity.mean.std_error"] = flex.sqrt(1 / sum_weights) if calculate_variances: # Initialise values: prev_index = None weighted_sum_square_residual = None for j in range(rtable.size()): index = rtable["miller_index.asu"][j] weight = weights[j] residual = rtable["intensity.sum.value"][j] - i_means[j] # Aggregate within a symmetry-equivalent group of reflections: if index == prev_index: count += 1 weighted_sum_square_residual += weight * residual * residual # Record the aggregated value for the group: elif prev_index: # The weighted variance is undefined for multiplicity=1, # use the measured variance instead in this case. if count == 1: variances.append( rtable["intensity.sum.variance"][j - 1]) else: sum_weight = sum_weights[j - 1] var_weight = 1 / ( sum_weight - sum_square_weights[j - 1] / sum_weight) variances.extend( flex.double( count, weighted_sum_square_residual * var_weight)) # Reinitialise: prev_index = index count = 1 weighted_sum_square_residual = weight * residual * residual # Handle the first row: else: prev_index = rtable["miller_index.asu"][j] count = 1 weighted_sum_square_residual = weight * residual * residual # Record the aggregated values for the last group: # The weighted variance is undefined for multiplicity=1, # use the measured variance instead in this case. if count == 1: variances.append(rtable["intensity.sum.variance"][-1]) else: sum_weight = sum_weights[-1] var_weight = 1 / (sum_weight - sum_square_weights[-1] / sum_weight) variances.extend( flex.double(count, weighted_sum_square_residual * var_weight)) # Record the variances in the reflection table. rtable["intensity.mean.variance"] = variances self.rtables[key] = rtable
def write_columns(self, integrated_data): """Write the column definitions AND data to the current dataset.""" # now create the actual data structures - first keep a track of the columns # H K L M/ISYM BATCH I SIGI IPR SIGIPR FRACTIONCALC XDET YDET ROT WIDTH # LP MPART FLAG BGPKRATIOS # gather the required information for the reflection file nref = len(integrated_data["miller_index"]) assert nref xdet, ydet, _ = [ flex.double(x) for x in integrated_data["xyzobs.px.value"].parts() ] # now add column information... # FIXME add DIALS_FLAG which can include e.g. was partial etc. type_table = { "H": "H", "K": "H", "L": "H", "I": "J", "SIGI": "Q", "IPR": "J", "SIGIPR": "Q", "BG": "R", "SIGBG": "R", "XDET": "R", "YDET": "R", "BATCH": "B", "BGPKRATIOS": "R", "WIDTH": "R", "MPART": "I", "M_ISYM": "Y", "FLAG": "I", "LP": "R", "FRACTIONCALC": "R", "ROT": "R", "QE": "R", } # derive index columns from original indices with # # from m.replace_original_index_miller_indices # # so all that is needed now is to make space for the reflections - fill with # zeros... self.mtz_file.adjust_column_array_sizes(nref) self.mtz_file.set_n_reflections(nref) dataset = self.current_dataset # assign H, K, L, M_ISYM space for column in "H", "K", "L", "M_ISYM": dataset.add_column(column, type_table[column]).set_values( flex.double(nref, 0.0).as_float()) self.mtz_file.replace_original_index_miller_indices( integrated_data["miller_index"]) dataset.add_column("BATCH", type_table["BATCH"]).set_values( integrated_data["batch"].as_double().as_float()) # if intensity values used in scaling exist, then just export these as I, SIGI if "intensity.scale.value" in integrated_data: I_scaling = integrated_data["intensity.scale.value"] V_scaling = integrated_data["intensity.scale.variance"] # Trap negative variances assert V_scaling.all_gt(0) dataset.add_column("I", type_table["I"]).set_values( I_scaling.as_float()) dataset.add_column("SIGI", type_table["SIGI"]).set_values( flex.sqrt(V_scaling).as_float()) dataset.add_column("SCALEUSED", "R").set_values( integrated_data["inverse_scale_factor"].as_float()) dataset.add_column("SIGSCALEUSED", "R").set_values( flex.sqrt(integrated_data["inverse_scale_factor_variance"]). as_float()) else: if "intensity.prf.value" in integrated_data: if "intensity.sum.value" in integrated_data: col_names = ("IPR", "SIGIPR") else: col_names = ("I", "SIGI") I_profile = integrated_data["intensity.prf.value"] V_profile = integrated_data["intensity.prf.variance"] # Trap negative variances assert V_profile.all_gt(0) dataset.add_column(col_names[0], type_table["I"]).set_values( I_profile.as_float()) dataset.add_column(col_names[1], type_table["SIGI"]).set_values( flex.sqrt(V_profile).as_float()) if "intensity.sum.value" in integrated_data: I_sum = integrated_data["intensity.sum.value"] V_sum = integrated_data["intensity.sum.variance"] # Trap negative variances assert V_sum.all_gt(0) dataset.add_column("I", type_table["I"]).set_values( I_sum.as_float()) dataset.add_column("SIGI", type_table["SIGI"]).set_values( flex.sqrt(V_sum).as_float()) if ("background.sum.value" in integrated_data and "background.sum.variance" in integrated_data): bg = integrated_data["background.sum.value"] varbg = integrated_data["background.sum.variance"] assert (varbg >= 0).count(False) == 0 sigbg = flex.sqrt(varbg) dataset.add_column("BG", type_table["BG"]).set_values(bg.as_float()) dataset.add_column("SIGBG", type_table["SIGBG"]).set_values( sigbg.as_float()) dataset.add_column("FRACTIONCALC", type_table["FRACTIONCALC"]).set_values( integrated_data["fractioncalc"].as_float()) dataset.add_column("XDET", type_table["XDET"]).set_values(xdet.as_float()) dataset.add_column("YDET", type_table["YDET"]).set_values(ydet.as_float()) dataset.add_column("ROT", type_table["ROT"]).set_values( integrated_data["ROT"].as_float()) if "lp" in integrated_data: dataset.add_column("LP", type_table["LP"]).set_values( integrated_data["lp"].as_float()) if "qe" in integrated_data: dataset.add_column("QE", type_table["QE"]).set_values( integrated_data["qe"].as_float()) elif "dqe" in integrated_data: dataset.add_column("QE", type_table["QE"]).set_values( integrated_data["dqe"].as_float()) else: dataset.add_column("QE", type_table["QE"]).set_values( flex.double(nref, 1.0).as_float())
def integrate(self, experiments, indexed): from time import time st = time() logger.info('*' * 80) logger.info('Integrating Reflections') logger.info('*' * 80) indexed,_ = self.process_reference(indexed) # Get the integrator from the input parameters logger.info('Configuring integrator from input parameters') from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator experiments = ProfileModelFactory.create(self.params, experiments, indexed) logger.info("") logger.info("=" * 80) logger.info("") logger.info("Predicting reflections") logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=self.params.prediction.d_min, dmax=self.params.prediction.d_max, margin=self.params.prediction.margin, force_static=self.params.prediction.force_static) predicted.match_with_reference(indexed) logger.info("") integrator = IntegratorFactory.create(self.params, experiments, predicted) # Integrate the reflections integrated = integrator.integrate() # Select only those reflections which were integrated if 'intensity.prf.variance' in integrated: selection = integrated.get_flags( integrated.flags.integrated, all=True) else: selection = integrated.get_flags( integrated.flags.integrated_sum) integrated = integrated.select(selection) len_all = len(integrated) integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels)) print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all) # verify sigmas are sensible if 'intensity.prf.value' in integrated: if (integrated['intensity.prf.variance'] <= 0).count(True) > 0: raise Sorry("Found negative variances") if 'intensity.sum.value' in integrated: if (integrated['intensity.sum.variance'] <= 0).count(True) > 0: raise Sorry("Found negative variances") # apply detector gain to summation variances integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain if 'background.sum.value' in integrated: if (integrated['background.sum.variance'] < 0).count(True) > 0: raise Sorry("Found negative variances") if (integrated['background.sum.variance'] == 0).count(True) > 0: print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True)) integrated = integrated.select(integrated['background.sum.variance'] > 0) # apply detector gain to background summation variances integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain if self.params.output.integrated_filename: # Save the reflections self.save_reflections(integrated, self.params.output.integrated_filename) self.write_integration_pickles(integrated, experiments) from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed) log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed) for i in xrange(6): bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i) if len(bright_integrated) > 0: rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated) else: rmsd_integrated = 0 log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated) crystal_model = experiments.crystals()[0] if hasattr(crystal_model, '._ML_domain_size_ang'): log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg) logger.info(log_str) logger.info('') logger.info('Time Taken = %f seconds' % (time() - st)) return integrated
def export_sadabs(integrated_data, experiment_list, hklout, run=0, summation=False, include_partials=False, keep_partials=False, debug=False, predict=True): '''Export data from integrated_data corresponding to experiment_list to a file for input to SADABS. FIXME probably need to make a .p4p file as well...''' from dials.array_family import flex from scitbx import matrix import math # for the moment assume (and assert) that we will convert data from exactly # one lattice... assert(len(experiment_list) == 1) # select reflections that are assigned to an experiment (i.e. non-negative id) integrated_data = integrated_data.select(integrated_data['id'] >= 0) assert max(integrated_data['id']) == 0 if not summation: assert('intensity.prf.value' in integrated_data) # strip out negative variance reflections: these should not really be there # FIXME Doing select on summation results. Should do on profile result if # present? Yes if 'intensity.prf.variance' in integrated_data: selection = integrated_data.get_flags( integrated_data.flags.integrated, all=True) else: selection = integrated_data.get_flags( integrated_data.flags.integrated_sum) integrated_data = integrated_data.select(selection) selection = integrated_data['intensity.sum.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d reflections with negative variance' % \ selection.count(True)) if 'intensity.prf.variance' in integrated_data: selection = integrated_data['intensity.prf.variance'] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info('Removing %d profile reflections with negative variance' % \ selection.count(True)) if include_partials: integrated_data = sum_partial_reflections(integrated_data) integrated_data = scale_partial_reflections(integrated_data) if 'partiality' in integrated_data: selection = integrated_data['partiality'] < 0.99 if selection.count(True) > 0 and not keep_partials: integrated_data.del_selected(selection) logger.info('Removing %d incomplete reflections' % \ selection.count(True)) experiment = experiment_list[0] assert(not experiment.scan is None) # sort data before output nref = len(integrated_data['miller_index']) indices = flex.size_t_range(nref) perm = sorted(indices, key=lambda k: integrated_data['miller_index'][k]) integrated_data = integrated_data.select(flex.size_t(perm)) assert (not experiment.goniometer is None) axis = matrix.col(experiment.goniometer.get_rotation_axis_datum()) beam = matrix.col(experiment.beam.get_direction()) s0 = matrix.col(experiment.beam.get_s0()) F = matrix.sqr(experiment.goniometer.get_fixed_rotation()) S = matrix.sqr(experiment.goniometer.get_setting_rotation()) unit_cell = experiment.crystal.get_unit_cell() if debug: m_format = '%6.3f%6.3f%6.3f\n%6.3f%6.3f%6.3f\n%6.3f%6.3f%6.3f' c_format = '%.2f %.2f %.2f %.2f %.2f %.2f' logger.info('Unit cell parameters from experiment: %s' % (c_format % unit_cell.parameters())) logger.info('Symmetry: %s' % experiment.crystal.get_space_group().type( ).lookup_symbol()) logger.info('Goniometer fixed matrix:\n%s' % (m_format % F.elems)) logger.info('Goniometer setting matrix:\n%s' % (m_format % S.elems)) logger.info('Goniometer scan axis:\n%6.3f%6.3f%6.3f' % (axis.elems)) # detector scaling info assert(len(experiment.detector) == 1) panel = experiment.detector[0] dims = panel.get_image_size() pixel = panel.get_pixel_size() fast_axis = matrix.col(panel.get_fast_axis()) slow_axis = matrix.col(panel.get_slow_axis()) normal = fast_axis.cross(slow_axis) detector2t = s0.angle(normal, deg=True) origin = matrix.col(panel.get_origin()) if debug: logger.info('Detector fast, slow axes:') logger.info('%6.3f%6.3f%6.3f' % (fast_axis.elems)) logger.info('%6.3f%6.3f%6.3f' % (slow_axis.elems)) logger.info('Detector two theta (degrees): %.2f' % detector2t) scl_x = 512.0 / (dims[0] * pixel[0]) scl_y = 512.0 / (dims[1] * pixel[1]) image_range = experiment.scan.get_image_range() from cctbx.array_family import flex as cflex # implicit import from cctbx.miller import map_to_asu_isym # implicit import # gather the required information for the reflection file nref = len(integrated_data['miller_index']) zdet = flex.double(integrated_data['xyzcal.px'].parts()[2]) miller_index = integrated_data['miller_index'] I = None sigI = None # export including scale factors if 'lp' in integrated_data: lp = integrated_data['lp'] else: lp = flex.double(nref, 1.0) if 'dqe' in integrated_data: dqe = integrated_data['dqe'] else: dqe = flex.double(nref, 1.0) scl = lp / dqe if summation: I = integrated_data['intensity.sum.value'] * scl V = integrated_data['intensity.sum.variance'] * scl * scl assert V.all_gt(0) sigI = flex.sqrt(V) else: I = integrated_data['intensity.prf.value'] * scl V = integrated_data['intensity.prf.variance'] * scl * scl assert V.all_gt(0) sigI = flex.sqrt(V) # figure out scaling to make sure data fit into format 2F8.2 i.e. Imax < 1e5 Imax = flex.max(I) if debug: logger.info('Maximum intensity in file: %8.2f' % Imax) if Imax > 99999.0: scale = 99999.0 / Imax I = I * scale sigI = sigI * scale phi_start, phi_range = experiment.scan.get_image_oscillation(image_range[0]) if predict: logger.info('Using scan static predicted spot locations') from dials.algorithms.spot_prediction import ScanStaticReflectionPredictor predictor = ScanStaticReflectionPredictor(experiment) UB = experiment.crystal.get_A() predictor.for_reflection_table(integrated_data, UB) if not experiment.crystal.num_scan_points: logger.info('No scan varying model: use static') static = True else: static = False fout = open(hklout, 'w') for j in range(nref): h, k, l = miller_index[j] if predict: x_mm, y_mm, z_rad = integrated_data['xyzcal.mm'][j] else: x_mm, y_mm, z_rad = integrated_data['xyzobs.mm.value'][j] z0 = integrated_data['xyzcal.px'][j][2] istol = int(round(10000 * unit_cell.stol((h, k, l)))) if predict or static: # work from a scan static model & assume perfect goniometer # FIXME maybe should work back in the option to predict spot positions UB = experiment.crystal.get_A() phi = phi_start + z0 * phi_range R = axis.axis_and_angle_as_r3_rotation_matrix(phi, deg=True) RUB = S * R * F * UB else: # properly compute RUB for every reflection UB = experiment.crystal.get_A_at_scan_point(int(round(z0))) phi = phi_start + z0 * phi_range R = axis.axis_and_angle_as_r3_rotation_matrix(phi, deg=True) RUB = S * R * F * UB x = RUB * (h, k, l) s = (s0 + x).normalize() # can also compute s based on centre of mass of spot # s = (origin + x_mm * fast_axis + y_mm * slow_axis).normalize() astar = (RUB * (1, 0, 0)).normalize() bstar = (RUB * (0, 1, 0)).normalize() cstar = (RUB * (0, 0, 1)).normalize() ix = beam.dot(astar) iy = beam.dot(bstar) iz = beam.dot(cstar) dx = s.dot(astar) dy = s.dot(bstar) dz = s.dot(cstar) x = x_mm * scl_x y = y_mm * scl_y z = (z_rad * 180 / math.pi - phi_start) / phi_range fout.write('%4d%4d%4d%8.2f%8.2f%4d%8.5f%8.5f%8.5f%8.5f%8.5f%8.5f' % \ (h, k, l, I[j], sigI[j], run, ix, dx, iy, dy, iz, dz)) fout.write('%7.2f%7.2f%8.2f%7.2f%5d\n' % (x, y, z, detector2t, istol)) fout.close() logger.info('Output %d reflections to %s' % (nref, hklout)) return
def run(args): import libtbx.load_env usage = "%s experiments.json indexed.pickle [options]" %libtbx.env.dispatcher_name parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if len(experiments) == 0: parser.print_help() return elif len(experiments) > 1: raise Sorry("More than one experiment present") experiment = experiments[0] assert(len(reflections) == 1) reflections = reflections[0] intensities = reflections['intensity.sum.value'] variances = reflections['intensity.sum.variance'] if 'intensity.prf.value' in reflections: intensities = reflections['intensity.prf.value'] variances = reflections['intensity.prf.variance'] sel = (variances > 0) intensities = intensities.select(sel) variances = variances.select(sel) sigmas = flex.sqrt(variances) indices = reflections['miller_index'].select(sel) from cctbx import crystal, miller crystal_symmetry = crystal.symmetry( space_group=experiment.crystal.get_space_group(), unit_cell=experiment.crystal.get_unit_cell()) miller_set = miller.set( crystal_symmetry=crystal_symmetry, anomalous_flag=True, indices=indices) miller_array = miller.array( miller_set=miller_set, data=intensities, sigmas=sigmas).set_observation_type_xray_intensity() #miller_array.setup_binner(n_bins=50, reflections_per_bin=100) miller_array.setup_binner(auto_binning=True, n_bins=20) result = miller_array.i_over_sig_i(use_binning=True) result.show() from cctbx import uctbx d_star_sq_centre = result.binner.bin_centers(2) i_over_sig_i = flex.double( [d if d is not None else 0 for d in result.data[1:-1]]) sel = (i_over_sig_i > 0) d_star_sq_centre = d_star_sq_centre.select(sel) i_over_sig_i = i_over_sig_i.select(sel) log_i_over_sig_i = flex.log(i_over_sig_i) weights = result.binner.counts()[1:-1].as_double().select(sel) fit = flex.linear_regression( d_star_sq_centre, log_i_over_sig_i, weights=weights) m = fit.slope() c = fit.y_intercept() import math y_cutoff = math.log(params.i_sigi_cutoff) x_cutoff = (y_cutoff - c)/m estimated_d_min = uctbx.d_star_sq_as_d(x_cutoff) print "estimated d_min: %.2f" %estimated_d_min if params.plot: from matplotlib import pyplot fig = pyplot.figure() ax = fig.add_subplot(1,1,1) ax.plot( list(d_star_sq_centre), list(log_i_over_sig_i), label=r"ln(I/sigI)") ax.plot(pyplot.xlim(), [(m * x + c) for x in pyplot.xlim()], color='red') ax.plot([x_cutoff, x_cutoff], pyplot.ylim(), color='grey', linestyle='dashed') ax.plot(pyplot.xlim(), [y_cutoff, y_cutoff], color='grey', linestyle='dashed') ax.set_xlabel("d_star_sq") ax.set_ylabel("ln(I/sigI)") ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis xticks = ax.get_xticks() xlim = ax.get_xlim() xticks_d = [ uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ] xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks] ax_.set_xticks(xticks) ax_.set_xlim(ax.get_xlim()) ax_.set_xlabel(r"Resolution ($\AA$)") ax_.set_xticklabels(["%.1f" %d for d in xticks_d]) pyplot.savefig("estimate_resolution_limit.png") pyplot.clf()
frame, len(subset), average) # from matplotlib import pylab # pylab.imshow((count > 0).as_numpy_array()) # pylab.show() average = flex.double(len(sum_background)) variance = flex.double(len(sum_background)) count_mask = count > 1 indices = flex.size_t(range(len(mask))).select(count_mask.as_1d()) from matplotlib import pylab pylab.imshow(count_mask.as_numpy_array()) pylab.show() sumb = sum_background.as_1d().select(indices) numb = count.as_1d().select(indices).as_double() avrb = sumb / numb sumsqb = sum_sq_background.as_1d().select(indices) varb = (sumsqb - sumb*sumb / numb) / (numb - 1) average.set_selected(indices, avrb) average.reshape(count_mask.accessor()) variance.set_selected(indices, varb) variance.reshape(count_mask.accessor()) print "Saving to model.pickle" with open("model.pickle", "w") as outfile: import cPickle as pickle pickle.dump((average, count_mask, flex.sqrt(variance)), outfile)
def export_mtz( integrated_data, experiment_list, hklout, ignore_panels=False, include_partials=False, keep_partials=False, min_isigi=None, force_static_model=False, filter_ice_rings=False, ): """Export data from integrated_data corresponding to experiment_list to an MTZ file hklout.""" from dials.array_family import flex # for the moment assume (and assert) that we will convert data from exactly # one lattice... # FIXME allow for more than one experiment in here: this is fine just add # multiple MTZ data sets (DIALS1...DIALSN) and multiple batch headers: one # range of batches for each experiment assert len(experiment_list) == 1 # select reflections that are assigned to an experiment (i.e. non-negative id) integrated_data = integrated_data.select(integrated_data["id"] >= 0) assert max(integrated_data["id"]) == 0 # strip out negative variance reflections: these should not really be there # FIXME Doing select on summation results. Should do on profile result if # present? Yes if "intensity.prf.variance" in integrated_data: selection = integrated_data.get_flags(integrated_data.flags.integrated, all=True) else: selection = integrated_data.get_flags(integrated_data.flags.integrated_sum) integrated_data = integrated_data.select(selection) selection = integrated_data["intensity.sum.variance"] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info("Removing %d reflections with negative variance" % selection.count(True)) if "intensity.prf.variance" in integrated_data: selection = integrated_data["intensity.prf.variance"] <= 0 if selection.count(True) > 0: integrated_data.del_selected(selection) logger.info("Removing %d profile reflections with negative variance" % selection.count(True)) if filter_ice_rings: selection = integrated_data.get_flags(integrated_data.flags.in_powder_ring) integrated_data.del_selected(selection) logger.info("Removing %d reflections in ice ring resolutions" % selection.count(True)) if min_isigi is not None: selection = ( integrated_data["intensity.sum.value"] / flex.sqrt(integrated_data["intensity.sum.variance"]) ) < min_isigi integrated_data.del_selected(selection) logger.info("Removing %d reflections with I/Sig(I) < %s" % (selection.count(True), min_isigi)) if "intensity.prf.variance" in integrated_data: selection = ( integrated_data["intensity.prf.value"] / flex.sqrt(integrated_data["intensity.prf.variance"]) ) < min_isigi integrated_data.del_selected(selection) logger.info("Removing %d profile reflections with I/Sig(I) < %s" % (selection.count(True), min_isigi)) # FIXME in here work on including partial reflections => at this stage best # to split off the partial refections into a different selection & handle # gracefully... better to work on a short list as will need to "pop" them & # find matching parts to combine. if include_partials: integrated_data = sum_partial_reflections(integrated_data) integrated_data = scale_partial_reflections(integrated_data) if "partiality" in integrated_data: selection = integrated_data["partiality"] < 0.99 if selection.count(True) > 0 and not keep_partials: integrated_data.del_selected(selection) logger.info("Removing %d incomplete reflections" % selection.count(True)) # FIXME TODO for more than one experiment into an MTZ file: # # - add an epoch (or recover an epoch) from the scan and add this as an extra # column to the MTZ file for scaling, so we know that the two lattices were # integrated at the same time # - decide a sensible BATCH increment to apply to the BATCH value between # experiments and add this # # At the moment this is probably enough to be working on. experiment = experiment_list[0] # also only work with one panel(for the moment) if not ignore_panels: assert len(experiment.detector) == 1 from scitbx import matrix if experiment.goniometer: axis = matrix.col(experiment.goniometer.get_rotation_axis()) else: axis = 0.0, 0.0, 0.0 s0 = experiment.beam.get_s0() wavelength = experiment.beam.get_wavelength() panel = experiment.detector[0] origin = matrix.col(panel.get_origin()) fast = matrix.col(panel.get_fast_axis()) slow = matrix.col(panel.get_slow_axis()) pixel_size = panel.get_pixel_size() fast *= pixel_size[0] slow *= pixel_size[1] cb_op_to_ref = experiment.crystal.get_space_group().info().change_of_basis_op_to_reference_setting() experiment.crystal = experiment.crystal.change_basis(cb_op_to_ref) U = experiment.crystal.get_U() if experiment.goniometer is not None: F = matrix.sqr(experiment.goniometer.get_fixed_rotation()) else: F = matrix.sqr((1, 0, 0, 0, 1, 0, 0, 0, 1)) unit_cell = experiment.crystal.get_unit_cell() from iotbx import mtz from scitbx.array_family import flex from math import floor, sqrt m = mtz.object() m.set_title("from dials.export_mtz") m.set_space_group_info(experiment.crystal.get_space_group().info()) if experiment.scan: image_range = experiment.scan.get_image_range() else: image_range = 1, 1 # pointless (at least) doesn't like batches starting from zero b_incr = max(image_range[0], 1) for b in range(image_range[0], image_range[1] + 1): o = m.add_batch().set_num(b + b_incr).set_nbsetid(1).set_ncryst(1) o.set_time1(0.0).set_time2(0.0).set_title("Batch %d" % (b + b_incr)) o.set_ndet(1).set_theta(flex.float((0.0, 0.0))).set_lbmflg(0) o.set_alambd(wavelength).set_delamb(0.0).set_delcor(0.0) o.set_divhd(0.0).set_divvd(0.0) # FIXME hard-coded assumption on indealized beam vector below... this may be # broken when we come to process data from a non-imgCIF frame o.set_so(flex.float(s0)).set_source(flex.float((0, 0, -1))) # these are probably 0, 1 respectively, also flags for how many are set, sd o.set_bbfac(0.0).set_bscale(1.0) o.set_sdbfac(0.0).set_sdbscale(0.0).set_nbscal(0) # unit cell (this is fine) and the what-was-refined-flags FIXME hardcoded # take time-varying parameters from the *end of the frame* unlikely to # be much different at the end - however only exist if time-varying refinement # was used if not force_static_model and experiment.crystal.num_scan_points > 0: _unit_cell = experiment.crystal.get_unit_cell_at_scan_point(b - image_range[0]) _U = experiment.crystal.get_U_at_scan_point(b - image_range[0]) else: _unit_cell = unit_cell _U = U # apply the fixed rotation to this to unify matrix definitions - F * U # was what was used in the actual prediction: U appears to be stored # as the transpose?! At least is for Mosflm... # # FIXME Do we need to apply the setting rotation here somehow? i.e. we have # the U.B. matrix assuming that the axis is equal to S * axis_datum but # here we are just giving the effective axis so at scan angle 0 this will # not be correct... FIXME 2 not even sure we can express the stack of # matrices S * R * F * U * B in MTZ format?... _U = dials_u_to_mosflm(F * _U, _unit_cell) # FIXME need to get what was refined and what was constrained from the # crystal model o.set_cell(flex.float(_unit_cell.parameters())) o.set_lbcell(flex.int((-1, -1, -1, -1, -1, -1))) o.set_umat(flex.float(_U.transpose().elems)) # get the mosaic spread though today it may not actually be set mosaic = experiment.crystal.get_mosaicity() o.set_crydat(flex.float([mosaic, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])) o.set_lcrflg(0) o.set_datum(flex.float((0.0, 0.0, 0.0))) # detector size, distance o.set_detlm(flex.float([0.0, panel.get_image_size()[0], 0.0, panel.get_image_size()[1], 0, 0, 0, 0])) o.set_dx(flex.float([panel.get_directed_distance(), 0.0])) # goniometer axes and names, and scan axis number, and number of axes, missets o.set_e1(flex.float(axis)) o.set_e2(flex.float((0.0, 0.0, 0.0))) o.set_e3(flex.float((0.0, 0.0, 0.0))) o.set_gonlab(flex.std_string(("AXIS", "", ""))) o.set_jsaxs(1) o.set_ngonax(1) o.set_phixyz(flex.float((0.0, 0.0, 0.0, 0.0, 0.0, 0.0))) # scan ranges, axis if experiment.scan: phi_start, phi_range = experiment.scan.get_image_oscillation(b) else: phi_start, phi_range = 0.0, 0.0 o.set_phistt(phi_start) o.set_phirange(phi_range) o.set_phiend(phi_start + phi_range) o.set_scanax(flex.float(axis)) # number of misorientation angles o.set_misflg(0) # crystal axis closest to rotation axis (why do I want this?) o.set_jumpax(0) # type of data - 1; 2D, 2; 3D, 3; Laue o.set_ldtype(2) # now create the actual data structures - first keep a track of the columns # H K L M/ISYM BATCH I SIGI IPR SIGIPR FRACTIONCALC XDET YDET ROT WIDTH # LP MPART FLAG BGPKRATIOS from cctbx.array_family import flex as cflex # implicit import from cctbx.miller import map_to_asu_isym # implicit import # gather the required information for the reflection file nref = len(integrated_data["miller_index"]) x_px, y_px, z_px = integrated_data["xyzcal.px"].parts() xdet = flex.double(x_px) ydet = flex.double(y_px) zdet = flex.double(z_px) # compute ROT values if experiment.scan: rot = flex.double([experiment.scan.get_angle_from_image_index(z) for z in zdet]) else: rot = zdet # compute BATCH values batch = flex.floor(zdet).iround() + 1 + b_incr # we're working with full reflections so... fractioncalc = flex.double(nref, 1.0) # now go for it and make an MTZ file... x = m.add_crystal("XTAL", "DIALS", unit_cell.parameters()) d = x.add_dataset("FROMDIALS", wavelength) # now add column information... # FIXME add DIALS_FLAG which can include e.g. was partial etc. type_table = { "H": "H", "K": "H", "L": "H", "I": "J", "SIGI": "Q", "IPR": "J", "SIGIPR": "Q", "BG": "R", "SIGBG": "R", "XDET": "R", "YDET": "R", "BATCH": "B", "BGPKRATIOS": "R", "WIDTH": "R", "MPART": "I", "M_ISYM": "Y", "FLAG": "I", "LP": "R", "FRACTIONCALC": "R", "ROT": "R", "DQE": "R", } # derive index columns from original indices with # # from m.replace_original_index_miller_indices # # so all that is needed now is to make space for the reflections - fill with # zeros... m.adjust_column_array_sizes(nref) m.set_n_reflections(nref) # assign H, K, L, M_ISYM space for column in "H", "K", "L", "M_ISYM": d.add_column(column, type_table[column]).set_values(flex.double(nref, 0.0).as_float()) m.replace_original_index_miller_indices(cb_op_to_ref.apply(integrated_data["miller_index"])) d.add_column("BATCH", type_table["BATCH"]).set_values(batch.as_double().as_float()) if "lp" in integrated_data: lp = integrated_data["lp"] else: lp = flex.double(nref, 1.0) if "dqe" in integrated_data: dqe = integrated_data["dqe"] else: dqe = flex.double(nref, 1.0) I_profile = None V_profile = None I_sum = None V_sum = None # FIXME errors in e.g. LP correction need to be propogated here scl = lp / dqe if "intensity.prf.value" in integrated_data: I_profile = integrated_data["intensity.prf.value"] * scl V_profile = integrated_data["intensity.prf.variance"] * scl * scl # Trap negative variances assert V_profile.all_gt(0) d.add_column("IPR", type_table["I"]).set_values(I_profile.as_float()) d.add_column("SIGIPR", type_table["SIGI"]).set_values(flex.sqrt(V_profile).as_float()) if "intensity.sum.value" in integrated_data: I_sum = integrated_data["intensity.sum.value"] * scl V_sum = integrated_data["intensity.sum.variance"] * scl * scl # Trap negative variances assert V_sum.all_gt(0) d.add_column("I", type_table["I"]).set_values(I_sum.as_float()) d.add_column("SIGI", type_table["SIGI"]).set_values(flex.sqrt(V_sum).as_float()) if "background.sum.value" in integrated_data and "background.sum.variance" in integrated_data: bg = integrated_data["background.sum.value"] varbg = integrated_data["background.sum.variance"] assert (varbg >= 0).count(False) == 0 sigbg = flex.sqrt(varbg) d.add_column("BG", type_table["BG"]).set_values(bg.as_float()) d.add_column("SIGBG", type_table["SIGBG"]).set_values(sigbg.as_float()) d.add_column("FRACTIONCALC", type_table["FRACTIONCALC"]).set_values(fractioncalc.as_float()) d.add_column("XDET", type_table["XDET"]).set_values(xdet.as_float()) d.add_column("YDET", type_table["YDET"]).set_values(ydet.as_float()) d.add_column("ROT", type_table["ROT"]).set_values(rot.as_float()) d.add_column("LP", type_table["LP"]).set_values(lp.as_float()) d.add_column("DQE", type_table["DQE"]).set_values(dqe.as_float()) m.write(hklout) return m