def image_correlation(a, b): a = a.as_1d().as_double() b = b.as_1d().as_double() sel = (a > 0) & (b > 0) _a = a.select(sel) _b = b.select(sel) return sel.count(True), flex.linear_correlation(x=a, y=b).coefficient()
def correlation(a, b): from astrotbx.input_output.loader import load_image_gs from dials.array_family import flex data_a = load_image_gs(a).as_1d() data_b = load_image_gs(b).as_1d() c = flex.linear_correlation(data_a, data_b) print(c.coefficient())
def image_correlation(a, b): sig_a = extract_signal_mask(a) sig_b = extract_signal_mask(b) a = a.as_1d().as_double() b = b.as_1d().as_double() sel = (a > 0) & (b > 0) & sig_a & sig_b _a = a.select(sel) _b = b.select(sel) return sel.count(True), flex.linear_correlation(x=a, y=b).coefficient()
def correlation(self, x): """ The correlation of the Jacobian """ J = self.jacobian(x) C = flex.double(flex.grid(J.all()[1], J.all()[1])) for j in range(C.all()[0]): for i in range(C.all()[1]): a = J[:, i:i + 1].as_1d() b = J[:, j:j + 1].as_1d() C[j, i] = flex.linear_correlation(a, b).coefficient() return C
def profile_correlation(data, model): """Compute CC between reflection profiles data and model.""" assert data.focus() == model.focus() sel = data.as_1d() > 0 model = model.as_1d().select(sel) data = data.as_1d().select(sel).as_double() assert len(data) == len(model) correlation = flex.linear_correlation(data, model) return correlation.coefficient()
def profile_correlation(data, model): '''Compute CC between reflection profiles data and model.''' from dials.array_family import flex assert (data.focus() == model.focus()) sel = data.as_1d() > 0 model = model.as_1d().select(sel) data = data.as_1d().select(sel).as_double() assert (len(data) == len(model)) correlation = flex.linear_correlation(data, model) return correlation.coefficient()
def profile_correlation(data, model): '''Compute CC between reflection profiles data and model.''' from dials.array_family import flex assert(data.focus() == model.focus()) sel = data.as_1d() > 0 model = model.as_1d().select(sel) data = data.as_1d().select(sel).as_double() assert(len(data) == len(model)) correlation = flex.linear_correlation(data, model) return correlation.coefficient()
def compare_pickle_with_mtz(params): """Compare intensities in pickle file with the scaled and merged intensities provided through the mtz file.""" from libtbx.utils import Sorry from iotbx import mtz from dials.array_family import flex import cPickle as pickle m = mtz.object(params.reference) d = pickle.load(open(params.data)) # fnd the right reference data mad = m.as_miller_arrays_dict(merge_equivalents=False) i = None for k in mad.keys(): if k[2] == "IMEAN": i = mad[k].as_non_anomalous_array().expand_to_p1() assert not i is None # print reference information print("Reference") i.show_summary() # clean up - remove non-integrated data - TODO look at just strong spots if params.filter_integrated: d = d.select(d.get_flags(d.flags.integrated)) if params.filter_partiality: d = d.select(d["partiality"] > params.filter_partiality) # apply scale factors from input file, if present # if requested, scale data to reference using simple kB model if params.kb_scale: raise Sorry("No can do, implementing kB scaling is on the to do") # match data to reference from cctbx import miller mi = miller.set( crystal_symmetry=i.crystal_symmetry(), anomalous_flag=False, indices=d["miller_index"], ).expand_to_p1() match = mi.match_indices(i) pairs = match.pairs() i1 = flex.double() i2 = flex.double() scl = flex.double() # TODO remove outliers here from the paired up list => do not need to # worry about them biasing the correlation for p0, p1 in pairs: i1.append(d["intensity.sum.value"][p0]) i2.append(i.data()[p1]) if "partiality" in d: scl.append(d["partiality"][p0]) else: scl.append(1.0) corr = flex.linear_correlation(i1, i2) corr2 = flex.linear_correlation(i1 / scl, i2) print("Correlation:", corr.coefficient(), corr2.coefficient(), pairs.size(), d.size()) if params.out: with open(params.out, "w") as f: for iis in zip(i1, i2, scl): f.write("%f %f %f\n" % iis)
def run(self): ''' Parse the options. ''' from dials.util.options import flatten_experiments, flatten_reflections # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) self.params = params experiments = flatten_experiments(params.input.experiments) # Find all detector objects detectors = experiments.detectors() # Verify inputs if len(params.input.reflections) == len(detectors) and len(detectors) > 1: # case for passing in multiple images on the command line assert len(params.input.reflections) == len(detectors) reflections = flex.reflection_table() for expt_id in xrange(len(detectors)): subset = params.input.reflections[expt_id].data subset['id'] = flex.int(len(subset), expt_id) reflections.extend(subset) else: # case for passing in combined experiments and reflections reflections = flatten_reflections(params.input.reflections)[0] detector = detectors[0] #from dials.algorithms.refinement.prediction import ExperimentsPredictor #ref_predictor = ExperimentsPredictor(experiments, force_stills=experiments.all_stills()) print "N reflections total:", len(reflections) if params.residuals.exclude_outliers: reflections = reflections.select(reflections.get_flags(reflections.flags.used_in_refinement)) print "N reflections used in refinement:", len(reflections) print "Reporting only on those reflections used in refinement" if self.params.residuals.i_sigi_cutoff is not None: sel = (reflections['intensity.sum.value']/flex.sqrt(reflections['intensity.sum.variance'])) >= self.params.residuals.i_sigi_cutoff reflections = reflections.select(sel) print "After filtering by I/sigi cutoff of %f, there are %d reflections left"%(self.params.residuals.i_sigi_cutoff,len(reflections)) reflections['difference_vector_norms'] = (reflections['xyzcal.mm']-reflections['xyzobs.mm.value']).norms() n = len(reflections) rmsd = self.get_weighted_rmsd(reflections) print "Dataset RMSD (microns)", rmsd * 1000 if params.tag is None: tag = '' else: tag = '%s '%params.tag # set up delta-psi ratio heatmap p = flex.int() # positive n = flex.int() # negative for i in set(reflections['id']): exprefls = reflections.select(reflections['id']==i) p.append(len(exprefls.select(exprefls['delpsical.rad']>0))) n.append(len(exprefls.select(exprefls['delpsical.rad']<0))) plt.hist2d(p, n, bins=30) cb = plt.colorbar() cb.set_label("N images") plt.title(r"%s2D histogram of pos vs. neg $\Delta\Psi$ per image"%tag) plt.xlabel(r"N reflections with $\Delta\Psi$ > 0") plt.ylabel(r"N reflections with $\Delta\Psi$ < 0") self.delta_scalar = 50 # Iterate through the detectors, computing detector statistics at the per-panel level (IE one statistic per panel) # Per panel dictionaries rmsds = {} refl_counts = {} transverse_rmsds = {} radial_rmsds = {} ttdpcorr = {} pg_bc_dists = {} mean_delta_two_theta = {} # per panelgroup flex arrays pg_rmsds = flex.double() pg_r_rmsds = flex.double() pg_t_rmsds = flex.double() pg_refls_count = flex.int() pg_refls_count_d = {} table_header = ["PG id", "RMSD","Radial", "Transverse", "N refls"] table_header2 = ["","(um)","RMSD (um)","RMSD (um)",""] table_data = [] table_data.append(table_header) table_data.append(table_header2) # Compute a set of radial and transverse displacements for each reflection print "Setting up stats..." tmp = flex.reflection_table() # Need to construct a variety of vectors for panel_id, panel in enumerate(detector): panel_refls = reflections.select(reflections['panel'] == panel_id) bcl = flex.vec3_double() tto = flex.double() ttc = flex.double() # Compute the beam center in lab space (a vector pointing from the origin to where the beam would intersect # the panel, if it did intersect the panel) for expt_id in set(panel_refls['id']): beam = experiments[expt_id].beam s0 = beam.get_s0() expt_refls = panel_refls.select(panel_refls['id'] == expt_id) beam_centre = panel.get_beam_centre_lab(s0) bcl.extend(flex.vec3_double(len(expt_refls), beam_centre)) obs_x, obs_y, _ = expt_refls['xyzobs.px.value'].parts() cal_x, cal_y, _ = expt_refls['xyzcal.px'].parts() tto.extend(flex.double([panel.get_two_theta_at_pixel(s0, (obs_x[i], obs_y[i])) for i in xrange(len(expt_refls))])) ttc.extend(flex.double([panel.get_two_theta_at_pixel(s0, (cal_x[i], cal_y[i])) for i in xrange(len(expt_refls))])) panel_refls['beam_centre_lab'] = bcl panel_refls['two_theta_obs'] = tto * (180/math.pi) panel_refls['two_theta_cal'] = ttc * (180/math.pi) #+ (0.5*panel_refls['delpsical.rad']*panel_refls['two_theta_obs']) # Compute obs in lab space x, y, _ = panel_refls['xyzobs.mm.value'].parts() c = flex.vec2_double(x, y) panel_refls['obs_lab_coords'] = panel.get_lab_coord(c) # Compute deltaXY in panel space. This vector is relative to the panel origin x, y, _ = (panel_refls['xyzcal.mm'] - panel_refls['xyzobs.mm.value']).parts() # Convert deltaXY to lab space, subtracting off of the panel origin panel_refls['delta_lab_coords'] = panel.get_lab_coord(flex.vec2_double(x,y)) - panel.get_origin() tmp.extend(panel_refls) reflections = tmp # The radial vector points from the center of the reflection to the beam center radial_vectors = (reflections['obs_lab_coords'] - reflections['beam_centre_lab']).each_normalize() # The transverse vector is orthogonal to the radial vector and the beam vector transverse_vectors = radial_vectors.cross(reflections['beam_centre_lab']).each_normalize() # Compute the raidal and transverse components of each deltaXY reflections['radial_displacements'] = reflections['delta_lab_coords'].dot(radial_vectors) reflections['transverse_displacements'] = reflections['delta_lab_coords'].dot(transverse_vectors) # Iterate through the detector at the specified hierarchy level for pg_id, pg in enumerate(iterate_detector_at_level(detector.hierarchy(), 0, params.hierarchy_level)): pg_msd_sum = 0 pg_r_msd_sum = 0 pg_t_msd_sum = 0 pg_refls = 0 pg_delpsi = flex.double() pg_deltwotheta = flex.double() for p in iterate_panels(pg): panel_id = id_from_name(detector, p.get_name()) panel_refls = reflections.select(reflections['panel'] == panel_id) n = len(panel_refls) pg_refls += n delta_x = panel_refls['xyzcal.mm'].parts()[0] - panel_refls['xyzobs.mm.value'].parts()[0] delta_y = panel_refls['xyzcal.mm'].parts()[1] - panel_refls['xyzobs.mm.value'].parts()[1] tmp = flex.sum((delta_x**2)+(delta_y**2)) pg_msd_sum += tmp r = panel_refls['radial_displacements'] t = panel_refls['transverse_displacements'] pg_r_msd_sum += flex.sum_sq(r) pg_t_msd_sum += flex.sum_sq(t) pg_delpsi.extend(panel_refls['delpsical.rad']*180/math.pi) pg_deltwotheta.extend(panel_refls['two_theta_obs'] - panel_refls['two_theta_cal']) bc = col(pg.get_beam_centre_lab(s0)) ori = get_center(pg) pg_bc_dists[pg.get_name()] = (ori-bc).length() if len(pg_deltwotheta) > 0: mean_delta_two_theta[pg.get_name()] = flex.mean(pg_deltwotheta) else: mean_delta_two_theta[pg.get_name()] = 0 if pg_refls == 0: pg_rmsd = pg_r_rmsd = pg_t_rmsd = 0 else: pg_rmsd = math.sqrt(pg_msd_sum/pg_refls) * 1000 pg_r_rmsd = math.sqrt(pg_r_msd_sum/pg_refls) * 1000 pg_t_rmsd = math.sqrt(pg_t_msd_sum/pg_refls) * 1000 pg_rmsds.append(pg_rmsd) pg_r_rmsds.append(pg_r_rmsd) pg_t_rmsds.append(pg_t_rmsd) pg_refls_count.append(pg_refls) pg_refls_count_d[pg.get_name()] = pg_refls table_data.append(["%d"%pg_id, "%.1f"%pg_rmsd, "%.1f"%pg_r_rmsd, "%.1f"%pg_t_rmsd, "%6d"%pg_refls]) refl_counts[pg.get_name()] = pg_refls if pg_refls == 0: rmsds[p.get_name()] = -1 radial_rmsds[p.get_name()] = -1 transverse_rmsds[p.get_name()] = -1 ttdpcorr[pg.get_name()] = -1 else: rmsds[pg.get_name()] = pg_rmsd radial_rmsds[pg.get_name()] = pg_r_rmsd transverse_rmsds[pg.get_name()] = pg_t_rmsd lc = flex.linear_correlation(pg_delpsi, pg_deltwotheta) ttdpcorr[pg.get_name()] = lc.coefficient() r1 = ["Weighted mean"] r2 = ["Weighted stddev"] if len(pg_rmsds) > 1: stats = flex.mean_and_variance(pg_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(pg_r_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(pg_t_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) else: r1.extend([""]*3) r2.extend([""]*3) r1.append("") r2.append("") table_data.append(r1) table_data.append(r2) table_data.append(["Mean", "", "", "", "%8.1f"%flex.mean(pg_refls_count.as_double())]) from libtbx import table_utils print "Detector statistics. Angles in degrees, RMSDs in microns" print table_utils.format(table_data,has_header=2,justify='center',delim=" ") self.histogram(reflections, '%sDifference vector norms (mm)'%tag) if params.show_plots: if self.params.tag is None: t = "" else: t = "%s "%self.params.tag self.image_rmsd_histogram(reflections, tag) # Plots! these are plots with callbacks to draw on individual panels self.detector_plot_refls(detector, reflections, '%sOverall positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltas) self.detector_plot_refls(detector, reflections, '%sRadial positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_radial_deltas) self.detector_plot_refls(detector, reflections, '%sTransverse positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_transverse_deltas) self.detector_plot_refls(detector, reflections, r'%s$\Delta\Psi$'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltapsi, colorbar_units=r"$\circ$") self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY*%s'%(tag, self.delta_scalar), show=False, plot_callback=self.plot_deltas) self.detector_plot_refls(detector, reflections, '%sSP Manual CDF'%tag, show=False, plot_callback=self.plot_cdf_manually) self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY Histograms'%tag, show=False, plot_callback=self.plot_histograms) self.detector_plot_refls(detector, reflections, r'%sRadial displacements vs. $\Delta\Psi$, colored by $\Delta$XY'%tag, show=False, plot_callback=self.plot_radial_displacements_vs_deltapsi) self.detector_plot_refls(detector, reflections, r'%sDistance vector norms'%tag, show=False, plot_callback=self.plot_difference_vector_norms_histograms) # Plot intensity vs. radial_displacement fig = plt.figure() panel_id = 15 panel_refls = reflections.select(reflections['panel'] == panel_id) a = panel_refls['radial_displacements'] b = panel_refls['intensity.sum.value'] sel = (a > -0.2) & (a < 0.2) & (b < 50000) plt.hist2d(a.select(sel), b.select(sel), bins=100) plt.title("%s2D histogram of intensity vs. radial displacement for panel %d"%(tag, panel_id)) plt.xlabel("Radial displacement (mm)") plt.ylabel("Intensity") ax = plt.colorbar() ax.set_label("Counts") # Plot delta 2theta vs. deltapsi n_bins = 10 bin_size = len(reflections)//n_bins bin_low = [] bin_high = [] data = flex.sorted(reflections['two_theta_obs']) for i in xrange(n_bins): bin_low = data[i*bin_size] if (i+1)*bin_size >= len(reflections): bin_high = data[-1] else: bin_high = data[(i+1)*bin_size] refls = reflections.select((reflections['two_theta_obs'] >= bin_low) & (reflections['two_theta_obs'] <= bin_high)) a = refls['delpsical.rad']*180/math.pi b = refls['two_theta_obs'] - refls['two_theta_cal'] fig = plt.figure() sel = (a > -0.2) & (a < 0.2) & (b > -0.05) & (b < 0.05) plt.hist2d(a.select(sel), b.select(sel), bins=50, range = [[-0.2, 0.2], [-0.05, 0.05]]) cb = plt.colorbar() cb.set_label("N reflections") plt.title(r'%sBin %d (%.02f, %.02f 2$\Theta$) $\Delta2\Theta$ vs. $\Delta\Psi$. Showing %d of %d refls'%(tag,i,bin_low,bin_high,len(a.select(sel)),len(a))) plt.xlabel(r'$\Delta\Psi \circ$') plt.ylabel(r'$\Delta2\Theta \circ$') # Plot delta 2theta vs. 2theta a = reflections['two_theta_obs']#[:71610] b = reflections['two_theta_obs'] - reflections['two_theta_cal'] fig = plt.figure() limits = -0.05, 0.05 sel = (b > limits[0]) & (b < limits[1]) plt.hist2d(a.select(sel), b.select(sel), bins=100, range=((0,50), limits)) plt.clim((0,100)) cb = plt.colorbar() cb.set_label("N reflections") plt.title(r'%s$\Delta2\Theta$ vs. 2$\Theta$. Showing %d of %d refls'%(tag,len(a.select(sel)),len(a))) plt.xlabel(r'2$\Theta \circ$') plt.ylabel(r'$\Delta2\Theta \circ$') # calc the trendline z = np.polyfit(a.select(sel), b.select(sel), 1) print 'y=%.7fx+(%.7f)'%(z[0],z[1]) # Plots with single values per panel self.detector_plot_dict(detector, refl_counts, u"%s N reflections"%t, u"%6d", show=False) self.detector_plot_dict(detector, rmsds, "%s Positional RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, radial_rmsds, "%s Radial RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, transverse_rmsds, "%s Transverse RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, ttdpcorr, r"%s $\Delta2\Theta$ vs. $\Delta\Psi$ CC"%t, u"%5.3f", show=False) self.plot_unitcells(experiments) self.plot_data_by_two_theta(reflections, tag) # Plot data by panel group sorted_values = sorted(pg_bc_dists.values()) vdict = {} for k in pg_bc_dists: vdict[pg_bc_dists[k]] = k sorted_keys = [vdict[v] for v in sorted_values if vdict[v] in rmsds] x = [sorted_values[i] for i in xrange(len(sorted_values)) if pg_bc_dists.keys()[i] in rmsds] self.plot_multi_data(x, [[pg_refls_count_d[k] for k in sorted_keys], ([rmsds[k] for k in sorted_keys], [radial_rmsds[k] for k in sorted_keys], [transverse_rmsds[k] for k in sorted_keys]), [radial_rmsds[k]/transverse_rmsds[k] for k in sorted_keys], [mean_delta_two_theta[k] for k in sorted_keys]], "Panel group distance from beam center (mm)", ["N reflections", ("Overall RMSD", "Radial RMSD", "Transverse RMSD"), "R/T RMSD ratio", "Delta two theta"], ["N reflections", "RMSD (microns)", "R/T RMSD ratio", "Delta two theta (degrees)"], "%sData by panelgroup"%tag) if self.params.save_pdf: pp = PdfPages('residuals_%s.pdf'%(tag.strip())) for i in plt.get_fignums(): pp.savefig(plt.figure(i)) pp.close() else: plt.show()
def compute_local_cc_vs_ref(reflections, reference, kernel_size): """ Compute the local cc vs a reference set of intrnsities """ H, K, L, I1, I2 = match_reflections(reflections, reference) # Get the range of miller indices min_H, max_H = min(H), max(H) min_K, max_K = min(K), max(K) min_L, max_L = min(L), max(L) n_H = max_H - min_H + 1 n_K = max_K - min_K + 1 n_L = max_L - min_L + 1 mask = flex.bool(flex.grid(n_L, n_K, n_H)) data1 = flex.double(mask.accessor()) data2 = flex.double(mask.accessor()) H_array = flex.double(mask.accessor()) K_array = flex.double(mask.accessor()) L_array = flex.double(mask.accessor()) for h, k, l, intensity1, intensity2 in zip(H, K, L, I1, I2): x = h - min_H y = k - min_K z = l - min_L assert x >= 0 and y >= 0 and z >= 0 mask[z, y, x] = True data1[z, y, x] = intensity1 data2[z, y, x] = intensity2 H_array[z, y, x] = h K_array[z, y, x] = k L_array[z, y, x] = l indices = [] for k in range(mask.all()[0]): for j in range(mask.all()[1]): for i in range(mask.all()[2]): if mask[k, j, i] > 0: indices.append((k, j, i)) cc_array = flex.double(mask.accessor()) mask_out = flex.bool(mask.accessor()) for k, j, i in indices: k0 = k - kernel_size k1 = k + kernel_size + 1 j0 = j - kernel_size j1 = j + kernel_size + 1 i0 = i - kernel_size i1 = i + kernel_size + 1 if k0 < 0: k0 = 0 if j0 < 0: j0 = 0 if i0 < 0: i0 = 0 if k1 > mask.all()[0]: k1 = mask.all()[0] if j1 > mask.all()[1]: j1 = mask.all()[1] if i1 > mask.all()[2]: i1 = mask.all()[2] X = flex.double() Y = flex.double() for kk in range(k0, k1): for jj in range(j0, j1): for ii in range(i0, i1): if mask[kk, jj, ii]: X.append(data1[kk, jj, ii]) Y.append(data2[kk, jj, ii]) if len(X) > 0.25 * (2 * kernel_size + 1)**3: c = flex.linear_correlation(X, Y) cc_array[k, j, i] = c.coefficient() print(c.coefficient()) mask_out[k, j, i] = True selection = (mask_out == True).as_1d() H_sub = H_array.as_1d().select(selection) K_sub = K_array.as_1d().select(selection) L_sub = L_array.as_1d().select(selection) R_sub = cc_array.as_1d().select(selection) return H_sub, K_sub, L_sub, R_sub