def intensities(self): ''' Compare the intensities. ''' from dials.array_family import flex # Sort by resolution d = self.refl1['d'] index = flex.size_t(reversed(sorted(range(len(d)), key=lambda x: d[x]))) self.refl1.reorder(index) self.refl2.reorder(index) # Get the intensities I1 = self.refl1['intensity.sum.value'] I2 = self.refl2['intensity.sum.value'] S1 = flex.sqrt(self.refl1['intensity.sum.variance']) S2 = flex.sqrt(self.refl2['intensity.sum.variance']) xyz1 = self.refl1['xyzcal.px'] xyz2 = self.refl2['xyzcal.px'] # Compute chunked statistics corr = [] R = [] scale = [] res = [] for i in range(len(self.refl1) // 1000): # Get the chunks of data a = i * 1000 b = (i+1) * 1000 II1 = I1[a:b] II2 = I2[a:b] res.append(d[a]) # Compute the mean and standard deviation per chunk mv1 = flex.mean_and_variance(II1) mv2 = flex.mean_and_variance(II2) m1 = mv1.mean() m2 = mv2.mean() s1 = mv1.unweighted_sample_standard_deviation() s2 = mv2.unweighted_sample_standard_deviation() # compute the correlation coefficient r = (1/(len(II1) - 1))*sum(((II1[j] - m1) / s1) * ((II2[j] - m2) / s2) for j in range(len(II1))) corr.append(r) # Compute the scale between the chunks s = sum(II1) / sum(II2) scale.append(s) # Compute R between the chunks r = sum(abs(abs(II1[j]) - abs(s * II2[j])) for j in range(len(II1))) \ / sum(abs(II1[j]) for j in range(len(II1))) R.append(r) from matplotlib import pylab pylab.plot(corr, label="CC") pylab.plot(R, label="R") pylab.plot(scale, label="K") pylab.legend() pylab.show()
def plot_unitcells(self, experiments): if len(experiments) == 1: return all_a = flex.double() all_b = flex.double() all_c = flex.double() for crystal in experiments.crystals(): a, b, c = crystal.get_unit_cell().parameters()[0:3] all_a.append(a); all_b.append(b); all_c.append(c) fig, axes = plt.subplots(nrows=3, ncols=1) for ax, axis, data in zip(axes, ['A', 'B', 'C'], [all_a, all_b, all_c]): stats = flex.mean_and_variance(data) cutoff = 4*stats.unweighted_sample_standard_deviation() if cutoff < 0.5: cutoff = 0.5 limits = stats.mean()-cutoff, stats.mean()+cutoff sel = (data >= limits[0]) & (data <= limits[1]) subset = data.select(sel) h = flex.histogram(subset,n_slots=50) ax.plot(h.slot_centers().as_numpy_array(),h.slots().as_numpy_array(),'-') ax.set_title("%s axis histogram (showing %d of %d xtals). Mean: %7.2f Stddev: %7.2f"%( axis, len(subset), len(data), stats.mean(), stats.unweighted_sample_standard_deviation())) ax.set_ylabel("N lattices") ax.set_xlabel(r"$\AA$") ax.set_xlim(limits) plt.tight_layout()
def tst_for_dataset(self, creator, filename): from dials.array_family import flex from dials.algorithms.shoebox import MaskCode print filename rlist = flex.reflection_table.from_pickle(filename) shoebox = rlist['shoebox'] background = [sb.background.deep_copy() for sb in shoebox] success = creator(shoebox) assert(success.count(True) == len(success)) diff = [] for i in range(len(rlist)): mask = flex.bool([(m & MaskCode.Foreground) != 0 for m in shoebox[i].mask]) px1 = background[i].select(mask) px2 = shoebox[i].background.select(mask) den = max([flex.mean(px1), 1.0]) diff.append(flex.mean(px2 - px1) / den) diff = flex.double(diff) mv = flex.mean_and_variance(flex.double(diff)) mean = mv.mean() sdev = mv.unweighted_sample_standard_deviation() try: assert(abs(mean) < 0.01) except Exception: print "Mean: %f, Sdev: %f", mean, sdev from matplotlib import pylab pylab.hist(diff) pylab.show() raise
def assert_is_correct(self, data, mask): from scitbx.array_family import flex invalid, foreground, background, background_used, background_valid = \ assert_basic_mask_is_correct(mask, self.ninvalid, self.nforeground) subdata = data.select(background_valid) mv = flex.mean_and_variance(subdata) m = mv.mean() s = mv.unweighted_sample_standard_deviation() p0 = m - self.lower * s p1 = m + self.upper * s mask = (subdata >= p0) & (subdata <= p1) exp = background_valid.select(mask) assert(len(exp) == len(background_used)) assert(all(ii == jj) for ii, jj in zip(exp, background_used))
def check_reference(self, reference): ''' Check the reference spots. ''' from dials.array_family import flex from dials.algorithms.image.centroid import centroid_image from math import sqrt # Get a load of stuff I_sim = reference['intensity.sim'] I_exp = reference['intensity.exp'] I_cal = reference['intensity.prf.value'] I_var = reference['intensity.prf.variance'] # Get the transformed shoeboxes profiles = reference['rs_shoebox'] n_sigma = 3 n_sigma2 = 5 grid_size = 4 step_size = n_sigma2 / (grid_size + 0.5) eps = 1e-7 for i in range(len(profiles)): data = profiles[i].data #dmax = flex.max(data) #data = 100 * data / dmax #p = data.as_numpy_array() #p = p.astype(numpy.int) #print p print flex.sum(data), I_exp[i], I_cal[i] #assert(abs(flex.sum(data) - I_exp[i]) < eps) centroid = centroid_image(data) m = centroid.mean() v = centroid.variance() s1 = tuple(sqrt(vv) for vv in v) s2 = tuple(ss * step_size for ss in s1) assert(all(abs(mm - (grid_size + 0.5)) < 0.25 for mm in m)) assert(all(abs(ss2 - n_sigma / n_sigma2) < 0.25 for ss2 in s2)) # Calculate Z Z = (I_cal - I_exp) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f" % (Z_mean, Z_var) from matplotlib import pylab pylab.hist((I_cal - I_exp) / I_exp) pylab.show()
def run(self): '''Execute the script.''' from dials.util.options import flatten_experiments from libtbx.utils import Sorry from dials.array_family import flex # Parse the command line params, options = self.parser.parse_args(show_diff_phil=True) # Try to load the experiments if not params.input.experiments: print "No Experiments found in the input" self.parser.print_help() return experiments = flatten_experiments(params.input.experiments) print "{0} experiments loaded".format(len(experiments)) us0_vecs = self.extract_us0_vecs(experiments) e_vecs = self.extract_rotation_axes(experiments) angles = [us0.angle(e, deg=True) for us0, e in zip(us0_vecs, e_vecs)] fmt = "{:." + str(params.print_precision) + "f}" header = ['Exp\nid','Beam direction', 'Rotation axis', 'Angle (deg)'] rows = [] for iexp, (us0, e, ang) in enumerate(zip(us0_vecs, e_vecs, angles)): beam_str = " ".join([fmt] * 3).format(*us0.elems) e_str = " ".join([fmt] * 3).format(*e.elems) rows.append([str(iexp), beam_str, e_str, fmt.format(ang)]) if len(rows) > 0: st = simple_table(rows, header) print st.format() # mean and sd if len(rows) > 1: angles = flex.double(angles) mv = flex.mean_and_variance(angles) print "Mean and standard deviation of the angle" print (fmt.format(mv.mean()) + " +/- " + fmt.format( mv.unweighted_sample_standard_deviation())) print return
def test_for_reflections(self, refl, filename): from dials.algorithms.integration import ProfileFittingReciprocalSpace from dials.array_family import flex from dials.algorithms.shoebox import MaskCode from dials.algorithms.statistics import kolmogorov_smirnov_test_standard_normal from math import erf, sqrt, pi from copy import deepcopy from dials.algorithms.simulation.reciprocal_space import Simulator from os.path import basename print(basename(filename)) # refl = self.reference # Get the calculated background and simulated background B_sim = refl["background.sim"].as_double() I_sim = refl["intensity.sim"].as_double() I_exp = refl["intensity.exp"] # Set the background as simulated shoebox = refl["shoebox"] for i in range(len(shoebox)): bg = shoebox[i].background ms = shoebox[i].mask for j in range(len(bg)): bg[j] = B_sim[i] # Integrate integration = ProfileFittingReciprocalSpace( grid_size=4, threshold=0.02, frame_interval=0, n_sigma=4, sigma_b=0.024 * pi / 180.0, sigma_m=0.044 * pi / 180.0, ) old_size = len(refl) refl.extend(self.reference) integration(self.experiment, refl) reference = refl[-len(self.reference):] refl = refl[:len(self.reference)] assert len(refl) == old_size assert len(reference) == len(self.reference) I_cal = refl["intensity.sum.value"] I_var = refl["intensity.sum.variance"] # Check the reference profiles and spots are ok self.check_profiles(integration.learner) self.check_reference(reference) # reference = integration.learner # np = reference.locate().size() # for i in range(np): # profile = reference.locate().profile(i) # print "Profile: %d" % i # p = (profile.as_numpy_array() * 1000) # import numpy as np # p = p.astype(np.int) # print p # Only select variances greater than zero mask = refl.get_flags(refl.flags.integrated) I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) # Calculate the z score perc = self.mv3n_tolerance_interval(3 * 3) Z = (I_cal - I_exp) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print("Z: mean: %f, var: %f" % (Z_mean, Z_var)) # Do the kolmogorov smirnov test D, p = kolmogorov_smirnov_test_standard_normal(Z) print("KS: D: %f, p-value: %f" % (D, p)) # FIXME Z score should be a standard normal distribution. When background is # the main component, we do indeed see that the z score is in a standard # normal distribution. When the intensity dominates, the variance of the Z # scores decreases indicating that for increasing intensity of the signal, # the variance is over estimated. # assert(abs(Z_mean) <= 3 * Z_var) # from matplotlib import pylab # pylab.hist(Z, 20) # pylab.show() # Z_I = sorted(Z) ##n = int(0.05 * len(Z_I)) ##Z_I = Z_I[n:-n] ##mv = flex.mean_and_variance(flex.double(Z_I)) ##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation()) # edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))] # cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I] print("OK")
def run(self): ''' Parse the options. ''' from dials.util.options import flatten_experiments, flatten_reflections # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) self.params = params experiments = flatten_experiments(params.input.experiments) # Find all detector objects detectors = experiments.detectors() # Verify inputs if len(params.input.reflections) == len(detectors) and len(detectors) > 1: # case for passing in multiple images on the command line assert len(params.input.reflections) == len(detectors) reflections = flex.reflection_table() for expt_id in xrange(len(detectors)): subset = params.input.reflections[expt_id].data subset['id'] = flex.int(len(subset), expt_id) reflections.extend(subset) else: # case for passing in combined experiments and reflections reflections = flatten_reflections(params.input.reflections)[0] detector = detectors[0] #from dials.algorithms.refinement.prediction import ExperimentsPredictor #ref_predictor = ExperimentsPredictor(experiments, force_stills=experiments.all_stills()) print "N reflections total:", len(reflections) if params.residuals.exclude_outliers: reflections = reflections.select(reflections.get_flags(reflections.flags.used_in_refinement)) print "N reflections used in refinement:", len(reflections) print "Reporting only on those reflections used in refinement" if self.params.residuals.i_sigi_cutoff is not None: sel = (reflections['intensity.sum.value']/flex.sqrt(reflections['intensity.sum.variance'])) >= self.params.residuals.i_sigi_cutoff reflections = reflections.select(sel) print "After filtering by I/sigi cutoff of %f, there are %d reflections left"%(self.params.residuals.i_sigi_cutoff,len(reflections)) reflections['difference_vector_norms'] = (reflections['xyzcal.mm']-reflections['xyzobs.mm.value']).norms() n = len(reflections) rmsd = self.get_weighted_rmsd(reflections) print "Dataset RMSD (microns)", rmsd * 1000 if params.tag is None: tag = '' else: tag = '%s '%params.tag # set up delta-psi ratio heatmap p = flex.int() # positive n = flex.int() # negative for i in set(reflections['id']): exprefls = reflections.select(reflections['id']==i) p.append(len(exprefls.select(exprefls['delpsical.rad']>0))) n.append(len(exprefls.select(exprefls['delpsical.rad']<0))) plt.hist2d(p, n, bins=30) cb = plt.colorbar() cb.set_label("N images") plt.title(r"%s2D histogram of pos vs. neg $\Delta\Psi$ per image"%tag) plt.xlabel(r"N reflections with $\Delta\Psi$ > 0") plt.ylabel(r"N reflections with $\Delta\Psi$ < 0") self.delta_scalar = 50 # Iterate through the detectors, computing detector statistics at the per-panel level (IE one statistic per panel) # Per panel dictionaries rmsds = {} refl_counts = {} transverse_rmsds = {} radial_rmsds = {} ttdpcorr = {} pg_bc_dists = {} mean_delta_two_theta = {} # per panelgroup flex arrays pg_rmsds = flex.double() pg_r_rmsds = flex.double() pg_t_rmsds = flex.double() pg_refls_count = flex.int() pg_refls_count_d = {} table_header = ["PG id", "RMSD","Radial", "Transverse", "N refls"] table_header2 = ["","(um)","RMSD (um)","RMSD (um)",""] table_data = [] table_data.append(table_header) table_data.append(table_header2) # Compute a set of radial and transverse displacements for each reflection print "Setting up stats..." tmp = flex.reflection_table() # Need to construct a variety of vectors for panel_id, panel in enumerate(detector): panel_refls = reflections.select(reflections['panel'] == panel_id) bcl = flex.vec3_double() tto = flex.double() ttc = flex.double() # Compute the beam center in lab space (a vector pointing from the origin to where the beam would intersect # the panel, if it did intersect the panel) for expt_id in set(panel_refls['id']): beam = experiments[expt_id].beam s0 = beam.get_s0() expt_refls = panel_refls.select(panel_refls['id'] == expt_id) beam_centre = panel.get_beam_centre_lab(s0) bcl.extend(flex.vec3_double(len(expt_refls), beam_centre)) obs_x, obs_y, _ = expt_refls['xyzobs.px.value'].parts() cal_x, cal_y, _ = expt_refls['xyzcal.px'].parts() tto.extend(flex.double([panel.get_two_theta_at_pixel(s0, (obs_x[i], obs_y[i])) for i in xrange(len(expt_refls))])) ttc.extend(flex.double([panel.get_two_theta_at_pixel(s0, (cal_x[i], cal_y[i])) for i in xrange(len(expt_refls))])) panel_refls['beam_centre_lab'] = bcl panel_refls['two_theta_obs'] = tto * (180/math.pi) panel_refls['two_theta_cal'] = ttc * (180/math.pi) #+ (0.5*panel_refls['delpsical.rad']*panel_refls['two_theta_obs']) # Compute obs in lab space x, y, _ = panel_refls['xyzobs.mm.value'].parts() c = flex.vec2_double(x, y) panel_refls['obs_lab_coords'] = panel.get_lab_coord(c) # Compute deltaXY in panel space. This vector is relative to the panel origin x, y, _ = (panel_refls['xyzcal.mm'] - panel_refls['xyzobs.mm.value']).parts() # Convert deltaXY to lab space, subtracting off of the panel origin panel_refls['delta_lab_coords'] = panel.get_lab_coord(flex.vec2_double(x,y)) - panel.get_origin() tmp.extend(panel_refls) reflections = tmp # The radial vector points from the center of the reflection to the beam center radial_vectors = (reflections['obs_lab_coords'] - reflections['beam_centre_lab']).each_normalize() # The transverse vector is orthogonal to the radial vector and the beam vector transverse_vectors = radial_vectors.cross(reflections['beam_centre_lab']).each_normalize() # Compute the raidal and transverse components of each deltaXY reflections['radial_displacements'] = reflections['delta_lab_coords'].dot(radial_vectors) reflections['transverse_displacements'] = reflections['delta_lab_coords'].dot(transverse_vectors) # Iterate through the detector at the specified hierarchy level for pg_id, pg in enumerate(iterate_detector_at_level(detector.hierarchy(), 0, params.hierarchy_level)): pg_msd_sum = 0 pg_r_msd_sum = 0 pg_t_msd_sum = 0 pg_refls = 0 pg_delpsi = flex.double() pg_deltwotheta = flex.double() for p in iterate_panels(pg): panel_id = id_from_name(detector, p.get_name()) panel_refls = reflections.select(reflections['panel'] == panel_id) n = len(panel_refls) pg_refls += n delta_x = panel_refls['xyzcal.mm'].parts()[0] - panel_refls['xyzobs.mm.value'].parts()[0] delta_y = panel_refls['xyzcal.mm'].parts()[1] - panel_refls['xyzobs.mm.value'].parts()[1] tmp = flex.sum((delta_x**2)+(delta_y**2)) pg_msd_sum += tmp r = panel_refls['radial_displacements'] t = panel_refls['transverse_displacements'] pg_r_msd_sum += flex.sum_sq(r) pg_t_msd_sum += flex.sum_sq(t) pg_delpsi.extend(panel_refls['delpsical.rad']*180/math.pi) pg_deltwotheta.extend(panel_refls['two_theta_obs'] - panel_refls['two_theta_cal']) bc = col(pg.get_beam_centre_lab(s0)) ori = get_center(pg) pg_bc_dists[pg.get_name()] = (ori-bc).length() if len(pg_deltwotheta) > 0: mean_delta_two_theta[pg.get_name()] = flex.mean(pg_deltwotheta) else: mean_delta_two_theta[pg.get_name()] = 0 if pg_refls == 0: pg_rmsd = pg_r_rmsd = pg_t_rmsd = 0 else: pg_rmsd = math.sqrt(pg_msd_sum/pg_refls) * 1000 pg_r_rmsd = math.sqrt(pg_r_msd_sum/pg_refls) * 1000 pg_t_rmsd = math.sqrt(pg_t_msd_sum/pg_refls) * 1000 pg_rmsds.append(pg_rmsd) pg_r_rmsds.append(pg_r_rmsd) pg_t_rmsds.append(pg_t_rmsd) pg_refls_count.append(pg_refls) pg_refls_count_d[pg.get_name()] = pg_refls table_data.append(["%d"%pg_id, "%.1f"%pg_rmsd, "%.1f"%pg_r_rmsd, "%.1f"%pg_t_rmsd, "%6d"%pg_refls]) refl_counts[pg.get_name()] = pg_refls if pg_refls == 0: rmsds[p.get_name()] = -1 radial_rmsds[p.get_name()] = -1 transverse_rmsds[p.get_name()] = -1 ttdpcorr[pg.get_name()] = -1 else: rmsds[pg.get_name()] = pg_rmsd radial_rmsds[pg.get_name()] = pg_r_rmsd transverse_rmsds[pg.get_name()] = pg_t_rmsd lc = flex.linear_correlation(pg_delpsi, pg_deltwotheta) ttdpcorr[pg.get_name()] = lc.coefficient() r1 = ["Weighted mean"] r2 = ["Weighted stddev"] if len(pg_rmsds) > 1: stats = flex.mean_and_variance(pg_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(pg_r_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(pg_t_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) else: r1.extend([""]*3) r2.extend([""]*3) r1.append("") r2.append("") table_data.append(r1) table_data.append(r2) table_data.append(["Mean", "", "", "", "%8.1f"%flex.mean(pg_refls_count.as_double())]) from libtbx import table_utils print "Detector statistics. Angles in degrees, RMSDs in microns" print table_utils.format(table_data,has_header=2,justify='center',delim=" ") self.histogram(reflections, '%sDifference vector norms (mm)'%tag) if params.show_plots: if self.params.tag is None: t = "" else: t = "%s "%self.params.tag self.image_rmsd_histogram(reflections, tag) # Plots! these are plots with callbacks to draw on individual panels self.detector_plot_refls(detector, reflections, '%sOverall positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltas) self.detector_plot_refls(detector, reflections, '%sRadial positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_radial_deltas) self.detector_plot_refls(detector, reflections, '%sTransverse positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_transverse_deltas) self.detector_plot_refls(detector, reflections, r'%s$\Delta\Psi$'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltapsi, colorbar_units=r"$\circ$") self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY*%s'%(tag, self.delta_scalar), show=False, plot_callback=self.plot_deltas) self.detector_plot_refls(detector, reflections, '%sSP Manual CDF'%tag, show=False, plot_callback=self.plot_cdf_manually) self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY Histograms'%tag, show=False, plot_callback=self.plot_histograms) self.detector_plot_refls(detector, reflections, r'%sRadial displacements vs. $\Delta\Psi$, colored by $\Delta$XY'%tag, show=False, plot_callback=self.plot_radial_displacements_vs_deltapsi) self.detector_plot_refls(detector, reflections, r'%sDistance vector norms'%tag, show=False, plot_callback=self.plot_difference_vector_norms_histograms) # Plot intensity vs. radial_displacement fig = plt.figure() panel_id = 15 panel_refls = reflections.select(reflections['panel'] == panel_id) a = panel_refls['radial_displacements'] b = panel_refls['intensity.sum.value'] sel = (a > -0.2) & (a < 0.2) & (b < 50000) plt.hist2d(a.select(sel), b.select(sel), bins=100) plt.title("%s2D histogram of intensity vs. radial displacement for panel %d"%(tag, panel_id)) plt.xlabel("Radial displacement (mm)") plt.ylabel("Intensity") ax = plt.colorbar() ax.set_label("Counts") # Plot delta 2theta vs. deltapsi n_bins = 10 bin_size = len(reflections)//n_bins bin_low = [] bin_high = [] data = flex.sorted(reflections['two_theta_obs']) for i in xrange(n_bins): bin_low = data[i*bin_size] if (i+1)*bin_size >= len(reflections): bin_high = data[-1] else: bin_high = data[(i+1)*bin_size] refls = reflections.select((reflections['two_theta_obs'] >= bin_low) & (reflections['two_theta_obs'] <= bin_high)) a = refls['delpsical.rad']*180/math.pi b = refls['two_theta_obs'] - refls['two_theta_cal'] fig = plt.figure() sel = (a > -0.2) & (a < 0.2) & (b > -0.05) & (b < 0.05) plt.hist2d(a.select(sel), b.select(sel), bins=50, range = [[-0.2, 0.2], [-0.05, 0.05]]) cb = plt.colorbar() cb.set_label("N reflections") plt.title(r'%sBin %d (%.02f, %.02f 2$\Theta$) $\Delta2\Theta$ vs. $\Delta\Psi$. Showing %d of %d refls'%(tag,i,bin_low,bin_high,len(a.select(sel)),len(a))) plt.xlabel(r'$\Delta\Psi \circ$') plt.ylabel(r'$\Delta2\Theta \circ$') # Plot delta 2theta vs. 2theta a = reflections['two_theta_obs']#[:71610] b = reflections['two_theta_obs'] - reflections['two_theta_cal'] fig = plt.figure() limits = -0.05, 0.05 sel = (b > limits[0]) & (b < limits[1]) plt.hist2d(a.select(sel), b.select(sel), bins=100, range=((0,50), limits)) plt.clim((0,100)) cb = plt.colorbar() cb.set_label("N reflections") plt.title(r'%s$\Delta2\Theta$ vs. 2$\Theta$. Showing %d of %d refls'%(tag,len(a.select(sel)),len(a))) plt.xlabel(r'2$\Theta \circ$') plt.ylabel(r'$\Delta2\Theta \circ$') # calc the trendline z = np.polyfit(a.select(sel), b.select(sel), 1) print 'y=%.7fx+(%.7f)'%(z[0],z[1]) # Plots with single values per panel self.detector_plot_dict(detector, refl_counts, u"%s N reflections"%t, u"%6d", show=False) self.detector_plot_dict(detector, rmsds, "%s Positional RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, radial_rmsds, "%s Radial RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, transverse_rmsds, "%s Transverse RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, ttdpcorr, r"%s $\Delta2\Theta$ vs. $\Delta\Psi$ CC"%t, u"%5.3f", show=False) self.plot_unitcells(experiments) self.plot_data_by_two_theta(reflections, tag) # Plot data by panel group sorted_values = sorted(pg_bc_dists.values()) vdict = {} for k in pg_bc_dists: vdict[pg_bc_dists[k]] = k sorted_keys = [vdict[v] for v in sorted_values if vdict[v] in rmsds] x = [sorted_values[i] for i in xrange(len(sorted_values)) if pg_bc_dists.keys()[i] in rmsds] self.plot_multi_data(x, [[pg_refls_count_d[k] for k in sorted_keys], ([rmsds[k] for k in sorted_keys], [radial_rmsds[k] for k in sorted_keys], [transverse_rmsds[k] for k in sorted_keys]), [radial_rmsds[k]/transverse_rmsds[k] for k in sorted_keys], [mean_delta_two_theta[k] for k in sorted_keys]], "Panel group distance from beam center (mm)", ["N reflections", ("Overall RMSD", "Radial RMSD", "Transverse RMSD"), "R/T RMSD ratio", "Delta two theta"], ["N reflections", "RMSD (microns)", "R/T RMSD ratio", "Delta two theta (degrees)"], "%sData by panelgroup"%tag) if self.params.save_pdf: pp = PdfPages('residuals_%s.pdf'%(tag.strip())) for i in plt.get_fignums(): pp.savefig(plt.figure(i)) pp.close() else: plt.show()
def test_for_reference(self): from dials.algorithms.integration import ProfileFittingReciprocalSpace from dials.array_family import flex from dials.algorithms.shoebox import MaskCode from dials.algorithms.statistics import kolmogorov_smirnov_test_standard_normal from math import erf, sqrt, pi from copy import deepcopy from dials.algorithms.simulation.reciprocal_space import Simulator from os.path import basename # Integrate integration = ProfileFittingReciprocalSpace( grid_size=4, threshold=0.00, frame_interval=100, n_sigma=5, mask_n_sigma=3, sigma_b=0.024 * pi / 180.0, sigma_m=0.044 * pi / 180.0, ) # Integrate the reference profiles integration(self.experiment, self.reference) p = integration.learner.locate().profile(0) m = integration.learner.locate().mask(0) locator = integration.learner.locate() cor = locator.correlations() for j in range(cor.all()[0]): print(" ".join([str(cor[j, i]) for i in range(cor.all()[1])])) # exit(0) # from matplotlib import pylab # pylab.imshow(cor.as_numpy_array(), interpolation='none', vmin=-1, vmax=1) # pylab.show() # n = locator.size() # for i in range(n): # c = locator.coord(i) # p = locator.profile(i) # vmax = flex.max(p) # from matplotlib import pylab # for j in range(9): # pylab.subplot(3, 3, j+1) # pylab.imshow(p.as_numpy_array()[j], vmin=0, vmax=vmax, # interpolation='none') # pylab.show() # print "NRef: ", n # x = [] # y = [] # for i in range(n): # c = locator.coord(i) # x.append(c[0]) # y.append(c[1]) # from matplotlib import pylab # pylab.scatter(x,y) # pylab.show() # exit(0) import numpy # pmax = flex.max(p) # scale = 100 / pmax # print "Scale: ", 100 / pmax # p = p.as_numpy_array() *100 / pmax # p = p.astype(numpy.int) # print p # print m.as_numpy_array() # Check the reference profiles and spots are ok # self.check_profiles(integration.learner) # Make sure background is zero profiles = self.reference["rs_shoebox"] eps = 1e-7 for p in profiles: assert abs(flex.sum(p.background) - 0) < eps print("OK") # Only select variances greater than zero mask = self.reference.get_flags(self.reference.flags.integrated) I_cal = self.reference["intensity.sum.value"] I_var = self.reference["intensity.sum.variance"] B_sim = self.reference["background.sim"].as_double() I_sim = self.reference["intensity.sim"].as_double() I_exp = self.reference["intensity.exp"] P_cor = self.reference["profile.correlation"] X_pos, Y_pos, Z_pos = self.reference["xyzcal.px"].parts() I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) P_cor = P_cor.select(mask) max_ind = flex.max_index(I_cal) max_I = I_cal[max_ind] max_P = self.reference[max_ind]["rs_shoebox"].data max_C = self.reference[max_ind]["xyzcal.px"] max_S = self.reference[max_ind]["shoebox"].data min_ind = flex.min_index(P_cor) min_I = I_cal[min_ind] min_P = self.reference[min_ind]["rs_shoebox"].data min_C = self.reference[min_ind]["xyzcal.px"] min_S = self.reference[min_ind]["shoebox"].data ##for k in range(max_S.all()[0]): # if False: # for j in range(max_S.all()[1]): # for i in range(max_S.all()[2]): # max_S[k,j,i] = 0 # if (abs(i - max_S.all()[2] // 2) < 2 and # abs(j - max_S.all()[1] // 2) < 2 and # abs(k - max_S.all()[0] // 2) < 2): # max_S[k,j,i] = 100 # p = max_P.as_numpy_array() * 100 / flex.max(max_P) # p = p.astype(numpy.int) # print p # from dials.scratch.jmp.misc.test_transform import test_transform # grid_size = 4 # ndiv = 5 # sigma_b = 0.024 * pi / 180.0 # sigma_m = 0.044 * pi / 180.0 # n_sigma = 4.0 # max_P2 = test_transform( # self.experiment, # self.reference[max_ind]['shoebox'], # self.reference[max_ind]['s1'], # self.reference[max_ind]['xyzcal.mm'][2], # grid_size, # sigma_m, # sigma_b, # n_sigma, # ndiv) # max_P = max_P2 ref_ind = locator.index(max_C) ref_P = locator.profile(ref_ind) ref_C = locator.coord(ref_ind) print("Max Index: ", max_ind, max_I, flex.sum(max_P), flex.sum(max_S)) print("Coord: ", max_C, "Ref Coord: ", ref_C) print("Min Index: ", min_ind, min_I, flex.sum(min_P), flex.sum(min_S)) print("Coord: ", min_C, "Ref Coord: ", ref_C) # vmax = flex.max(max_P) # print sum(max_S) # print sum(max_P) # from matplotlib import pylab, cm # for j in range(9): # pylab.subplot(3, 3, j+1) # pylab.imshow(max_P.as_numpy_array()[j], vmin=0, vmax=vmax, # interpolation='none', cmap=cm.Greys_r) # pylab.show() # vmax = flex.max(min_P) # print sum(min_S) # print sum(min_P) # from matplotlib import pylab, cm # for j in range(9): # pylab.subplot(3, 3, j+1) # pylab.imshow(min_P.as_numpy_array()[j], vmin=0, vmax=vmax, # interpolation='none', cmap=cm.Greys_r) # pylab.show() # for k in range(max_S.all()[0]): # print '' # print 'Slice %d' % k # for j in range(max_S.all()[1]): # print ' '.join(["%-4d" % int(max_S[k,j,i]) for i in range(max_S.all()[2])]) print("Testing") def f(I): mask = flex.bool(flex.grid(9, 9, 9), False) for k in range(9): for j in range(9): for i in range(9): dx = 5 * (i - 4.5) / 4.5 dy = 5 * (j - 4.5) / 4.5 dz = 5 * (k - 4.5) / 4.5 dd = sqrt(dx**2 + dy**2 + dz**2) if dd <= 3: mask[k, j, i] = True mask = mask.as_1d() & (ref_P.as_1d() > 0) p = ref_P.as_1d().select(mask) c = max_P.as_1d().select(mask) return flex.sum((c - I * p)**2 / (I * p)) def df(I): mask = flex.bool(flex.grid(9, 9, 9), False) for k in range(9): for j in range(9): for i in range(9): dx = 5 * (i - 4.5) / 4.5 dy = 5 * (j - 4.5) / 4.5 dz = 5 * (k - 4.5) / 4.5 dd = sqrt(dx**2 + dy**2 + dz**2) if dd <= 3: mask[k, j, i] = True mask = mask.as_1d() & (ref_P.as_1d() > 0) p = ref_P.as_1d().select(mask) c = max_P.as_1d().select(mask) b = 0 return flex.sum(p) - flex.sum(c * c / (I * I * p)) # return flex.sum(p - p*c*c / ((b + I*p)**2)) # return flex.sum(3*p*p + (c*c*p*p - 4*b*p*p) / ((b + I*p)**2)) # return flex.sum(p - c*c / (I*I*p)) # return flex.sum(p * (-c+p*I)*(c+p*I)/((p*I)**2)) def d2f(I): mask = flex.bool(flex.grid(9, 9, 9), False) for k in range(9): for j in range(9): for i in range(9): dx = 5 * (i - 4.5) / 4.5 dy = 5 * (j - 4.5) / 4.5 dz = 5 * (k - 4.5) / 4.5 dd = sqrt(dx**2 + dy**2 + dz**2) if dd <= 3: mask[k, j, i] = True mask = mask.as_1d() & (ref_P.as_1d() > 0) p = ref_P.as_1d().select(mask) c = max_P.as_1d().select(mask) return flex.sum(2 * c * c * p * p / (p * I)**3) I = 10703 # flex.sum(max_P) mask = ref_P.as_1d() > 0 p = ref_P.as_1d().select(mask) c = max_P.as_1d().select(mask) for i in range(10): I = I - df(I) / d2f(I) # v = I*p # I = flex.sum(c * p / v) / flex.sum(p*p / v) print(I) from math import log ff = [] for I in range(9500, 11500): ff.append(f(I)) print(sorted(range(len(ff)), key=lambda x: ff[x])[0] + 9500) from matplotlib import pylab pylab.plot(range(9500, 11500), ff) pylab.show() # exit(0) # I = 10000 # print flex.sum((max_P - I * ref_P)**2) / flex.sum(I * ref_P) # I = 10100 # print flex.sum((max_P - I * ref_P)**2) / flex.sum(I * ref_P) # exit(0) print(flex.sum(self.reference[0]["rs_shoebox"].data)) print(I_cal[0]) # Calculate the z score perc = self.mv3n_tolerance_interval(3 * 3) Z = (I_cal - I_sim) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print("Z: mean: %f, var: %f, sig: %f" % (Z_mean, Z_var, sqrt(Z_var))) print(len(I_cal)) from matplotlib import pylab from mpl_toolkits.mplot3d import Axes3D # fig = pylab.figure() # ax = fig.add_subplot(111, projection='3d') # ax.scatter(X_pos, Y_pos, P_cor) # pylab.scatter(X_pos, P_cor) # pylab.scatter(Y_pos, P_cor) # pylab.scatter(Z_pos, P_cor) # pylab.hist(P_cor,100) # pylab.scatter(P_cor, (I_cal - I_exp) / I_exp) pylab.hist(Z, 100) # pylab.hist(I_cal,100) # pylab.hist(I_cal - I_sim, 100) pylab.show()
def __init__(self, Ibase, Gbase, FSIM, curvatures=False, **kwargs): # For backward compatibility handle the case where phil is undefined if "params" in kwargs.keys(): self.params = kwargs["params"] else: from xfel.command_line.cxi_merge import master_phil phil = iotbx.phil.process_command_line( args=[], master_string=master_phil).show() self.params = phil.work.extract() self.params.levmar.parameter_flags.append( "Bfactor") # default example refines Bfactor self.counter = 0 self.x = flex.double(list(Ibase) + list(Gbase)) self.N_I = len(Ibase) self.N_G = len(Gbase) self.N_raw_obs = FSIM.raw_obs.size() print "# structure factors:", self.N_I, "# frames:", self.N_G, "(Visited set; refined parameters)" step_threshold = self.params.levmar.termination.step_threshold objective_decrease_threshold = self.params.levmar.termination.objective_decrease_threshold if "Bfactor" in self.params.levmar.parameter_flags: self.x = self.x.concatenate(flex.double(len(Gbase), 0.0)) if "Deff" in self.params.levmar.parameter_flags: D_values = flex.double( [2. * e.crystal.domain_size for e in kwargs["experiments"]]) self.x = self.x.concatenate(D_values) if "Rxy" in self.params.levmar.parameter_flags: self.x = self.x.concatenate(flex.double(2 * len(Gbase), 0.0)) levenberg_helper = choice_as_helper_base( self.params.levmar.parameter_flags) self.helper = levenberg_helper(initial_estimates=self.x) self.helper.set_cpp_data(FSIM, self.N_I, self.N_G) if kwargs.has_key("experiments"): self.helper.set_wavelength( [e.beam.get_wavelength() for e in kwargs["experiments"]]) self.helper.set_domain_size([ 2. * e.crystal.domain_size for e in kwargs["experiments"] ]) #ad hoc factor of 2 self.helper.set_Astar_matrix( [e.crystal.get_A() for e in kwargs["experiments"]]) bitflags = choice_as_bitflag(self.params.levmar.parameter_flags) self.helper.set_parameter_flags(bitflags) self.helper.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations_encapsulated_eqns( non_linear_ls=self.helper, n_max_iterations=5000, track_all=True, step_threshold=step_threshold, objective_decrease_threshold=objective_decrease_threshold, verbose_iterations=True, ) if "Deff" in self.params.levmar.parameter_flags: newDeff = self.helper.x[self.N_I + self.N_G:] # XXX specific Dstats = flex.mean_and_variance(newDeff) print "Refined Deff mean & standard deviation:", print Dstats.mean(), Dstats.unweighted_sample_standard_deviation() if "Rxy" in self.params.levmar.parameter_flags: AX = self.helper.x[self.N_I + self.N_G:self.N_I + 2 * self.N_G] # XXX specific AY = self.helper.x[self.N_I + 2 * self.N_G:self.N_I + 3 * self.N_G] # XXX specific stats = flex.mean_and_variance(AX) print "Rx rotational increments in degrees: %8.6f +/- %8.6f" % ( stats.mean(), stats.unweighted_sample_standard_deviation()) stats = flex.mean_and_variance(AY) print "Ry rotational increments in degrees: %8.6f +/- %8.6f" % ( stats.mean(), stats.unweighted_sample_standard_deviation()) print "End of minimisation: Converged", self.helper.counter, "cycles" chi_squared = self.helper.objective() * 2. print "obj", chi_squared print "# of obs:", FSIM.raw_obs.size() dof = FSIM.raw_obs.size() - (len(self.x)) print "degrees of freedom =", dof print "chisq/dof: %7.3f" % (chi_squared / dof) print
def test_for_reference(self): from dials.algorithms.integration import ProfileFittingReciprocalSpace from dials.array_family import flex from dials.algorithms.shoebox import MaskCode from dials.algorithms.statistics import \ kolmogorov_smirnov_test_standard_normal from math import erf, sqrt, pi from copy import deepcopy from dials.algorithms.simulation.reciprocal_space import Simulator from os.path import basename # Integrate integration = ProfileFittingReciprocalSpace( grid_size=4, threshold=0.00, frame_interval=100, n_sigma=5, mask_n_sigma=3, sigma_b=0.024 * pi / 180.0, sigma_m=0.044 * pi / 180.0 ) # Integrate the reference profiles integration(self.experiment, self.reference) p = integration.learner.locate().profile(0) m = integration.learner.locate().mask(0) locator = integration.learner.locate() cor = locator.correlations() for j in range(cor.all()[0]): print ' '.join([str(cor[j,i]) for i in range(cor.all()[1])]) #exit(0) #from matplotlib import pylab #pylab.imshow(cor.as_numpy_array(), interpolation='none', vmin=-1, vmax=1) #pylab.show() #n = locator.size() #for i in range(n): #c = locator.coord(i) #p = locator.profile(i) #vmax = flex.max(p) #from matplotlib import pylab #for j in range(9): #pylab.subplot(3, 3, j+1) #pylab.imshow(p.as_numpy_array()[j], vmin=0, vmax=vmax, #interpolation='none') #pylab.show() #print "NRef: ", n #x = [] #y = [] #for i in range(n): #c = locator.coord(i) #x.append(c[0]) #y.append(c[1]) #from matplotlib import pylab #pylab.scatter(x,y) #pylab.show() #exit(0) import numpy #pmax = flex.max(p) #scale = 100 / pmax #print "Scale: ", 100 / pmax #p = p.as_numpy_array() *100 / pmax #p = p.astype(numpy.int) #print p #print m.as_numpy_array() # Check the reference profiles and spots are ok #self.check_profiles(integration.learner) # Make sure background is zero profiles = self.reference['rs_shoebox'] eps = 1e-7 for p in profiles: assert(abs(flex.sum(p.background) - 0) < eps) print 'OK' # Only select variances greater than zero mask = self.reference.get_flags(self.reference.flags.integrated) I_cal = self.reference['intensity.sum.value'] I_var = self.reference['intensity.sum.variance'] B_sim = self.reference['background.sim'].as_double() I_sim = self.reference['intensity.sim'].as_double() I_exp = self.reference['intensity.exp'] P_cor = self.reference['profile.correlation'] X_pos, Y_pos, Z_pos = self.reference['xyzcal.px'].parts() I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) P_cor = P_cor.select(mask) max_ind = flex.max_index(I_cal) max_I = I_cal[max_ind] max_P = self.reference[max_ind]['rs_shoebox'].data max_C = self.reference[max_ind]['xyzcal.px'] max_S = self.reference[max_ind]['shoebox'].data min_ind = flex.min_index(P_cor) min_I = I_cal[min_ind] min_P = self.reference[min_ind]['rs_shoebox'].data min_C = self.reference[min_ind]['xyzcal.px'] min_S = self.reference[min_ind]['shoebox'].data ##for k in range(max_S.all()[0]): #if False: #for j in range(max_S.all()[1]): #for i in range(max_S.all()[2]): #max_S[k,j,i] = 0 #if (abs(i - max_S.all()[2] // 2) < 2 and #abs(j - max_S.all()[1] // 2) < 2 and #abs(k - max_S.all()[0] // 2) < 2): #max_S[k,j,i] = 100 #p = max_P.as_numpy_array() * 100 / flex.max(max_P) #p = p.astype(numpy.int) #print p #from dials.scratch.jmp.misc.test_transform import test_transform #grid_size = 4 #ndiv = 5 #sigma_b = 0.024 * pi / 180.0 #sigma_m = 0.044 * pi / 180.0 #n_sigma = 4.0 #max_P2 = test_transform( #self.experiment, #self.reference[max_ind]['shoebox'], #self.reference[max_ind]['s1'], #self.reference[max_ind]['xyzcal.mm'][2], #grid_size, #sigma_m, #sigma_b, #n_sigma, #ndiv) #max_P = max_P2 ref_ind = locator.index(max_C) ref_P = locator.profile(ref_ind) ref_C = locator.coord(ref_ind) print "Max Index: ", max_ind, max_I, flex.sum(max_P), flex.sum(max_S) print "Coord: ", max_C, "Ref Coord: ", ref_C print "Min Index: ", min_ind, min_I, flex.sum(min_P), flex.sum(min_S) print "Coord: ", min_C, "Ref Coord: ", ref_C #vmax = flex.max(max_P) #print sum(max_S) #print sum(max_P) #from matplotlib import pylab, cm #for j in range(9): #pylab.subplot(3, 3, j+1) #pylab.imshow(max_P.as_numpy_array()[j], vmin=0, vmax=vmax, #interpolation='none', cmap=cm.Greys_r) #pylab.show() #vmax = flex.max(min_P) #print sum(min_S) #print sum(min_P) #from matplotlib import pylab, cm #for j in range(9): #pylab.subplot(3, 3, j+1) #pylab.imshow(min_P.as_numpy_array()[j], vmin=0, vmax=vmax, #interpolation='none', cmap=cm.Greys_r) #pylab.show() #for k in range(max_S.all()[0]): #print '' #print 'Slice %d' % k #for j in range(max_S.all()[1]): #print ' '.join(["%-4d" % int(max_S[k,j,i]) for i in range(max_S.all()[2])]) print "Testing" def f(I): mask = flex.bool(flex.grid(9,9,9), False) for k in range(9): for j in range(9): for i in range(9): dx = 5 * (i - 4.5) / 4.5 dy = 5 * (j - 4.5) / 4.5 dz = 5 * (k - 4.5) / 4.5 dd = sqrt(dx**2 + dy**2 + dz**2) if dd <= 3: mask[k,j,i] = True mask = mask.as_1d() & (ref_P.as_1d() > 0) p = ref_P.as_1d().select(mask) c = max_P.as_1d().select(mask) return flex.sum((c - I * p)**2 / (I * p)) def df(I): mask = flex.bool(flex.grid(9,9,9), False) for k in range(9): for j in range(9): for i in range(9): dx = 5 * (i - 4.5) / 4.5 dy = 5 * (j - 4.5) / 4.5 dz = 5 * (k - 4.5) / 4.5 dd = sqrt(dx**2 + dy**2 + dz**2) if dd <= 3: mask[k,j,i] = True mask = mask.as_1d() & (ref_P.as_1d() > 0) p = ref_P.as_1d().select(mask) c = max_P.as_1d().select(mask) b = 0 return flex.sum(p) - flex.sum(c*c / (I*I*p)) #return flex.sum(p - p*c*c / ((b + I*p)**2)) #return flex.sum(3*p*p + (c*c*p*p - 4*b*p*p) / ((b + I*p)**2)) #return flex.sum(p - c*c / (I*I*p)) #return flex.sum(p * (-c+p*I)*(c+p*I)/((p*I)**2)) def d2f(I): mask = flex.bool(flex.grid(9,9,9), False) for k in range(9): for j in range(9): for i in range(9): dx = 5 * (i - 4.5) / 4.5 dy = 5 * (j - 4.5) / 4.5 dz = 5 * (k - 4.5) / 4.5 dd = sqrt(dx**2 + dy**2 + dz**2) if dd <= 3: mask[k,j,i] = True mask = mask.as_1d() & (ref_P.as_1d() > 0) p = ref_P.as_1d().select(mask) c = max_P.as_1d().select(mask) return flex.sum(2*c*c*p*p / (p*I)**3) I = 10703#flex.sum(max_P) mask = ref_P.as_1d() > 0 p = ref_P.as_1d().select(mask) c = max_P.as_1d().select(mask) for i in range(10): I = I - df(I) / d2f(I) #v = I*p #I = flex.sum(c * p / v) / flex.sum(p*p / v) print I from math import log ff = [] for I in range(9500, 11500): ff.append(f(I)) print sorted(range(len(ff)), key=lambda x: ff[x])[0] + 9500 from matplotlib import pylab pylab.plot(range(9500,11500), ff) pylab.show() #exit(0) #I = 10000 #print flex.sum((max_P - I * ref_P)**2) / flex.sum(I * ref_P) #I = 10100 #print flex.sum((max_P - I * ref_P)**2) / flex.sum(I * ref_P) #exit(0) print flex.sum(self.reference[0]['rs_shoebox'].data) print I_cal[0] # Calculate the z score perc = self.mv3n_tolerance_interval(3*3) Z = (I_cal - I_sim) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f, sig: %f" % (Z_mean, Z_var, sqrt(Z_var)) print len(I_cal) from matplotlib import pylab from mpl_toolkits.mplot3d import Axes3D #fig = pylab.figure() #ax = fig.add_subplot(111, projection='3d') #ax.scatter(X_pos, Y_pos, P_cor) #pylab.scatter(X_pos, P_cor) #pylab.scatter(Y_pos, P_cor) #pylab.scatter(Z_pos, P_cor) #pylab.hist(P_cor,100) #pylab.scatter(P_cor, (I_cal - I_exp) / I_exp) pylab.hist(Z, 100) #pylab.hist(I_cal,100) #pylab.hist(I_cal - I_sim, 100) pylab.show()
def test_for_reflections(self, refl, filename): from dials.array_family import flex from dials.algorithms.statistics import \ kolmogorov_smirnov_test_standard_normal from os.path import basename print basename(filename) #refl = self.reference # Get the calculated background and simulated background B_sim = refl['background.sim.a'].as_double() I_sim = refl['intensity.sim'].as_double() I_exp = refl['intensity.exp'] # Set the background as simulated shoebox = refl['shoebox'] for i in range(len(shoebox)): bg = shoebox[i].background ms = shoebox[i].mask for j in range(len(bg)): bg[j] = B_sim[i] # Integrate integration = self.experiments[0].profile.fitting_class()( self.experiments[0]) old_size = len(refl) refl.extend(self.reference) integration.model(self.reference) integration.fit(refl) reference = refl[-len(self.reference):] refl = refl[:len(self.reference)] assert (len(refl) == old_size) assert (len(reference) == len(self.reference)) I_cal = refl['intensity.prf.value'] I_var = refl['intensity.prf.variance'] # Check the reference profiles and spots are ok #self.check_profiles(integration.learner) #self.check_reference(reference) #np = reference.locate().size() #for i in range(np): #profile = reference.locate().profile(i) #print "Profile: %d" % i #p = (profile.as_numpy_array() * 1000) #import numpy as np #p = p.astype(np.int) #print p # Only select variances greater than zero mask = refl.get_flags(refl.flags.integrated_prf) assert (mask.count(True) > 0) I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) mask = I_var > 0 I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) # Calculate the z score perc = self.mv3n_tolerance_interval(3 * 3) Z = (I_cal - I_sim) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f" % (Z_mean, Z_var) # Do the kolmogorov smirnov test D, p = kolmogorov_smirnov_test_standard_normal(Z) print "KS: D: %f, p-value: %f" % (D, p) # FIXME Z score should be a standard normal distribution. When background is # the main component, we do indeed see that the z score is in a standard # normal distribution. When the intensity dominates, the variance of the Z # scores decreases indicating that for increasing intensity of the signal, # the variance is over estimated. #assert(abs(Z_mean) <= 3 * Z_var) #from matplotlib import pylab #pylab.hist(Z, 20) #pylab.show() #Z_I = sorted(Z) ##n = int(0.05 * len(Z_I)) ##Z_I = Z_I[n:-n] ##mv = flex.mean_and_variance(flex.double(Z_I)) ##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation()) #edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))] #cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I] print 'OK'
def test_for_reflections(self, refl, filename): from dials.array_family import flex from dials.algorithms.statistics import \ kolmogorov_smirnov_test_standard_normal from os.path import basename print basename(filename) #refl = self.reference # Get the calculated background and simulated background B_sim = refl['background.sim.a'].as_double() I_sim = refl['intensity.sim'].as_double() I_exp = refl['intensity.exp'] # Set the background as simulated shoebox = refl['shoebox'] for i in range(len(shoebox)): bg = shoebox[i].background ms = shoebox[i].mask for j in range(len(bg)): bg[j] = B_sim[i] # Integrate integration = self.experiments[0].profile.fitting_class()(self.experiments[0]) old_size = len(refl) refl.extend(self.reference) integration.model(self.reference) integration.fit(refl) reference = refl[-len(self.reference):] refl = refl[:len(self.reference)] assert(len(refl) == old_size) assert(len(reference) == len(self.reference)) I_cal = refl['intensity.prf.value'] I_var = refl['intensity.prf.variance'] # Check the reference profiles and spots are ok #self.check_profiles(integration.learner) #self.check_reference(reference) #np = reference.locate().size() #for i in range(np): #profile = reference.locate().profile(i) #print "Profile: %d" % i #p = (profile.as_numpy_array() * 1000) #import numpy as np #p = p.astype(np.int) #print p # Only select variances greater than zero mask = refl.get_flags(refl.flags.integrated_prf) assert(mask.count(True) > 0) I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) mask = I_var > 0 I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) # Calculate the z score perc = self.mv3n_tolerance_interval(3*3) Z = (I_cal - I_sim) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f" % (Z_mean, Z_var) # Do the kolmogorov smirnov test D, p = kolmogorov_smirnov_test_standard_normal(Z) print "KS: D: %f, p-value: %f" % (D, p) # FIXME Z score should be a standard normal distribution. When background is # the main component, we do indeed see that the z score is in a standard # normal distribution. When the intensity dominates, the variance of the Z # scores decreases indicating that for increasing intensity of the signal, # the variance is over estimated. #assert(abs(Z_mean) <= 3 * Z_var) #from matplotlib import pylab #pylab.hist(Z, 20) #pylab.show() #Z_I = sorted(Z) ##n = int(0.05 * len(Z_I)) ##Z_I = Z_I[n:-n] ##mv = flex.mean_and_variance(flex.double(Z_I)) ##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation()) #edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))] #cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I] print 'OK'
def from_reflections(self, experiment, reflections): """ Generate the required data from the reflections """ # Get the beam vector s0 = matrix.col(experiment.beam.get_s0()) # Get the reciprocal lattice vector s2_list = reflections["s2"] # Initialise the list of observed intensities and covariances ctot_list = flex.double(len(s2_list)) mobs_list = flex.vec2_double(len(s2_list)) Sobs_list = flex.double(flex.grid(len(s2_list), 4)) Bmean = matrix.sqr((0, 0, 0, 0)) logger.info("Computing observed covariance for %d reflections" % len(reflections)) s0_length = s0.length() assert len(experiment.detector) == 1 panel = experiment.detector[0] sbox = reflections["shoebox"] for r in range(len(reflections)): # Create the coordinate system cs = CoordinateSystem2d(s0, s2_list[r]) # Get data and compute total counts data = sbox[r].data mask = sbox[r].mask bgrd = sbox[r].background # Get array of vectors i0 = sbox[r].bbox[0] j0 = sbox[r].bbox[2] assert data.all()[0] == 1 X = flex.vec2_double(flex.grid(data.all()[1], data.all()[2])) ctot = 0 C = flex.double(X.accessor()) for j in range(data.all()[1]): for i in range(data.all()[2]): c = data[0, j, i] - bgrd[0, j, i] if mask[0, j, i] == 5 and c > 0: ctot += c ii = i + i0 jj = j + j0 s = panel.get_pixel_lab_coord((ii + 0.5, jj + 0.5)) s = matrix.col(s).normalize() * s0_length X[j, i] = cs.from_beam_vector(s) C[j, i] = c # Check we have a sensible number of counts assert ctot > 0, "BUG: strong spots should have more than 0 counts!" # Compute the mean vector xbar = matrix.col((0, 0)) for j in range(X.all()[0]): for i in range(X.all()[1]): x = matrix.col(X[j, i]) xbar += C[j, i] * x xbar /= ctot # Compute the covariance matrix Sobs = matrix.sqr((0, 0, 0, 0)) for j in range(X.all()[0]): for i in range(X.all()[1]): x = matrix.col(X[j, i]) Sobs += (x - xbar) * (x - xbar).transpose() * C[j, i] Sobs /= ctot # Compute the bias zero = matrix.col((0, 0)) Bias_sq = (xbar - zero) * (xbar - zero).transpose() Bmean += Bias_sq # Add to the lists ctot_list[r] = ctot mobs_list[r] = xbar Sobs_list[r, 0] = Sobs[0] Sobs_list[r, 1] = Sobs[1] Sobs_list[r, 2] = Sobs[2] Sobs_list[r, 3] = Sobs[3] # Print some information logger.info("I_min = %.2f, I_max = %.2f" % (flex.min(ctot_list), flex.max(ctot_list))) # Print the mean covariance Smean = matrix.sqr((0, 0, 0, 0)) for r in range(Sobs_list.all()[0]): Smean += matrix.sqr(tuple(Sobs_list[r:r + 1, :])) Smean /= Sobs_list.all()[0] Bmean /= len(reflections) logger.info("") logger.info("Mean observed covariance:") print_matrix(Smean) print_eigen_values_and_vectors_of_observed_covariance(Smean) logger.info("") logger.info("Mean observed bias^2:") print_matrix(Bmean) # Compute the distance from the Ewald sphere epsilon = flex.double(s0.length() - matrix.col(s).length() for s in reflections["s2"]) mv = flex.mean_and_variance(epsilon) logger.info("") logger.info("Mean distance from Ewald sphere: %.3g" % mv.mean()) logger.info("Variance in distance from Ewald sphere: %.3g" % mv.unweighted_sample_variance()) # Return the profile refiner data return ProfileRefinerData(s0, s2_list, ctot_list, mobs_list, Sobs_list)
def __init__(self,Ibase,Gbase,FSIM,curvatures=False,**kwargs): # For backward compatibility handle the case where phil is undefined if "params" in kwargs.keys(): self.params = kwargs["params"] else: from xfel.command_line.cxi_merge import master_phil phil = iotbx.phil.process_command_line(args=[], master_string=master_phil).show() self.params = phil.work.extract() self.params.levmar.parameter_flags.append("Bfactor") # default example refines Bfactor self.counter = 0 self.x = flex.double(list(Ibase) + list(Gbase)) self.N_I = len(Ibase) self.N_G = len(Gbase) self.N_raw_obs = FSIM.raw_obs.size() print "# structure factors:",self.N_I, "# frames:",self.N_G, "(Visited set; refined parameters)" step_threshold = self.params.levmar.termination.step_threshold objective_decrease_threshold = self.params.levmar.termination.objective_decrease_threshold if "Bfactor" in self.params.levmar.parameter_flags: self.x = self.x.concatenate(flex.double(len(Gbase),0.0)) if "Deff" in self.params.levmar.parameter_flags: D_values = flex.double([2.*e.crystal.domain_size for e in kwargs["experiments"]]) self.x = self.x.concatenate(D_values) if "Rxy" in self.params.levmar.parameter_flags: self.x = self.x.concatenate(flex.double(2*len(Gbase),0.0)) levenberg_helper = choice_as_helper_base(self.params.levmar.parameter_flags) self.helper = levenberg_helper(initial_estimates = self.x) self.helper.set_cpp_data(FSIM, self.N_I, self.N_G) if kwargs.has_key("experiments"): self.helper.set_wavelength([e.beam.get_wavelength() for e in kwargs["experiments"]]) self.helper.set_domain_size([2.*e.crystal.domain_size for e in kwargs["experiments"]])#ad hoc factor of 2 self.helper.set_Astar_matrix([e.crystal.get_A() for e in kwargs["experiments"]]) bitflags = choice_as_bitflag(self.params.levmar.parameter_flags) self.helper.set_parameter_flags(bitflags) self.helper.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations_encapsulated_eqns( non_linear_ls = self.helper, n_max_iterations = 5000, track_all=True, step_threshold = step_threshold, objective_decrease_threshold = objective_decrease_threshold ) if "Deff" in self.params.levmar.parameter_flags: newDeff = self.helper.x[self.N_I+self.N_G:] # XXX specific Dstats=flex.mean_and_variance(newDeff) print "Refined Deff mean & standard deviation:", print Dstats.mean(),Dstats.unweighted_sample_standard_deviation() if "Rxy" in self.params.levmar.parameter_flags: AX = self.helper.x[self.N_I+self.N_G:self.N_I+2*self.N_G] # XXX specific AY = self.helper.x[self.N_I+2*self.N_G:self.N_I+3*self.N_G] # XXX specific stats=flex.mean_and_variance(AX) print "Rx rotational increments in degrees: %8.6f +/- %8.6f"%( stats.mean(),stats.unweighted_sample_standard_deviation()) stats=flex.mean_and_variance(AY) print "Ry rotational increments in degrees: %8.6f +/- %8.6f"%( stats.mean(),stats.unweighted_sample_standard_deviation()) print "End of minimisation: Converged", self.helper.counter,"cycles" chi_squared = self.helper.objective() * 2. print "obj",chi_squared print "# of obs:",FSIM.raw_obs.size() dof = FSIM.raw_obs.size() - ( len(self.x) ) print "degrees of freedom =",dof print "chisq/dof: %7.3f"%(chi_squared / dof) print
def dI_derrorterms(self): refls = self.scaler.ISIGI ct = self.scaler.crystal_table # notation: dP1_dP2 is derivative of parameter 1 with respect to parameter 2. Here, # for example, is the derivative of rx wrt thetax drx_dthetax = flex.mat3_double() dry_dthetay = flex.mat3_double() s0hat = flex.vec3_double(len(refls), (0,0,-1)) ex = col((1,0,0)) ey = col((0,1,0)) # Compute derivatives sre = symmetrize_reduce_enlarge(self.scaler.params.target_space_group.group()) gstar_params = None gstar_derivatives = None for i in range(len(ct)): n_refl = ct['n_refl'][i] # Derivatives of rx/y wrt thetax/y come from cctbx drx_dthetax.extend(flex.mat3_double(n_refl, ex.axis_and_angle_as_r3_derivative_wrt_angle(ct['thetax'][i]))) dry_dthetay.extend(flex.mat3_double(n_refl, ey.axis_and_angle_as_r3_derivative_wrt_angle(ct['thetay'][i]))) # Derivatives of the B matrix wrt to the unit cell parameters also come from cctbx sre.set_orientation(orientation=ct['b_matrix'][i]) p = sre.forward_independent_parameters() dB_dp = sre.forward_gradients() if gstar_params is None: assert gstar_derivatives is None gstar_params = [flex.double() for j in range(len(p))] gstar_derivatives = [flex.mat3_double() for j in range(len(p))] assert len(p) == len(dB_dp) == len(gstar_params) == len(gstar_derivatives) for j in range(len(p)): gstar_params[j].extend(flex.double(n_refl, p[j])) gstar_derivatives[j].extend(flex.mat3_double(n_refl, tuple(dB_dp[j]))) # Compute the scalar terms used while computing derivatives self.r = r = self.compute_intensity_parameters() # Begin computing derivatives sigma_Iobs = refls['scaled_intensity']/refls['isigi'] dI_dIobs = 1/r['D'] def compute_dI_dp(dq_dp): """ Deriviatives of the scaled intensity I wrt to thetax, thetay and the unit cell parameters are computed the same, starting with the deriviatives of those parameters wrt to q """ dqlen_dp = r['q'].dot(dq_dp)/r['qlen'] dd_dp = -(1/(r['qlen']**2)) * dqlen_dp drs_dp = -(r['eta']/(2 * r['d']**2)) * dd_dp dslen_dp = r['s'].dot(dq_dp)/r['slen'] drhsq_dp = 2 * (r['slen'] - (1/r['wavelength'])) * dslen_dp dPn_dp = 2 * r['rs'] * drs_dp dPd_dp = 2 * ((r['rs'] * drs_dp) + drhsq_dp) dP_dp = ((r['p_d'] * dPn_dp)-(r['p_n'] * dPd_dp))/(r['p_d']**2) dI_dp = -(refls['iobs']/(r['partiality']**2 * r['G'] * r['eepsilon'])) * dP_dp return dI_dp # Derivatives wrt the unit cell parameters dI_dgstar = [] for j in range(len(gstar_params)): dI_dgstar.append(compute_dI_dp(r['ry'] * r['rx'] * r['u'] * gstar_derivatives[j] * r['h'])) # Derivatives wrt the crystal orientation dI_dthetax = compute_dI_dp(r['ry'] * drx_dthetax * r['u'] * r['b'] * r['h']) dI_dthetay = compute_dI_dp(dry_dthetay * r['rx'] * r['u'] * r['b'] * r['h']) # Derivatives wrt to the wavelength dthetah_dlambda = 1/(flex.sqrt(1 - ((r['wavelength']/(2 * r['d']))**2)) * 2 * r['d']) den_dlambda = flex.cos(r['thetah']) * dthetah_dlambda der_dlambda = ((r['wavelength'] * den_dlambda) - r['sinthetah'])/r['wavelength']**2 depsilon_dlambda = -16 * r['B'] * r['er'] * der_dlambda ds0_dlambda = s0hat*(-1/r['wavelength']**2) dslen_dlambda = r['s'].dot(ds0_dlambda)/r['slen'] drhsq_dlambda = 2*(r['slen']-(1/r['wavelength']))*(dslen_dlambda+(1/r['wavelength']**2)) dP_dlambda = -2*(r['p_n']/r['p_d']**2) * drhsq_dlambda dD_dlambda = (r['G'] * r['eepsilon'] * dP_dlambda) + (r['partiality'] * r['G'] * r['eepsilon'] * depsilon_dlambda) dI_dlambda = -(refls['iobs']/r['D']**2) * dD_dlambda # Derivatives wrt to the deff drs_deff = -1/(r['deff']**2) dPn_deff = 2 * r['rs'] * drs_deff dPd_deff = 2 * r['rs'] * drs_deff dP_deff = ((r['p_d'] * dPn_deff)-(r['p_n'] * dPd_deff))/(r['p_d']**2) dI_deff = -(refls['iobs']/(r['partiality']**2 * r['G'] * r['eepsilon'])) * dP_deff # Derivatives wrt to eta (unused for RS refinement) # drs_deta = 1/(2*r['d']) # dPn_deta = 2 * r['rs'] * drs_deta # dPd_deta = 2 * r['rs'] * drs_deta # dP_deta = ((r['p_d']*dPn_deta)-(r['p_n']*dPd_deta))/(r['p_d']**2) # dI_deta = -(refls['iobs']/(r['partiality']**2 * r['G'] * r['eepsilon'])) * dP_deta if self.verbose: # Show comparisons to finite differences n_cryst_params = sre.constraints.n_independent_params() print "Showing finite differences and derivatives for each parameter (first few reflections only)" for parameter_name, table, derivatives, delta, in zip(['iobs', 'thetax', 'thetay', 'wavelength', 'deff'] + ['c%d'%cp for cp in range(n_cryst_params)], [refls, ct, ct, ct, ct] + [ct]*n_cryst_params, [dI_dIobs, dI_dthetax, dI_dthetay, dI_dlambda, dI_deff] + dI_dgstar, [1e-7]*5 + [1e-11]*n_cryst_params): finite_g = self.finite_difference(parameter_name, table, delta) print parameter_name for refl_id in range(min(10, len(refls))): print "%d % 21.1f % 21.1f"%(refl_id, finite_g[refl_id], derivatives[refl_id]) stats = flex.mean_and_variance(finite_g-derivatives) stats_finite = flex.mean_and_variance(finite_g) percent = 0 if stats_finite.mean() == 0 else 100*stats.mean()/stats_finite.mean() print "Mean difference between finite and analytical: % 24.4f +/- % 24.4f (%8.3f%% of finite d.)"%( \ stats.mean(), stats.unweighted_sample_standard_deviation(), percent) print return [dI_dIobs, dI_dthetax, dI_dthetay, dI_dlambda, dI_deff] + dI_dgstar
def run(self): ''' Parse the options. ''' # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) self.params = params experiments = [wrapper.data for wrapper in params.input.experiments] reflections = [wrapper.data for wrapper in params.input.reflections] # Find all detector objects detectors = [] beams = [] for expts in experiments: detectors.extend(expts.detectors()) beams.extend(expts.beams()) # Verify inputs if len(detectors) != 2: print("Please provide two experiments for comparison") return # These lines exercise the iterate_detector_at_level and iterate_panels functions # for a detector with 4 hierarchy levels """ print "Testing iterate_detector_at_level" for level in range(4): print "iterating at level", level for panelg in iterate_detector_at_level(detectors[0].hierarchy(), 0, level): print panelg.get_name() print "Testing iterate_panels" for level in range(4): print "iterating at level", level for panelg in iterate_detector_at_level(detectors[0].hierarchy(), 0, level): for panel in iterate_panels(panelg): print panel.get_name() """ tmp = [] for refls in reflections: print("N reflections total:", len(refls)) sel = refls.get_flags(refls.flags.used_in_refinement) if sel.count(True) > 0: refls = refls.select(sel) print("N reflections used in refinement", len(refls)) print("Reporting only on those reflections used in refinement") refls['difference_vector_norms'] = ( refls['xyzcal.mm'] - refls['xyzobs.mm.value']).norms() tmp.append(refls) reflections = tmp s0 = col(flex.vec3_double([col(b.get_s0()) for b in beams]).mean()) # Compute a set of radial and transverse displacements for each reflection print("Setting up stats...") tmp_refls = [] for refls, expts in zip(reflections, experiments): tmp = flex.reflection_table() assert len(expts.detectors()) == 1 dect = expts.detectors()[0] # Need to construct a variety of vectors for panel_id, panel in enumerate(dect): panel_refls = refls.select(refls['panel'] == panel_id) bcl = flex.vec3_double() # Compute the beam center in lab space (a vector pointing from the origin to where the beam would intersect # the panel, if it did intersect the panel) for expt_id in set(panel_refls['id']): beam = expts[expt_id].beam s0_ = beam.get_s0() expt_refls = panel_refls.select( panel_refls['id'] == expt_id) beam_centre = panel.get_beam_centre_lab(s0_) bcl.extend(flex.vec3_double(len(expt_refls), beam_centre)) panel_refls['beam_centre_lab'] = bcl # Compute obs in lab space x, y, _ = panel_refls['xyzobs.mm.value'].parts() c = flex.vec2_double(x, y) panel_refls['obs_lab_coords'] = panel.get_lab_coord(c) # Compute deltaXY in panel space. This vector is relative to the panel origin x, y, _ = (panel_refls['xyzcal.mm'] - panel_refls['xyzobs.mm.value']).parts() # Convert deltaXY to lab space, subtracting off of the panel origin panel_refls['delta_lab_coords'] = panel.get_lab_coord( flex.vec2_double(x, y)) - panel.get_origin() tmp.extend(panel_refls) refls = tmp # The radial vector points from the center of the reflection to the beam center radial_vectors = (refls['obs_lab_coords'] - refls['beam_centre_lab']).each_normalize() # The transverse vector is orthogonal to the radial vector and the beam vector transverse_vectors = radial_vectors.cross( refls['beam_centre_lab']).each_normalize() # Compute the raidal and transverse components of each deltaXY refls['radial_displacements'] = refls['delta_lab_coords'].dot( radial_vectors) refls['transverse_displacements'] = refls['delta_lab_coords'].dot( transverse_vectors) tmp_refls.append(refls) reflections = tmp_refls # storage for plots refl_counts = {} # Data for all tables pg_bc_dists = flex.double() root1 = detectors[0].hierarchy() root2 = detectors[1].hierarchy() all_weights = flex.double() all_refls_count = flex.int() # Data for lab space table lab_table_data = [] lab_delta_table_data = [] all_lab_x = flex.double() all_lab_y = flex.double() all_lab_z = flex.double() pg_lab_x_sigmas = flex.double() pg_lab_y_sigmas = flex.double() pg_lab_z_sigmas = flex.double() all_rotX = flex.double() all_rotY = flex.double() all_rotZ = flex.double() pg_rotX_sigmas = flex.double() pg_rotY_sigmas = flex.double() pg_rotZ_sigmas = flex.double() all_delta_x = flex.double() all_delta_y = flex.double() all_delta_z = flex.double() all_delta_xy = flex.double() all_delta_xyz = flex.double() all_delta_r = flex.double() all_delta_t = flex.double() all_delta_norm = flex.double() if params.hierarchy_level > 0: # Data for local table local_table_data = [] local_delta_table_data = [] all_local_x = flex.double() all_local_y = flex.double() all_local_z = flex.double() pg_local_x_sigmas = flex.double() pg_local_y_sigmas = flex.double() pg_local_z_sigmas = flex.double() all_local_rotX = flex.double() all_local_rotY = flex.double() all_local_rotZ = flex.double() pg_local_rotX_sigmas = flex.double() pg_local_rotY_sigmas = flex.double() pg_local_rotZ_sigmas = flex.double() all_local_delta_x = flex.double() all_local_delta_y = flex.double() all_local_delta_z = flex.double() all_local_delta_xy = flex.double() all_local_delta_xyz = flex.double() # Data for RMSD table rmsds_table_data = [] for pg_id, (pg1, pg2) in enumerate( zip( iterate_detector_at_level(root1, 0, params.hierarchy_level), iterate_detector_at_level(root2, 0, params.hierarchy_level))): # Count up the number of reflections in this panel group pair for use as a weighting scheme total_refls = 0 pg1_refls = 0 pg2_refls = 0 for p1, p2 in zip(iterate_panels(pg1), iterate_panels(pg2)): r1 = len(reflections[0].select( reflections[0]['panel'] == id_from_name( detectors[0], p1.get_name()))) r2 = len(reflections[1].select( reflections[1]['panel'] == id_from_name( detectors[1], p2.get_name()))) total_refls += r1 + r2 pg1_refls += r1 pg2_refls += r2 if pg1_refls == 0 and pg2_refls == 0: print("No reflections on panel group", pg_id) continue all_refls_count.append(total_refls) all_weights.append(pg1_refls) all_weights.append(pg2_refls) assert pg1.get_name() == pg2.get_name() refl_counts[pg1.get_name()] = total_refls # Compute RMSDs row = ["%d" % pg_id] for pg, refls, det in zip([pg1, pg2], reflections, detectors): pg_refls = flex.reflection_table() for p in iterate_panels(pg): pg_refls.extend( refls.select( refls['panel'] == id_from_name(det, p.get_name()))) if len(pg_refls) == 0: rmsd = r_rmsd = t_rmsd = 0 else: rmsd = math.sqrt( flex.sum_sq(pg_refls['difference_vector_norms']) / len(pg_refls)) * 1000 r_rmsd = math.sqrt( flex.sum_sq(pg_refls['radial_displacements']) / len(pg_refls)) * 1000 t_rmsd = math.sqrt( flex.sum_sq(pg_refls['transverse_displacements']) / len(pg_refls)) * 1000 row.extend([ "%6.1f" % rmsd, "%6.1f" % r_rmsd, "%6.1f" % t_rmsd, "%8d" % len(pg_refls) ]) rmsds_table_data.append(row) dists = flex.double() lab_x = flex.double() lab_y = flex.double() lab_z = flex.double() rot_X = flex.double() rot_Y = flex.double() rot_Z = flex.double() for pg in [pg1, pg2]: bc = col(pg.get_beam_centre_lab(s0)) ori = get_center(pg) dists.append((ori - bc).length()) ori_lab = pg.get_origin() lab_x.append(ori_lab[0]) lab_y.append(ori_lab[1]) lab_z.append(ori_lab[2]) f = col(pg.get_fast_axis()) s = col(pg.get_slow_axis()) n = col(pg.get_normal()) basis = sqr( [f[0], s[0], n[0], f[1], s[1], n[1], f[2], s[2], n[2]]) rotX, rotY, rotZ = basis.r3_rotation_matrix_as_x_y_z_angles( deg=True) rot_X.append(rotX) rot_Y.append(rotY) rot_Z.append(rotZ) all_lab_x.extend(lab_x) all_lab_y.extend(lab_y) all_lab_z.extend(lab_z) all_rotX.extend(rot_X) all_rotY.extend(rot_Y) all_rotZ.extend(rot_Z) pg_weights = flex.double([pg1_refls, pg2_refls]) if 0 in pg_weights: dist_m = dist_s = 0 lx_m = lx_s = ly_m = ly_s = lz_m = lz_s = 0 lrx_m = lrx_s = lry_m = lry_s = lrz_m = lrz_s = 0 dx = dy = dz = dxy = dxyz = dr = dt = dnorm = 0 else: stats = flex.mean_and_variance(dists, pg_weights) dist_m = stats.mean() dist_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(lab_x, pg_weights) lx_m = stats.mean() lx_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(lab_y, pg_weights) ly_m = stats.mean() ly_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(lab_z, pg_weights) lz_m = stats.mean() lz_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(rot_X, pg_weights) lrx_m = stats.mean() lrx_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(rot_Y, pg_weights) lry_m = stats.mean() lry_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(rot_Z, pg_weights) lrz_m = stats.mean() lrz_s = stats.gsl_stats_wsd() dx = lab_x[0] - lab_x[1] dy = lab_y[0] - lab_y[1] dz = lab_z[0] - lab_z[1] dxy = math.sqrt(dx**2 + dy**2) dxyz = math.sqrt(dx**2 + dy**2 + dz**2) delta = col([lab_x[0], lab_y[0], lab_z[0]]) - col( [lab_x[1], lab_y[1], lab_z[1]]) pg1_center = get_center_lab(pg1).normalize() transverse = s0.cross(pg1_center).normalize() radial = transverse.cross(s0).normalize() dr = delta.dot(radial) dt = delta.dot(transverse) dnorm = col(pg1.get_normal()).angle(col(pg2.get_normal()), deg=True) pg_bc_dists.append(dist_m) pg_lab_x_sigmas.append(lx_s) pg_lab_y_sigmas.append(ly_s) pg_lab_z_sigmas.append(lz_s) pg_rotX_sigmas.append(lrx_s) pg_rotY_sigmas.append(lry_s) pg_rotZ_sigmas.append(lrz_s) all_delta_x.append(dx) all_delta_y.append(dy) all_delta_z.append(dz) all_delta_xy.append(dxy) all_delta_xyz.append(dxyz) all_delta_r.append(dr) all_delta_t.append(dt) all_delta_norm.append(dnorm) lab_table_data.append([ "%d" % pg_id, "%5.1f" % dist_m, "%9.3f" % lx_m, "%9.3f" % lx_s, "%9.3f" % ly_m, "%9.3f" % ly_s, "%9.3f" % lz_m, "%9.3f" % lz_s, "%9.3f" % lrx_m, "%9.3f" % lrx_s, "%9.3f" % lry_m, "%9.3f" % lry_s, "%9.3f" % lrz_m, "%9.3f" % lrz_s, "%6d" % total_refls ]) lab_delta_table_data.append([ "%d" % pg_id, "%5.1f" % dist_m, "%9.1f" % (dx * 1000), "%9.1f" % (dy * 1000), "%9.3f" % dz, "%9.1f" % (dxy * 1000), "%9.3f" % dxyz, "%9.1f" % (dr * 1000), "%9.1f" % (dt * 1000), "%9.3f" % dnorm, "%6d" % total_refls ]) if params.hierarchy_level > 0: local_x = flex.double() local_y = flex.double() local_z = flex.double() l_rot_X = flex.double() l_rot_Y = flex.double() l_rot_Z = flex.double() l_dx = flex.double() l_dy = flex.double() l_dz = flex.double() l_dxy = flex.double() l_dxyz = flex.double() for pg in [pg1, pg2]: l_ori = pg.get_local_origin() local_x.append(l_ori[0]) local_y.append(l_ori[1]) local_z.append(l_ori[2]) f = col(pg.get_local_fast_axis()) s = col(pg.get_local_slow_axis()) n = f.cross(s) basis = sqr( [f[0], s[0], n[0], f[1], s[1], n[1], f[2], s[2], n[2]]) rotX, rotY, rotZ = basis.r3_rotation_matrix_as_x_y_z_angles( deg=True) l_rot_X.append(rotX) l_rot_Y.append(rotY) l_rot_Z.append(rotZ) all_local_x.extend(local_x) all_local_y.extend(local_y) all_local_z.extend(local_z) all_local_rotX.extend(l_rot_X) all_local_rotY.extend(l_rot_Y) all_local_rotZ.extend(l_rot_Z) pg_weights = flex.double([pg1_refls, pg2_refls]) if 0 in pg_weights: lx_m = lx_s = ly_m = ly_s = lz_m = lz_s = 0 lrx_m = lrx_s = lry_m = lry_s = lrz_m = lrz_s = 0 ldx = ldy = ldz = ldxy = ldxyz = 0 else: stats = flex.mean_and_variance(local_x, pg_weights) lx_m = stats.mean() lx_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(local_y, pg_weights) ly_m = stats.mean() ly_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(local_z, pg_weights) lz_m = stats.mean() lz_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(l_rot_X, pg_weights) lrx_m = stats.mean() lrx_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(l_rot_Y, pg_weights) lry_m = stats.mean() lry_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(l_rot_Z, pg_weights) lrz_m = stats.mean() lrz_s = stats.gsl_stats_wsd() ldx = local_x[0] - local_x[1] ldy = local_y[0] - local_y[1] ldz = local_z[0] - local_z[1] ldxy = math.sqrt(ldx**2 + ldy**2) ldxyz = math.sqrt(ldx**2 + ldy**2 + ldz**2) pg_local_x_sigmas.append(lx_s) pg_local_y_sigmas.append(ly_s) pg_local_z_sigmas.append(lz_s) pg_local_rotX_sigmas.append(lrx_s) pg_local_rotY_sigmas.append(lry_s) pg_local_rotZ_sigmas.append(lrz_s) all_local_delta_x.append(ldx) all_local_delta_y.append(ldy) all_local_delta_z.append(ldz) all_local_delta_xy.append(ldxy) all_local_delta_xyz.append(ldxyz) local_table_data.append([ "%d" % pg_id, "%5.1f" % dist_m, "%9.3f" % lx_m, "%9.3f" % lx_s, "%9.3f" % ly_m, "%9.3f" % ly_s, "%9.3f" % lz_m, "%9.3f" % lz_s, "%9.3f" % lrx_m, "%9.3f" % lrx_s, "%9.3f" % lry_m, "%9.3f" % lry_s, "%9.3f" % lrz_m, "%9.3f" % lrz_s, "%6d" % total_refls ]) local_delta_table_data.append([ "%d" % pg_id, "%5.1f" % dist_m, "%9.1f" % (ldx * 1000), "%9.1f" % (ldy * 1000), "%9.3f" % ldz, "%9.1f" % (ldxy * 1000), "%9.3f" % ldxyz, "%6d" % total_refls ]) # Set up table output, starting with lab table table_d = {d: row for d, row in zip(pg_bc_dists, lab_table_data)} table_header = [ "PanelG", "Radial", "Lab X", "Lab X", "Lab Y", "Lab Y", "Lab Z", "Lab Z", "Rot X", "Rot X", "Rot Y", "Rot Y", "Rot Z", "Rot Z", "N" ] table_header2 = [ "Id", "Dist", "", "Sigma", "", "Sigma", "", "Sigma", "", "Sigma", "", "Sigma", "", "Sigma", "Refls" ] table_header3 = [ "", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)", "(deg)", "(deg)", "(deg)", "(deg)", "(deg)", "(deg)", "" ] lab_table_data = [table_header, table_header2, table_header3] lab_table_data.extend([table_d[key] for key in sorted(table_d)]) if len(all_weights) > 1: r1 = ["All"] r2 = ["Mean"] for data, weights, fmt in [ [None, None, None], [all_lab_x, all_weights.as_double(), "%9.3f"], [pg_lab_x_sigmas, all_refls_count.as_double(), "%9.3f"], [all_lab_y, all_weights.as_double(), "%9.3f"], [pg_lab_y_sigmas, all_refls_count.as_double(), "%9.3f"], [all_lab_z, all_weights.as_double(), "%9.3f"], [pg_lab_z_sigmas, all_refls_count.as_double(), "%9.3f"], [all_rotX, all_weights.as_double(), "%9.3f"], [pg_rotX_sigmas, all_refls_count.as_double(), "%9.3f"], [all_rotY, all_weights.as_double(), "%9.3f"], [pg_rotY_sigmas, all_refls_count.as_double(), "%9.3f"], [all_rotZ, all_weights.as_double(), "%9.3f"], [pg_rotZ_sigmas, all_refls_count.as_double(), "%9.3f"] ]: r2.append("") if data is None and weights is None: r1.append("") continue stats = flex.mean_and_variance(data, weights) r1.append(fmt % stats.mean()) r1.append("") r2.append("%6.1f" % flex.mean(all_refls_count.as_double())) lab_table_data.append(r1) lab_table_data.append(r2) from libtbx import table_utils print("Detector statistics relative to lab origin") print( table_utils.format(lab_table_data, has_header=3, justify='center', delim=" ")) print( "PanelG Id: panel group id or panel id, depending on hierarchy_level. For each panel group, weighted means and weighted standard deviations (Sigmas) for the properties listed below are computed using the matching panel groups between the input experiments." ) print( "Radial dist: distance from center of panel group to the beam center" ) print("Lab X, Y and Z: mean coordinate in lab space") print( "Rot X, Y and Z: rotation of panel group around lab X, Y and Z axes" ) print( "N refls: number of reflections summed between both matching panel groups. This number is used as a weight when computing means and standard deviations." ) print("All: weighted mean of the values shown") print() # Next, deltas in lab space table_d = {d: row for d, row in zip(pg_bc_dists, lab_delta_table_data)} table_header = [ "PanelG", "Radial", "Lab dX", "Lab dY", "Lab dZ", "Lab dXY", "Lab dXYZ", "Lab dR", "Lab dT", "Lab dNorm", "N" ] table_header2 = ["Id", "Dist", "", "", "", "", "", "", "", "", "Refls"] table_header3 = [ "", "(mm)", "(microns)", "(microns)", "(mm)", "(microns)", "(mm)", "(microns)", "(microns)", "(deg)", "" ] lab_delta_table_data = [table_header, table_header2, table_header3] lab_delta_table_data.extend([table_d[key] for key in sorted(table_d)]) if len(all_weights) > 1: r1 = ["WMean"] r2 = ["WStddev"] r3 = ["Mean"] for data, weights, fmt in [ [None, None, None], [all_delta_x * 1000, all_refls_count.as_double(), "%9.1f"], [all_delta_y * 1000, all_refls_count.as_double(), "%9.1f"], [all_delta_z, all_refls_count.as_double(), "%9.3f"], [all_delta_xy * 1000, all_refls_count.as_double(), "%9.1f"], [all_delta_xyz, all_refls_count.as_double(), "%9.3f"], [all_delta_r * 1000, all_refls_count.as_double(), "%9.1f"], [all_delta_t * 1000, all_refls_count.as_double(), "%9.1f"], [all_delta_norm, all_refls_count.as_double(), "%9.3f"] ]: r3.append("") if data is None and weights is None: r1.append("") r2.append("") continue stats = flex.mean_and_variance(data, weights) r1.append(fmt % stats.mean()) if len(data) > 1: r2.append(fmt % stats.gsl_stats_wsd()) else: r2.append("-") r1.append("") r2.append("") r3.append("%6.1f" % flex.mean(all_refls_count.as_double())) lab_delta_table_data.append(r1) lab_delta_table_data.append(r2) lab_delta_table_data.append(r3) print("Detector deltas in lab space") print( table_utils.format(lab_delta_table_data, has_header=3, justify='center', delim=" ")) print( "PanelG Id: panel group id or panel id, depending on hierarchy_level. For each panel group, weighted means and weighted standard deviations (Sigmas) for the properties listed below are computed using the matching panel groups between the input experiments." ) print( "Radial dist: distance from center of panel group to the beam center" ) print( "Lab dX, dY and dZ: delta between X, Y and Z coordinates in lab space" ) print( "Lab dR, dT and dZ: radial and transverse components of dXY in lab space" ) print("Lab dNorm: angle between normal vectors in lab space") print( "N refls: number of reflections summed between both matching panel groups. This number is used as a weight when computing means and standard deviations." ) print("WMean: weighted mean of the values shown") print("WStddev: weighted standard deviation of the values shown") print("Mean: mean of the values shown") print() if params.hierarchy_level > 0: # Local table table_d = {d: row for d, row in zip(pg_bc_dists, local_table_data)} table_header = [ "PanelG", "Radial", "Local X", "Local X", "Local Y", "Local Y", "Local Z", "Local Z", "Rot X", "Rot X", "Rot Y", "Rot Y", "Rot Z", "Rot Z", "N" ] table_header2 = [ "Id", "Dist", "", "Sigma", "", "Sigma", "", "Sigma", "", "Sigma", "", "Sigma", "", "Sigma", "Refls" ] table_header3 = [ "", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)", "(mm)", "(deg)", "(deg)", "(deg)", "(deg)", "(deg)", "(deg)", "" ] local_table_data = [table_header, table_header2, table_header3] local_table_data.extend([table_d[key] for key in sorted(table_d)]) if len(all_weights) > 1: r1 = ["All"] r2 = ["Mean"] for data, weights, fmt in [ [None, None, None], [all_local_x, all_weights.as_double(), "%9.3f"], [pg_local_x_sigmas, all_refls_count.as_double(), "%9.3f"], [all_local_y, all_weights.as_double(), "%9.3f"], [pg_local_y_sigmas, all_refls_count.as_double(), "%9.3f"], [all_local_z, all_weights.as_double(), "%9.3f"], [pg_local_z_sigmas, all_refls_count.as_double(), "%9.3f"], [all_local_rotX, all_weights.as_double(), "%9.3f"], [ pg_local_rotX_sigmas, all_refls_count.as_double(), "%9.3f" ], [all_local_rotY, all_weights.as_double(), "%9.3f"], [ pg_local_rotY_sigmas, all_refls_count.as_double(), "%9.3f" ], [all_local_rotZ, all_weights.as_double(), "%9.3f"], [ pg_local_rotZ_sigmas, all_refls_count.as_double(), "%9.3f" ] ]: r2.append("") if data is None and weights is None: r1.append("") continue stats = flex.mean_and_variance(data, weights) r1.append(fmt % stats.mean()) r1.append("") r2.append("%6.1f" % flex.mean(all_refls_count.as_double())) local_table_data.append(r1) local_table_data.append(r2) print("Detector statistics in local frame of each panel group") print( table_utils.format(local_table_data, has_header=3, justify='center', delim=" ")) print( "PanelG Id: panel group id or panel id, depending on hierarchy_level. For each panel group, weighted means and weighted standard deviations (Sigmas) for the properties listed below are computed using the matching panel groups between the input experiments." ) print( "Radial dist: distance from center of panel group to the beam center" ) print( "Lab X, Y and Z: mean coordinate in relative to parent panel group" ) print( "Rot X, Y and Z: rotation of panel group around parent panel group X, Y and Z axes" ) print( "N refls: number of reflections summed between both matching panel groups. This number is used as a weight when computing means and standard deviations." ) print("All: weighted mean of the values shown") print() # Next, deltas in local space table_d = { d: row for d, row in zip(pg_bc_dists, local_delta_table_data) } table_header = [ "PanelG", "Radial", "Local dX", "Local dY", "Local dZ", "Local dXY", "Local dXYZ", "N" ] table_header2 = ["Id", "Dist", "", "", "", "", "", "Refls"] table_header3 = [ "", "(mm)", "(microns)", "(microns)", "(mm)", "(microns)", "(mm)", "" ] local_delta_table_data = [ table_header, table_header2, table_header3 ] local_delta_table_data.extend( [table_d[key] for key in sorted(table_d)]) if len(all_weights) > 1: r1 = ["WMean"] r2 = ["WStddev"] r3 = ["Mean"] for data, weights, fmt in [ [None, None, None], [ all_local_delta_x * 1000, all_refls_count.as_double(), "%9.1f" ], [ all_local_delta_y * 1000, all_refls_count.as_double(), "%9.1f" ], [all_local_delta_z, all_refls_count.as_double(), "%9.3f"], [ all_local_delta_xy * 1000, all_refls_count.as_double(), "%9.1f" ], [ all_local_delta_xyz, all_refls_count.as_double(), "%9.3f" ] ]: r3.append("") if data is None and weights is None: r1.append("") r2.append("") continue stats = flex.mean_and_variance(data, weights) r1.append(fmt % stats.mean()) r2.append(fmt % stats.gsl_stats_wsd()) r1.append("") r2.append("") r3.append("%6.1f" % flex.mean(all_refls_count.as_double())) local_delta_table_data.append(r1) local_delta_table_data.append(r2) local_delta_table_data.append(r3) print("Detector deltas relative to panel group origin") print( table_utils.format(local_delta_table_data, has_header=3, justify='center', delim=" ")) print( "PanelG Id: panel group id or panel id, depending on hierarchy_level. For each panel group, weighted means and weighted standard deviations (Sigmas) for the properties listed below are computed using the matching panel groups between the input experiments." ) print( "Radial dist: distance from center of panel group to the beam center" ) print( "Local dX, dY and dZ: delta between X, Y and Z coordinates in the local frame of the panel group" ) print( "N refls: number of reflections summed between both matching panel groups. This number is used as a weight when computing means and standard deviations." ) print("All: weighted mean of the values shown") print() #RMSD table table_d = {d: row for d, row in zip(pg_bc_dists, rmsds_table_data)} table_header = ["PanelG"] table_header2 = ["Id"] table_header3 = [""] for i in range(len(detectors)): table_header.extend(["D%d" % i] * 4) table_header2.extend(["RMSD", "rRMSD", "tRMSD", "N refls"]) table_header3.extend(["(microns)"] * 3) table_header3.append("") rmsds_table_data = [table_header, table_header2, table_header3] rmsds_table_data.extend([table_d[key] for key in sorted(table_d)]) row = ["Overall"] for refls in reflections: row.append("%6.1f" % (math.sqrt( flex.sum_sq(refls['difference_vector_norms']) / len(refls)) * 1000)) row.append("%6.1f" % (math.sqrt( flex.sum_sq(refls['radial_displacements']) / len(refls)) * 1000)) row.append("%6.1f" % (math.sqrt( flex.sum_sq(refls['transverse_displacements']) / len(refls)) * 1000)) row.append("%8d" % len(refls)) rmsds_table_data.append(row) print("RMSDs by detector number") print( table_utils.format(rmsds_table_data, has_header=3, justify='center', delim=" ")) print( "PanelG Id: panel group id or panel id, depending on hierarchy_level" ) print( "RMSD: root mean squared deviation between observed and predicted spot locations" ) print( "rRMSD: RMSD of radial components of the observed-predicted vectors" ) print( "tRMSD: RMSD of transverse components of the observed-predicted vectors" ) print("N refls: number of reflections") if params.tag is None: tag = "" else: tag = "%s " % params.tag if params.show_plots: # Plot the results self.detector_plot_dict(detectors[0], refl_counts, u"%sN reflections" % tag, u"%6d", show=False)
def intensities(self): """Compare the intensities.""" from dials.array_family import flex # Sort by resolution d = self.refl1["d"] index = flex.size_t(reversed(sorted(range(len(d)), key=lambda x: d[x]))) self.refl1.reorder(index) self.refl2.reorder(index) # Get the intensities I1 = self.refl1["intensity.sum.value"] I2 = self.refl2["intensity.sum.value"] # Compute chunked statistics corr = [] R = [] scale = [] res = [] for i in range(len(self.refl1) // 1000): # Get the chunks of data a = i * 1000 b = (i + 1) * 1000 II1 = I1[a:b] II2 = I2[a:b] res.append(d[a]) # Compute the mean and standard deviation per chunk mv1 = flex.mean_and_variance(II1) mv2 = flex.mean_and_variance(II2) m1 = mv1.mean() m2 = mv2.mean() s1 = mv1.unweighted_sample_standard_deviation() s2 = mv2.unweighted_sample_standard_deviation() # compute the correlation coefficient r = (1 / (len(II1) - 1)) * sum( ((II1[j] - m1) / s1) * ((II2[j] - m2) / s2) for j in range(len(II1))) corr.append(r) # Compute the scale between the chunks s = sum(II1) / sum(II2) scale.append(s) # Compute R between the chunks r = sum( abs(abs(II1[j]) - abs(s * II2[j])) for j in range(len(II1))) / sum( abs(II1[j]) for j in range(len(II1))) R.append(r) from matplotlib import pylab pylab.plot(corr, label="CC") pylab.plot(R, label="R") pylab.plot(scale, label="K") pylab.legend() pylab.show()
def per_sensor_analysis(self): # hardcoded Jungfrau 16M geometry for isensor in range(32): print ("Panel Sensor <Δx>(μm) <Δy>(μm) Nrefl RMS Δx(μm) RMS Δy(μm) ") if len(self.cumCALC[isensor]) <= 3: continue for ipanel in range(8*isensor, 8*(1+isensor)): if len(self.panel_deltax[ipanel])<2: continue Sx = flex.mean_and_variance(1000.*self.panel_deltax[ipanel]) Sy = flex.mean_and_variance(1000.*self.panel_deltay[ipanel]) RMSDx = 1000.*math.sqrt(flex.mean(self.panel_deltax[ipanel]*self.panel_deltax[ipanel])) RMSDy = 1000.*math.sqrt(flex.mean(self.panel_deltay[ipanel]*self.panel_deltay[ipanel])) print("%3d %3d"%(ipanel,ipanel//8),"%7.2f±%6.2f %7.2f±%6.2f %6d"%(Sx.mean(),Sx.unweighted_standard_error_of_mean(), Sy.mean(),Sy.unweighted_standard_error_of_mean(), len(self.panel_deltax[ipanel])), " %5.1f %5.1f"%(RMSDx,RMSDy), ) print("") cumD = (self.cumCALC[isensor]-self.cumOBS[isensor]).parts() print ( "All %3d %7.2f %7.2f %6d"%(isensor,1000.*flex.mean(cumD[0]), 1000.*flex.mean(cumD[1]), len(cumD[0]))) print("") # Now we'll do a linear least squares refinement over sensors: #Method 1. Simple rectilinear translation. if self.params.verbose: veclength = len(self.cumCALC[isensor]) correction = flex.vec3_double( veclength, (flex.mean(cumD[0]), flex.mean(cumD[1]), flex.mean(cumD[2])) ) new_delta = (self.cumCALC[isensor]-correction ) -self.cumOBS[isensor] for ipanel in range(8*isensor, 8*(1+isensor)): panel_delta = new_delta.select(self.cumPANNO[isensor]==ipanel) if len(panel_delta)<2: continue deltax_part, deltay_part = panel_delta.parts()[0:2] RMSDx = 1000.*math.sqrt( flex.mean(deltax_part * deltax_part) ) RMSDy = 1000.*math.sqrt( flex.mean(deltay_part * deltay_part) ) Sx = flex.mean_and_variance(1000.*deltax_part) Sy = flex.mean_and_variance(1000.*deltay_part) print("%3d %3d"%(ipanel,ipanel//8),"%7.2f±%6.2f %7.2f±%6.2f %6d"%(Sx.mean(),Sx.unweighted_standard_error_of_mean(), Sy.mean(),Sy.unweighted_standard_error_of_mean(), len(deltax_part)), " %5.1f %5.1f"%(RMSDx,RMSDy), ) print() # Method 2. Translation + rotation. src = [] dst = [] for icoord in range(len(self.cumCALC[isensor])): src.append(self.cumCALC[isensor][icoord][0:2]) dst.append(self.cumOBS[isensor][icoord][0:2]) src = np.array(src) dst = np.array(dst) # estimate affine transform model using all coordinates model = SimilarityTransform() model.estimate(src, dst) # robustly estimate affine transform model with RANSAC model_robust, inliers = ransac((src, dst), SimilarityTransform, min_samples=3, residual_threshold=2, max_trials=10) outliers = flex.bool(inliers == False) # compare "true" and estimated transform parameters if self.params.verbose: print("Similarity transform:") print("%2d"%isensor, "Scale: %.5f,"%(model.scale), "Translation(μm): (%7.2f,"%(1000.*model.translation[0]), "%7.2f),"%(1000.*model.translation[1]), "Rotation (°): %7.4f"%((180./math.pi)*model.rotation)) print("RANSAC:") print("%2d"%isensor, "Scale: %.5f,"%(model_robust.scale), "Translation(μm): (%7.2f,"%(1000.*model_robust.translation[0]), "%7.2f),"%(1000.*model_robust.translation[1]), "Rotation (°): %7.4f,"%((180./math.pi)*model_robust.rotation), "Outliers:",outliers.count(True) ) """from documentation: X = a0 * x - b0 * y + a1 = s * x * cos(rotation) - s * y * sin(rotation) + a1 Y = b0 * x + a0 * y + b1 = s * x * sin(rotation) + s * y * cos(rotation) + b1""" oldCALC = self.cumCALC[isensor].parts() ransacCALC = flex.vec3_double( (float(model_robust.scale) * oldCALC[0] * math.cos(model_robust.rotation) - float(model_robust.scale) * oldCALC[1] * math.sin(model_robust.rotation) + float(model_robust.translation[0])), (float(model_robust.scale) * oldCALC[0] * math.sin(model_robust.rotation) + float(model_robust.scale) * oldCALC[1] * math.cos(model_robust.rotation) + float(model_robust.translation[1])), oldCALC[2] ) new_delta = ransacCALC - self.cumOBS[isensor] inlier_delta = new_delta.select(~outliers) inlier_panno = self.cumPANNO[isensor].select(~outliers) for ipanel in range(8*isensor, 8*(1+isensor)): panel_delta = inlier_delta.select(inlier_panno==ipanel) if len(panel_delta)<2: continue deltax_part, deltay_part = panel_delta.parts()[0:2] RMSDx = 1000.*math.sqrt( flex.mean(deltax_part * deltax_part) ) RMSDy = 1000.*math.sqrt( flex.mean(deltay_part * deltay_part) ) Sx = flex.mean_and_variance(1000.*deltax_part) Sy = flex.mean_and_variance(1000.*deltay_part) print("%3d %3d"%(ipanel,ipanel//8),"%7.2f±%6.2f %7.2f±%6.2f %6d"%(Sx.mean(),Sx.unweighted_standard_error_of_mean(), Sy.mean(),Sy.unweighted_standard_error_of_mean(), len(deltax_part)), " %5.1f %5.1f"%(RMSDx,RMSDy), ) if self.params.verbose: print("") cumD = (inlier_delta).parts() print ( " %3d %7.2f %7.2f %6d\n"%(isensor,1000.*flex.mean(cumD[0]), 1000.*flex.mean(cumD[1]), len(cumD[0]))) print("----\n")
def test_for_reference(self): from dials.array_family import flex from math import sqrt, pi # Integrate integration = self.experiments[0].profile.fitting_class()( self.experiments[0]) # Integrate the reference profiles integration(self.experiments, self.reference) locator = integration.learner.locate() # Check the reference profiles and spots are ok #self.check_profiles(integration.learner) # Make sure background is zero profiles = self.reference['rs_shoebox'] eps = 1e-7 for p in profiles: assert (abs(flex.sum(p.background) - 0) < eps) print 'OK' # Only select variances greater than zero mask = self.reference.get_flags(self.reference.flags.integrated, all=False) assert (mask.count(True) > 0) I_cal = self.reference['intensity.prf.value'] I_var = self.reference['intensity.prf.variance'] B_sim = self.reference['background.sim.a'].as_double() I_sim = self.reference['intensity.sim'].as_double() I_exp = self.reference['intensity.exp'] P_cor = self.reference['profile.correlation'] X_pos, Y_pos, Z_pos = self.reference['xyzcal.px'].parts() I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) P_cor = P_cor.select(mask) max_ind = flex.max_index(flex.abs(I_cal - I_sim)) max_I = I_cal[max_ind] max_P = self.reference[max_ind]['rs_shoebox'].data max_C = self.reference[max_ind]['xyzcal.px'] max_S = self.reference[max_ind]['shoebox'].data ref_ind = locator.index(max_C) ref_P = locator.profile(ref_ind) ref_C = locator.coord(ref_ind) #def f(I): #mask = flex.bool(flex.grid(9,9,9), False) #for k in range(9): #for j in range(9): #for i in range(9): #dx = 5 * (i - 4.5) / 4.5 #dy = 5 * (j - 4.5) / 4.5 #dz = 5 * (k - 4.5) / 4.5 #dd = sqrt(dx**2 + dy**2 + dz**2) #if dd <= 3: #mask[k,j,i] = True #mask = mask.as_1d() & (ref_P.as_1d() > 0) #p = ref_P.as_1d().select(mask) #c = max_P.as_1d().select(mask) #return flex.sum((c - I * p)**2 / (I * p)) #ff = [] #for I in range(9500, 11500): #ff.append(f(I)) #print 'Old I: ', sorted(range(len(ff)), key=lambda x: ff[x])[0] + 9500 #from matplotlib import pylab #pylab.plot(range(9500, 11500), ff) #pylab.show() #def estI(I): #mask = flex.bool(flex.grid(9,9,9), False) #for k in range(9): #for j in range(9): #for i in range(9): #dx = 5 * (i - 4.5) / 4.5 #dy = 5 * (j - 4.5) / 4.5 #dz = 5 * (k - 4.5) / 4.5 #dd = sqrt(dx**2 + dy**2 + dz**2) #if dd <= 3: #mask[k,j,i] = True #mask = mask.as_1d() & (ref_P.as_1d() > 0) #p = ref_P.as_1d().select(mask) #c = max_P.as_1d().select(mask) #v = I * p #return flex.sum(c * p / v) / flex.sum(p*p/v) #def iterI(I0): #I = estI(I0) #print I #if abs(I - I0) < 1e-3: #return I #return iterI(I) #newI = iterI(10703)#flex.sum(max_P)) #print "New I: ", newI # Calculate the z score perc = self.mv3n_tolerance_interval(3 * 3) Z = (I_cal - I_sim) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f, sig: %f" % (Z_mean, Z_var, sqrt(Z_var))
def run(self, experiments, reflections): self.logger.log_step_time("SCALE_FRAMES") if self.params.scaling.algorithm != "mark0": # mark1 implies no scaling/post-refinement self.logger.log("No scaling was done") if self.mpi_helper.rank == 0: self.logger.main_log("No scaling was done") return experiments, reflections new_experiments = ExperimentList() new_reflections = flex.reflection_table() # scale experiments, one at a time. Reject experiments that do not correlate with the reference or fail to scale. results = [] slopes = [] correlations = [] high_res_experiments = 0 experiments_rejected_because_of_low_signal = 0 experiments_rejected_because_of_low_correlation_with_reference = 0 target_symm = symmetry( unit_cell=self.params.scaling.unit_cell, space_group_info=self.params.scaling.space_group) for experiment in experiments: exp_reflections = reflections.select( reflections['exp_id'] == experiment.identifier) # Build a miller array for the experiment reflections exp_miller_indices = miller.set( target_symm, exp_reflections['miller_index_asymmetric'], True) exp_intensities = miller.array( exp_miller_indices, exp_reflections['intensity.sum.value'], flex.double( flex.sqrt(exp_reflections['intensity.sum.variance']))) model_intensities = self.params.scaling.i_model # Extract an array of HKLs from the model to match the experiment HKLs matching_indices = miller.match_multi_indices( miller_indices_unique=model_intensities.indices(), miller_indices=exp_intensities.indices()) # Least squares if self.params.scaling.mark0.fit_reference_to_experiment: # RB: in cxi-merge we fit reference to experiment, but we should really do it the other way result = self.fit_reference_to_experiment( model_intensities, exp_intensities, matching_indices) else: result = self.fit_experiment_to_reference( model_intensities, exp_intensities, matching_indices) if result.error == scaling_result.err_low_signal: experiments_rejected_because_of_low_signal += 1 continue elif result.error == scaling_result.err_low_correlation: experiments_rejected_because_of_low_correlation_with_reference += 1 continue slopes.append(result.slope) correlations.append(result.correlation) if self.params.output.log_level == 0: self.logger.log( "Experiment ID: %s; Slope: %f; Correlation %f" % (experiment.identifier, result.slope, result.correlation)) # count high resolution experiments if exp_intensities.d_min() <= self.params.merging.d_min: high_res_experiments += 1 # apply scale factors if not self.params.postrefinement.enable: if self.params.scaling.mark0.fit_reference_to_experiment: exp_reflections['intensity.sum.value'] /= result.slope exp_reflections['intensity.sum.variance'] /= ( result.slope**2) else: exp_reflections['intensity.sum.value'] *= result.slope exp_reflections['intensity.sum.variance'] *= ( result.slope**2) new_experiments.append(experiment) new_reflections.extend(exp_reflections) rejected_experiments = len(experiments) - len(new_experiments) assert rejected_experiments == experiments_rejected_because_of_low_signal + \ experiments_rejected_because_of_low_correlation_with_reference reflections_removed_because_of_rejected_experiments = reflections.size( ) - new_reflections.size() self.logger.log("Experiments rejected because of low signal: %d" % experiments_rejected_because_of_low_signal) self.logger.log( "Experiments rejected because of low correlation with reference: %d" % experiments_rejected_because_of_low_correlation_with_reference) self.logger.log( "Reflections rejected because of rejected experiments: %d" % reflections_removed_because_of_rejected_experiments) self.logger.log("High resolution experiments: %d" % high_res_experiments) if self.params.postrefinement.enable: self.logger.log( "Note: scale factors were not applied, because postrefinement is enabled" ) # MPI-reduce all counts comm = self.mpi_helper.comm MPI = self.mpi_helper.MPI total_experiments_rejected_because_of_low_signal = comm.reduce( experiments_rejected_because_of_low_signal, MPI.SUM, 0) total_experiments_rejected_because_of_low_correlation_with_reference = comm.reduce( experiments_rejected_because_of_low_correlation_with_reference, MPI.SUM, 0) total_reflections_removed_because_of_rejected_experiments = comm.reduce( reflections_removed_because_of_rejected_experiments, MPI.SUM, 0) total_high_res_experiments = comm.reduce(high_res_experiments, MPI.SUM, 0) all_slopes = comm.reduce(slopes, MPI.SUM, 0) all_correlations = comm.reduce(correlations, MPI.SUM, 0) # rank 0: log data statistics if self.mpi_helper.rank == 0: self.logger.main_log( 'Experiments rejected because of low signal: %d' % total_experiments_rejected_because_of_low_signal) self.logger.main_log( 'Experiments rejected because of low correlation with reference: %d' % total_experiments_rejected_because_of_low_correlation_with_reference ) self.logger.main_log( 'Reflections rejected because of rejected experiments: %d' % total_reflections_removed_because_of_rejected_experiments) self.logger.main_log( 'Experiments with high resolution of %5.2f Angstrom or better: %d' % (self.params.merging.d_min, total_high_res_experiments)) if len(all_slopes) > 0: stats_slope = flex.mean_and_variance(flex.double(all_slopes)) self.logger.main_log( 'Average experiment scale factor wrt reference: %f' % (stats_slope.mean())) if len(all_correlations) > 0: stats_correlation = flex.mean_and_variance( flex.double(all_correlations)) self.logger.main_log( 'Average experiment correlation with reference: %f +/- %f' % (stats_correlation.mean(), stats_correlation.unweighted_sample_standard_deviation())) if self.params.postrefinement.enable: self.logger.main_log( "Note: scale factors were not applied, because postrefinement is enabled" ) self.logger.log_step_time("SCALE_FRAMES", True) return new_experiments, new_reflections
def test_for_reference(self): from dials.array_family import flex from math import sqrt, pi # Integrate integration = self.experiments[0].profile.fitting_class()(self.experiments[0]) # Integrate the reference profiles integration(self.experiments, self.reference) locator = integration.learner.locate() # Check the reference profiles and spots are ok #self.check_profiles(integration.learner) # Make sure background is zero profiles = self.reference['rs_shoebox'] eps = 1e-7 for p in profiles: assert(abs(flex.sum(p.background) - 0) < eps) print 'OK' # Only select variances greater than zero mask = self.reference.get_flags(self.reference.flags.integrated, all=False) assert(mask.count(True) > 0) I_cal = self.reference['intensity.prf.value'] I_var = self.reference['intensity.prf.variance'] B_sim = self.reference['background.sim.a'].as_double() I_sim = self.reference['intensity.sim'].as_double() I_exp = self.reference['intensity.exp'] P_cor = self.reference['profile.correlation'] X_pos, Y_pos, Z_pos = self.reference['xyzcal.px'].parts() I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) P_cor = P_cor.select(mask) max_ind = flex.max_index(flex.abs(I_cal-I_sim)) max_I = I_cal[max_ind] max_P = self.reference[max_ind]['rs_shoebox'].data max_C = self.reference[max_ind]['xyzcal.px'] max_S = self.reference[max_ind]['shoebox'].data ref_ind = locator.index(max_C) ref_P = locator.profile(ref_ind) ref_C = locator.coord(ref_ind) #def f(I): #mask = flex.bool(flex.grid(9,9,9), False) #for k in range(9): #for j in range(9): #for i in range(9): #dx = 5 * (i - 4.5) / 4.5 #dy = 5 * (j - 4.5) / 4.5 #dz = 5 * (k - 4.5) / 4.5 #dd = sqrt(dx**2 + dy**2 + dz**2) #if dd <= 3: #mask[k,j,i] = True #mask = mask.as_1d() & (ref_P.as_1d() > 0) #p = ref_P.as_1d().select(mask) #c = max_P.as_1d().select(mask) #return flex.sum((c - I * p)**2 / (I * p)) #ff = [] #for I in range(9500, 11500): #ff.append(f(I)) #print 'Old I: ', sorted(range(len(ff)), key=lambda x: ff[x])[0] + 9500 #from matplotlib import pylab #pylab.plot(range(9500, 11500), ff) #pylab.show() #def estI(I): #mask = flex.bool(flex.grid(9,9,9), False) #for k in range(9): #for j in range(9): #for i in range(9): #dx = 5 * (i - 4.5) / 4.5 #dy = 5 * (j - 4.5) / 4.5 #dz = 5 * (k - 4.5) / 4.5 #dd = sqrt(dx**2 + dy**2 + dz**2) #if dd <= 3: #mask[k,j,i] = True #mask = mask.as_1d() & (ref_P.as_1d() > 0) #p = ref_P.as_1d().select(mask) #c = max_P.as_1d().select(mask) #v = I * p #return flex.sum(c * p / v) / flex.sum(p*p/v) #def iterI(I0): #I = estI(I0) #print I #if abs(I - I0) < 1e-3: #return I #return iterI(I) #newI = iterI(10703)#flex.sum(max_P)) #print "New I: ", newI # Calculate the z score perc = self.mv3n_tolerance_interval(3*3) Z = (I_cal - I_sim) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f, sig: %f" % (Z_mean, Z_var, sqrt(Z_var))
def test_for_reflections(self, refl): from dials.algorithms.integration.sum import IntegrationAlgorithm from dials.array_family import flex from dials.algorithms.statistics import \ kolmogorov_smirnov_test_standard_normal # Get the calculated background and simulated background B_sim = refl['background.sim.a'].as_double() I_sim = refl['intensity.sim'].as_double() I_exp = refl['intensity.exp'] # Set the background as simulated shoebox = refl['shoebox'] for i in range(len(shoebox)): bg = shoebox[i].background ms = shoebox[i].mask for j in range(len(bg)): bg[j] = B_sim[i] # Integrate integration = IntegrationAlgorithm() integration(refl) I_cal = refl['intensity.sum.value'] I_var = refl['intensity.sum.variance'] # Only select variances greater than zero mask = I_var > 0 I_cal = I_cal.select(mask) I_var = I_var.select(mask) I_sim = I_sim.select(mask) I_exp = I_exp.select(mask) # Calculate the z score perc = self.mv3n_tolerance_interval(3*3) Z = (I_cal - I_exp) / flex.sqrt(I_var) mv = flex.mean_and_variance(Z) Z_mean = mv.mean() Z_var = mv.unweighted_sample_variance() print "Z: mean: %f, var: %f" % (Z_mean, Z_var) # Do the kolmogorov smirnov test D, p = kolmogorov_smirnov_test_standard_normal(Z) print "KS: D: %f, p-value: %f" % (D, p) # FIXME Z score should be a standard normal distribution. When background is # the main component, we do indeed see that the z score is in a standard # normal distribution. When the intensity dominates, the variance of the Z # scores decreases indicating that for increasing intensity of the signal, # the variance is over estimated. assert(abs(Z_mean) <= 3 * Z_var) #from matplotlib import pylab #pylab.hist(Z, 20) #pylab.show() #Z_I = sorted(Z) ##n = int(0.05 * len(Z_I)) ##Z_I = Z_I[n:-n] ##mv = flex.mean_and_variance(flex.double(Z_I)) ##print "Mean: %f, Sdev: %f" % (mv.mean(), mv.unweighted_sample_standard_deviation()) #edf = [float(i+1) / len(Z_I) for i in range(len(Z_I))] #cdf = [0.5 * (1.0 + erf(z / sqrt(2.0))) for z in Z_I] print 'OK'
def estimate_resolution_limit_distl_method1(reflections, imageset, ice_sel=None, plot_filename=None): # Implementation of Method 1 (section 2.4.4) of: # Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon # J. Appl. Cryst. (2006). 39, 112-119 # http://dx.doi.org/10.1107/S0021889805040677 if ice_sel is None: ice_sel = flex.bool(len(reflections), False) variances = reflections['intensity.sum.variance'] sel = variances > 0 intensities = reflections['intensity.sum.value'] variances = variances.select(sel) ice_sel = ice_sel.select(sel) reflections = reflections.select(sel) intensities = reflections['intensity.sum.value'] d_star_sq = flex.pow2(reflections['rlp'].norms()) d_spacings = uctbx.d_star_sq_as_d(d_star_sq) d_star_cubed = flex.pow(reflections['rlp'].norms(), 3) step = 2 while len(reflections) / step > 40: step += 1 order = flex.sort_permutation(d_spacings, reverse=True) ds3_subset = flex.double() d_subset = flex.double() for i in range(len(reflections) // step): ds3_subset.append(d_star_cubed[order[i * step]]) d_subset.append(d_spacings[order[i * step]]) x = flex.double(range(len(ds3_subset))) # (i) # Usually, Pm is the last point, that is, m = n. But m could be smaller than # n if an unusually high number of spots are detected around a certain # intermediate resolution. In that case, our search for the image resolution # does not go outside the spot 'bump;. This is particularly useful when # ice-rings are present. slopes = (ds3_subset[1:] - ds3_subset[0]) / (x[1:] - x[0]) skip_first = 3 p_m = flex.max_index(slopes[skip_first:]) + 1 + skip_first # (ii) from scitbx import matrix x1 = matrix.col((0, ds3_subset[0])) x2 = matrix.col((p_m, ds3_subset[p_m])) gaps = flex.double([0]) v = matrix.col(((x2[1] - x1[1]), -(x2[0] - x1[0]))).normalize() for i in range(1, p_m): x0 = matrix.col((i, ds3_subset[i])) r = x1 - x0 g = abs(v.dot(r)) gaps.append(g) mv = flex.mean_and_variance(gaps) s = mv.unweighted_sample_standard_deviation() # (iii) p_k = flex.max_index(gaps) g_k = gaps[p_k] p_g = p_k for i in range(p_k + 1, len(gaps)): g_i = gaps[i] if g_i > (g_k - 0.5 * s): p_g = i ds3_g = ds3_subset[p_g] d_g = d_subset[p_g] noisiness = 0 n = len(ds3_subset) for i in range(n - 1): for j in range(i + 1, n - 1): if slopes[i] >= slopes[j]: noisiness += 1 noisiness /= ((n - 1) * (n - 2) / 2) if plot_filename is not None: if pyplot is None: raise Sorry("matplotlib must be installed to generate a plot.") fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.scatter(range(len(ds3_subset)), ds3_subset) #ax.set_xlabel('') ax.set_ylabel('D^-3') xlim = pyplot.xlim() ylim = pyplot.ylim() ax.vlines(p_g, ylim[0], ylim[1], colors='red') pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.savefig(plot_filename) pyplot.close() return d_g, noisiness
def estimate_resolution_limit_distl_method1( reflections, imageset, ice_sel=None, plot_filename=None): # Implementation of Method 1 (section 2.4.4) of: # Z. Zhang, N. K. Sauter, H. van den Bedem, G. Snell and A. M. Deacon # J. Appl. Cryst. (2006). 39, 112-119 # http://dx.doi.org/10.1107/S0021889805040677 if ice_sel is None: ice_sel = flex.bool(len(reflections), False) variances = reflections['intensity.sum.variance'] sel = variances > 0 intensities = reflections['intensity.sum.value'] variances = variances.select(sel) ice_sel = ice_sel.select(sel) reflections = reflections.select(sel) intensities = reflections['intensity.sum.value'] d_star_sq = flex.pow2(reflections['rlp'].norms()) d_spacings = uctbx.d_star_sq_as_d(d_star_sq) d_star_cubed = flex.pow(reflections['rlp'].norms(), 3) step = 2 while len(reflections)/step > 40: step += 1 order = flex.sort_permutation(d_spacings, reverse=True) ds3_subset = flex.double() d_subset = flex.double() for i in range(len(reflections)//step): ds3_subset.append(d_star_cubed[order[i*step]]) d_subset.append(d_spacings[order[i*step]]) x = flex.double(range(len(ds3_subset))) # (i) # Usually, Pm is the last point, that is, m = n. But m could be smaller than # n if an unusually high number of spots are detected around a certain # intermediate resolution. In that case, our search for the image resolution # does not go outside the spot 'bump;. This is particularly useful when # ice-rings are present. slopes = (ds3_subset[1:] - ds3_subset[0])/(x[1:]-x[0]) skip_first = 3 p_m = flex.max_index(slopes[skip_first:]) + 1 + skip_first # (ii) from scitbx import matrix x1 = matrix.col((0, ds3_subset[0])) x2 = matrix.col((p_m, ds3_subset[p_m])) gaps = flex.double([0]) v = matrix.col(((x2[1] - x1[1]), -(x2[0] - x1[0]))).normalize() for i in range(1, p_m): x0 = matrix.col((i, ds3_subset[i])) r = x1 - x0 g = abs(v.dot(r)) gaps.append(g) mv = flex.mean_and_variance(gaps) s = mv.unweighted_sample_standard_deviation() # (iii) p_k = flex.max_index(gaps) g_k = gaps[p_k] p_g = p_k for i in range(p_k+1, len(gaps)): g_i = gaps[i] if g_i > (g_k - 0.5 * s): p_g = i ds3_g = ds3_subset[p_g] d_g = d_subset[p_g] noisiness = 0 n = len(ds3_subset) for i in range(n-1): for j in range(i+1, n-1): if slopes[i] >= slopes[j]: noisiness += 1 noisiness /= ((n-1)*(n-2)/2) if plot_filename is not None: if pyplot is None: raise Sorry("matplotlib must be installed to generate a plot.") fig = pyplot.figure() ax = fig.add_subplot(1,1,1) ax.scatter(range(len(ds3_subset)), ds3_subset) #ax.set_xlabel('') ax.set_ylabel('D^-3') xlim = pyplot.xlim() ylim = pyplot.ylim() ax.vlines(p_g, ylim[0], ylim[1], colors='red') pyplot.xlim(0, xlim[1]) pyplot.ylim(0, ylim[1]) pyplot.savefig(plot_filename) pyplot.close() return d_g, noisiness
def run(self): ''' Parse the options. ''' from dials.util.options import flatten_experiments, flatten_reflections # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) self.params = params experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) # Find all detector objects detectors = [] detectors.extend(experiments.detectors()) # Verify inputs if len(detectors) != 2: print("Please provide two experiments for comparison") return # These lines exercise the iterate_detector_at_level and iterate_panels functions # for a detector with 4 hierarchy levels """ print "Testing iterate_detector_at_level" for level in range(4): print "iterating at level", level for panelg in iterate_detector_at_level(detectors[0].hierarchy(), 0, level): print panelg.get_name() print "Testing iterate_panels" for level in range(4): print "iterating at level", level for panelg in iterate_detector_at_level(detectors[0].hierarchy(), 0, level): for panel in iterate_panels(panelg): print panel.get_name() """ tmp = [] for refls in reflections: print("N reflections total:", len(refls)) refls = refls.select(refls.get_flags(refls.flags.used_in_refinement)) print("N reflections used in refinement", len(refls)) print("Reporting only on those reflections used in refinement") refls['difference_vector_norms'] = (refls['xyzcal.mm']-refls['xyzobs.mm.value']).norms() tmp.append(refls) reflections = tmp # Iterate through the detectors, computing the congruence statistics delta_normals = {} z_angles = {} f_deltas = {} s_deltas = {} z_deltas = {} o_deltas = {} # overall z_offsets_d = {} refl_counts = {} all_delta_normals = flex.double() all_rdelta_normals = flex.double() all_tdelta_normals = flex.double() all_z_angles = flex.double() all_f_deltas = flex.double() all_s_deltas = flex.double() all_z_deltas = flex.double() all_deltas = flex.double() all_refls_count = flex.int() all_normal_angles = flex.double() all_rnormal_angles = flex.double() all_tnormal_angles = flex.double() pg_normal_angle_sigmas = flex.double() pg_rnormal_angle_sigmas = flex.double() pg_tnormal_angle_sigmas = flex.double() all_rot_z = flex.double() pg_rot_z_sigmas = flex.double() pg_bc_dists = flex.double() all_bc_dist = flex.double() all_f_offsets = flex.double() all_s_offsets = flex.double() all_z_offsets = flex.double() pg_f_offset_sigmas = flex.double() pg_s_offset_sigmas = flex.double() pg_z_offset_sigmas = flex.double() pg_offset_sigmas = flex.double() all_weights = flex.double() congruence_table_data = [] detector_table_data = [] rmsds_table_data = [] root1 = detectors[0].hierarchy() root2 = detectors[1].hierarchy() s0 = col(flex.vec3_double([col(b.get_s0()) for b in experiments.beams()]).mean()) # Compute a set of radial and transverse displacements for each reflection print("Setting up stats...") tmp_refls = [] for refls, expts in zip(reflections, [wrapper.data for wrapper in params.input.experiments]): tmp = flex.reflection_table() assert len(expts.detectors()) == 1 dect = expts.detectors()[0] # Need to construct a variety of vectors for panel_id, panel in enumerate(dect): panel_refls = refls.select(refls['panel'] == panel_id) bcl = flex.vec3_double() # Compute the beam center in lab space (a vector pointing from the origin to where the beam would intersect # the panel, if it did intersect the panel) for expt_id in set(panel_refls['id']): beam = expts[expt_id].beam s0 = beam.get_s0() expt_refls = panel_refls.select(panel_refls['id'] == expt_id) beam_centre = panel.get_beam_centre_lab(s0) bcl.extend(flex.vec3_double(len(expt_refls), beam_centre)) panel_refls['beam_centre_lab'] = bcl # Compute obs in lab space x, y, _ = panel_refls['xyzobs.mm.value'].parts() c = flex.vec2_double(x, y) panel_refls['obs_lab_coords'] = panel.get_lab_coord(c) # Compute deltaXY in panel space. This vector is relative to the panel origin x, y, _ = (panel_refls['xyzcal.mm'] - panel_refls['xyzobs.mm.value']).parts() # Convert deltaXY to lab space, subtracting off of the panel origin panel_refls['delta_lab_coords'] = panel.get_lab_coord(flex.vec2_double(x,y)) - panel.get_origin() tmp.extend(panel_refls) refls = tmp # The radial vector points from the center of the reflection to the beam center radial_vectors = (refls['obs_lab_coords'] - refls['beam_centre_lab']).each_normalize() # The transverse vector is orthogonal to the radial vector and the beam vector transverse_vectors = radial_vectors.cross(refls['beam_centre_lab']).each_normalize() # Compute the raidal and transverse components of each deltaXY refls['radial_displacements'] = refls['delta_lab_coords'].dot(radial_vectors) refls['transverse_displacements'] = refls['delta_lab_coords'].dot(transverse_vectors) tmp_refls.append(refls) reflections = tmp_refls for pg_id, (pg1, pg2) in enumerate(zip(iterate_detector_at_level(root1, 0, params.hierarchy_level), iterate_detector_at_level(root2, 0, params.hierarchy_level))): """ First compute statistics for detector congruence """ # Count up the number of reflections in this panel group pair for use as a weighting scheme total_refls = 0 pg1_refls = 0 pg2_refls = 0 for p1, p2 in zip(iterate_panels(pg1), iterate_panels(pg2)): r1 = len(reflections[0].select(reflections[0]['panel'] == id_from_name(detectors[0], p1.get_name()))) r2 = len(reflections[1].select(reflections[1]['panel'] == id_from_name(detectors[1], p2.get_name()))) total_refls += r1 + r2 pg1_refls += r1 pg2_refls += r2 if pg1_refls == 0 and pg2_refls == 0: print("No reflections on panel group", pg_id) continue assert pg1.get_name() == pg2.get_name() refl_counts[pg1.get_name()] = total_refls row = ["%d"%pg_id] for pg, refls, det in zip([pg1, pg2], reflections, detectors): pg_refls = flex.reflection_table() for p in iterate_panels(pg): pg_refls.extend(refls.select(refls['panel'] == id_from_name(det, p.get_name()))) if len(pg_refls) == 0: rmsd = r_rmsd = t_rmsd = 0 else: rmsd = math.sqrt(flex.sum_sq(pg_refls['difference_vector_norms'])/len(pg_refls))*1000 r_rmsd = math.sqrt(flex.sum_sq(pg_refls['radial_displacements'])/len(pg_refls))*1000 t_rmsd = math.sqrt(flex.sum_sq(pg_refls['transverse_displacements'])/len(pg_refls))*1000 row.extend(["%6.1f"%rmsd, "%6.1f"%r_rmsd, "%6.1f"%t_rmsd, "%8d"%len(pg_refls)]) rmsds_table_data.append(row) # Angle between normals of pg1 and pg2 delta_norm_angle = col(pg1.get_normal()).angle(col(pg2.get_normal()), deg=True) all_delta_normals.append(delta_norm_angle) # compute radial and transverse components of the delta between normal angles pgo = (get_center(pg1)+get_center(pg2))/2 ro = (get_center(root1)+get_center(root2))/2 rn = (col(root1.get_normal())+col(root2.get_normal()))/2 rf = (col(root1.get_fast_axis())+col(root2.get_fast_axis()))/2 rs = (col(root1.get_slow_axis())+col(root2.get_slow_axis()))/2 ro_pgo = pgo - ro # vector from the detector origin to the average panel group origin if ro_pgo.length() == 0: radial = col((0,0,0)) transverse = col((0,0,0)) else: radial = ((rf.dot(ro_pgo) * rf) + (rs.dot(ro_pgo) * rs)).normalize() # component of ro_pgo in rf rs plane transverse = rn.cross(radial).normalize() # now radial and transverse are vectors othogonal to each other and the detector normal, such that # radial points at the panel group origin # v1 and v2 are the components of pg 1 and 2 normals in the rn radial plane v1 = (radial.dot(col(pg1.get_normal())) * radial) + (rn.dot(col(pg1.get_normal())) * rn) v2 = (radial.dot(col(pg2.get_normal())) * radial) + (rn.dot(col(pg2.get_normal())) * rn) rdelta_norm_angle = v1.angle(v2, deg=True) if v1.cross(v2).dot(transverse) < 0: rdelta_norm_angle = -rdelta_norm_angle all_rdelta_normals.append(rdelta_norm_angle) # v1 and v2 are the components of pg 1 and 2 normals in the rn transverse plane v1 = (transverse.dot(col(pg1.get_normal())) * transverse) + (rn.dot(col(pg1.get_normal())) * rn) v2 = (transverse.dot(col(pg2.get_normal())) * transverse) + (rn.dot(col(pg2.get_normal())) * rn) tdelta_norm_angle = v1.angle(v2, deg=True) if v1.cross(v2).dot(radial) < 0: tdelta_norm_angle = -tdelta_norm_angle all_tdelta_normals.append(tdelta_norm_angle) # compute the angle between fast axes of these panel groups z_angle = col(pg1.get_fast_axis()[0:2]).angle(col(pg2.get_fast_axis()[0:2]), deg=True) all_z_angles.append(z_angle) z_angles[pg1.get_name()] = z_angle all_refls_count.append(total_refls) all_weights.append(pg1_refls) all_weights.append(pg2_refls) """ Now compute statistics measuring the reality of the detector. For example, instead of the distance between two things, we are concerned with the location of those things relative to laboratory space """ # Compute distances between panel groups and beam center # Also compute offset along Z axis dists = flex.double() f_offsets = flex.double() s_offsets = flex.double() z_offsets = flex.double() for pg, r in zip([pg1, pg2], [root1, root2]): bc = col(pg.get_beam_centre_lab(s0)) ori = get_center(pg) dists.append((ori-bc).length()) rori = col(r.get_origin()) delta_ori = ori-rori r_norm = col(r.get_normal()) r_fast = col(r.get_fast_axis()) r_slow = col(r.get_slow_axis()) f_offsets.append(r_fast.dot(delta_ori)*1000) s_offsets.append(r_slow.dot(delta_ori)*1000) z_offsets.append(r_norm.dot(delta_ori)*1000) fd = abs(f_offsets[0]-f_offsets[1]) sd = abs(s_offsets[0]-s_offsets[1]) zd = abs(z_offsets[0]-z_offsets[1]) od = math.sqrt(fd**2+sd**2+zd**2) f_deltas[pg1.get_name()] = fd s_deltas[pg1.get_name()] = sd z_deltas[pg1.get_name()] = zd o_deltas[pg1.get_name()] = od all_f_deltas.append(fd) all_s_deltas.append(sd) all_z_deltas.append(zd) all_deltas.append(od) all_f_offsets.extend(f_offsets) all_s_offsets.extend(s_offsets) all_z_offsets.extend(z_offsets) # Compute angle between detector normal and panel group normal # Compute rotation of panel group around detector normal pg_rotz = flex.double() norm_angles = flex.double() rnorm_angles = flex.double() tnorm_angles = flex.double() for pg, r in zip([pg1, pg2], [root1, root2]): pgo = get_center(pg) pgn = col(pg.get_normal()) pgf = col(pg.get_fast_axis()) ro = get_center(r) rn = col(r.get_normal()) rf = col(r.get_fast_axis()) rs = col(r.get_slow_axis()) norm_angle = rn.angle(pgn, deg=True) norm_angles.append(norm_angle) all_normal_angles.append(norm_angle) ro_pgo = pgo - ro # vector from the detector origin to the panel group origin if ro_pgo.length() == 0: radial = col((0,0,0)) transverse = col((0,0,0)) else: radial = ((rf.dot(ro_pgo) * rf) + (rs.dot(ro_pgo) * rs)).normalize() # component of ro_pgo in rf rs plane transverse = rn.cross(radial).normalize() # now radial and transverse are vectors othogonal to each other and the detector normal, such that # radial points at the panel group origin # v is the component of pgn in the rn radial plane v = (radial.dot(pgn) * radial) + (rn.dot(pgn) * rn) angle = rn.angle(v, deg=True) if rn.cross(v).dot(transverse) < 0: angle = -angle rnorm_angles.append(angle) all_rnormal_angles.append(angle) # v is the component of pgn in the rn transverse plane v = (transverse.dot(pgn) * transverse) + (rn.dot(pgn) * rn) angle = rn.angle(v, deg=True) if rn.cross(v).dot(radial) < 0: angle = -angle tnorm_angles.append(angle) all_tnormal_angles.append(angle) # v is the component of pgf in the rf rs plane v = (rf.dot(pgf) * rf) + (rs.dot(pgf) * rs) angle = rf.angle(v, deg=True) angle = angle-(round(angle/90)*90) # deviation from 90 degrees pg_rotz.append(angle) all_rot_z.append(angle) # Set up table rows using stats aggregated from above pg_weights = flex.double([pg1_refls, pg2_refls]) if 0 in pg_weights: dist_m = dist_s = norm_angle_m = norm_angle_s = rnorm_angle_m = rnorm_angle_s = 0 tnorm_angle_m = tnorm_angle_s = rotz_m = rotz_s = 0 fo_m = fo_s = so_m = so_s = zo_m = zo_s = o_s = 0 else: stats = flex.mean_and_variance(dists, pg_weights) dist_m = stats.mean() dist_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(norm_angles, pg_weights) norm_angle_m = stats.mean() norm_angle_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(rnorm_angles, pg_weights) rnorm_angle_m = stats.mean() rnorm_angle_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(tnorm_angles, pg_weights) tnorm_angle_m = stats.mean() tnorm_angle_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(pg_rotz, pg_weights) rotz_m = stats.mean() rotz_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(f_offsets, pg_weights) fo_m = stats.mean() fo_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(s_offsets, pg_weights) so_m = stats.mean() so_s = stats.gsl_stats_wsd() stats = flex.mean_and_variance(z_offsets, pg_weights) zo_m = stats.mean() zo_s = stats.gsl_stats_wsd() o_s = math.sqrt(fo_s**2+so_s**2+zo_s**2) pg_bc_dists.append(dist_m) all_bc_dist.extend(dists) pg_normal_angle_sigmas.append(norm_angle_s) pg_rnormal_angle_sigmas.append(rnorm_angle_s) pg_tnormal_angle_sigmas.append(tnorm_angle_s) pg_rot_z_sigmas.append(rotz_s) pg_f_offset_sigmas.append(fo_s) pg_s_offset_sigmas.append(so_s) pg_z_offset_sigmas.append(zo_s) pg_offset_sigmas.append(o_s) z_offsets_d[pg1.get_name()] = zo_m congruence_table_data.append(["%d"%pg_id, "%5.1f"%dist_m, #"%.4f"%dist_s, "%.4f"%delta_norm_angle, "%.4f"%rdelta_norm_angle, "%.4f"%tdelta_norm_angle, "%.4f"%z_angle, "%4.1f"%fd, "%4.1f"%sd, "%4.1f"%zd, "%4.1f"%od, "%6d"%total_refls]) detector_table_data.append(["%d"%pg_id, "%5.1f"%dist_m, #"%.4f"%dist_s, "%.4f"%norm_angle_m, "%.4f"%norm_angle_s, "%.4f"%rnorm_angle_m, "%.4f"%rnorm_angle_s, "%.4f"%tnorm_angle_m, "%.4f"%tnorm_angle_s, "%10.6f"%rotz_m, "%.6f"%rotz_s, #"%9.1f"%fo_m, "%5.3f"%fo_s, #"%9.1f"%so_m, "%5.3f"%so_s, "%9.3f"%fo_s, "%9.3f"%so_s, "%9.1f"%zo_m, "%9.1f"%zo_s, "%9.3f"%o_s, "%6d"%total_refls]) # Set up table output table_d = {d:row for d, row in zip(pg_bc_dists, congruence_table_data)} table_header = ["PanelG","Dist","Normal","RNormal","TNormal","Z rot","Delta","Delta","Delta","Delta","N"] table_header2 = ["Id","","Angle","Angle","Angle","Angle","F","S","Z","O","Refls"] table_header3 = ["", "(mm)","(mm)","(deg)","(deg)","(microns)","(microns)","(microns)","(microns)","(microns)",""] congruence_table_data = [table_header, table_header2, table_header3] congruence_table_data.extend([table_d[key] for key in sorted(table_d)]) table_d = {d:row for d, row in zip(pg_bc_dists, detector_table_data)} table_header = ["PanelG","Dist","Normal","Normal","RNormal","RNormal","TNormal","TNormal","RotZ", "RotZ","F Offset","S Offset","Z Offset","Z Offset","Offset","N"] table_header2 = ["Id","","","Sigma","","Sigma","","Sigma","","Sigma","Sigma","Sigma","","Sigma","Sigma","Refls"] table_header3 = ["", "(mm)","(deg)","(deg)","(deg)","(deg)","(deg)","(deg)","(deg)","(deg)","(microns)","(microns)","(microns)","(microns)","(microns)",""] detector_table_data = [table_header, table_header2, table_header3] detector_table_data.extend([table_d[key] for key in sorted(table_d)]) table_d = {d:row for d, row in zip(pg_bc_dists, rmsds_table_data)} table_header = ["PanelG"] table_header2 = ["Id"] table_header3 = [""] for i in range(len(detectors)): table_header.extend(["D%d"%i]*4) table_header2.extend(["RMSD", "rRMSD", "tRMSD", "N refls"]) table_header3.extend(["(microns)"]*3) table_header3.append("") rmsds_table_data = [table_header, table_header2, table_header3] rmsds_table_data.extend([table_d[key] for key in sorted(table_d)]) if len(all_refls_count) > 1: r1 = ["Weighted mean"] r2 = ["Weighted stddev"] r1.append("") r2.append("") #r1.append("") #r2.append("") stats = flex.mean_and_variance(all_delta_normals, all_refls_count.as_double()) r1.append("%.4f"%stats.mean()) r2.append("%.4f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(all_rdelta_normals, all_refls_count.as_double()) r1.append("%.4f"%stats.mean()) r2.append("%.4f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(all_tdelta_normals, all_refls_count.as_double()) r1.append("%.4f"%stats.mean()) r2.append("%.4f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(all_z_angles, all_refls_count.as_double()) r1.append("%.4f"%stats.mean()) r2.append("%.4f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(all_f_deltas, all_refls_count.as_double()) r1.append("%4.1f"%stats.mean()) r2.append("%4.1f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(all_s_deltas, all_refls_count.as_double()) r1.append("%4.1f"%stats.mean()) r2.append("%4.1f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(all_z_deltas, all_refls_count.as_double()) r1.append("%4.1f"%stats.mean()) r2.append("%4.1f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(all_deltas, all_refls_count.as_double()) r1.append("%4.1f"%stats.mean()) r2.append("%4.1f"%stats.gsl_stats_wsd()) r1.append("") r2.append("") congruence_table_data.append(r1) congruence_table_data.append(r2) congruence_table_data.append(["Mean", "", "", "","","", "", "", "", "", "", "%6.1f"%flex.mean(all_refls_count.as_double())]) from libtbx import table_utils print("Congruence statistics, I.E. the differences between the input detectors:") print(table_utils.format(congruence_table_data,has_header=3,justify='center',delim=" ")) print("PanelG Id: panel group id or panel id, depending on hierarchy_level. For each panel group, statistics are computed between the matching panel groups between the two input experiments.") print("Dist: distance from center of panel group to the beam center") print("Dist Sigma: weighted standard deviation of the measurements used to compute Dist") print("Normal angle: angle between the normal vectors of matching panel groups.") print("RNormal angle: radial component of the angle between the normal vectors of matching panel groups") print("TNormal angle: transverse component of the angle between the normal vectors of matching panel groups") print("Z rot: angle between the XY components of the fast axes of the panel groups.") print("Delta F: shift between matching panel groups along the detector fast axis.") print("Delta S: shift between matching panel groups along the detector slow axis.") print("Delta Z: Z shift between matching panel groups along the detector normal.") print("Delta O: Overall shift between matching panel groups along the detector normal.") print("N refls: number of reflections summed between both matching panel groups. This number is used as a weight when computing means and standard deviations.") print() print() if len(all_weights) > 1: r1 = ["All"] r2 = ["Mean"] for data, weights, fmt in [[None,None,None], #[None,None,None], [all_normal_angles, all_weights.as_double(), "%.4f"], [pg_normal_angle_sigmas, all_refls_count.as_double(), "%.4f"], [all_rnormal_angles, all_weights.as_double(), "%.4f"], [pg_rnormal_angle_sigmas, all_refls_count.as_double(), "%.4f"], [all_tnormal_angles, all_weights.as_double(), "%.4f"], [pg_tnormal_angle_sigmas, all_refls_count.as_double(), "%.4f"], [all_rot_z, all_weights.as_double(), "%10.6f"], [pg_rot_z_sigmas, all_refls_count.as_double(), "%.6f"], #[all_f_offsets, all_weights.as_double(), "%9.1f"], [pg_f_offset_sigmas, all_refls_count.as_double(), "%9.3f"], #[all_s_offsets, all_weights.as_double(), "%9.1f"], [pg_s_offset_sigmas, all_refls_count.as_double(), "%9.3f"], [all_z_offsets, all_weights.as_double(), "%9.1f"], [pg_z_offset_sigmas, all_refls_count.as_double(), "%9.1f"], [pg_offset_sigmas, all_refls_count.as_double(), "%9.1f"]]: r2.append("") if data is None and weights is None: r1.append("") continue stats = flex.mean_and_variance(data, weights) r1.append(fmt%stats.mean()) r1.append("") r2.append("%6.1f"%flex.mean(all_refls_count.as_double())) detector_table_data.append(r1) detector_table_data.append(r2) print("Detector statistics, I.E. measurements of parameters relative to the detector plane:") print(table_utils.format(detector_table_data,has_header=3,justify='center',delim=" ")) print("PanelG Id: panel group id or panel id, depending on hierarchy_level. For each panel group, weighted means and weighted standard deviations (Sigmas) for the properties listed below are computed using the matching panel groups between the input experiments.") print("Dist: distance from center of panel group to the beam center") print("Dist Sigma: weighted standard deviation of the measurements used to compute Dist") print("Normal Angle: angle between the normal vector of the detector at its root hierarchy level and the normal of the panel group") print("RNormal Angle: radial component of Normal Angle") print("TNormal Angle: transverse component of Normal Angle") print("RotZ: deviation from 90 degrees of the rotation of each panel group around the detector normal") print("F Offset: offset of panel group along the detector's fast axis") print("S Offset: offset of panel group along the detector's slow axis") print("Z Offset: offset of panel group along the detector normal") print("Offset: offset of panel group in F,S,Z space. Sigma is F, S, Z offset sigmas summed in quadrature.") print("N refls: number of reflections summed between both matching panel groups. This number is used as a weight when computing means and standard deviations.") print("All: weighted mean of the values shown") print() print("Sigmas in this table are computed using the standard deviation of 2 measurements (I.E. a panel's Z Offset is measured twice, once in each input dataset). This is related by a factor of sqrt(2)/2 to the mean of the Delta Z parameter in the congruence statistics table above, which is the difference between Z parameters.") print() row = ["Overall"] for refls in reflections: row.append("%6.1f"%(math.sqrt(flex.sum_sq(refls['difference_vector_norms'])/len(refls))*1000)) row.append("%6.1f"%(math.sqrt(flex.sum_sq(refls['radial_displacements'])/len(refls))*1000)) row.append("%6.1f"%(math.sqrt(flex.sum_sq(refls['transverse_displacements'])/len(refls))*1000)) row.append("%8d"%len(refls)) rmsds_table_data.append(row) print("RMSDs by detector number") print(table_utils.format(rmsds_table_data,has_header=3,justify='center',delim=" ")) print("PanelG Id: panel group id or panel id, depending on hierarchy_level") print("RMSD: root mean squared deviation between observed and predicted spot locations") print("rRMSD: RMSD of radial components of the observed-predicted vectors") print("tRMSD: RMSD of transverse components of the observed-predicted vectors") print("N refls: number of reflections") # Show stats for detector hierarchy root def _print_vector(v): for i in v: print("%10.5f"%i, end=' ') print() for d_id, d in enumerate(detectors): ori = d.hierarchy().get_origin() norm = d.hierarchy().get_normal() fast = d.hierarchy().get_fast_axis() slow = d.hierarchy().get_slow_axis() print("Detector", d_id, "origin: ", end=' '); _print_vector(ori) print("Detector", d_id, "normal: ", end=' '); _print_vector(norm) print("Detector", d_id, "fast axis:", end=' '); _print_vector(fast) print("Detector", d_id, "slow axis:", end=' '); _print_vector(slow) # Unit cell statstics lengths = flex.vec3_double() angles = flex.vec3_double() weights = flex.double() for refls, expts in zip(reflections, [d.data for d in params.input.experiments]): for crystal_id, crystal in enumerate(expts.crystals()): lengths.append(crystal.get_unit_cell().parameters()[0:3]) angles.append(crystal.get_unit_cell().parameters()[3:6]) weights.append(len(refls.select(refls['id'] == crystal_id))) print("Unit cell stats (angstroms and degrees), weighted means and standard deviations") for subset, tags in zip([lengths, angles], [["Cell a", "Cell b", "Cell c"],["Cell alpha", "Cell beta", "Cell gamma"]]): for data, tag in zip(subset.parts(), tags): stats = flex.mean_and_variance(data, weights) print("%s %5.1f +/- %6.3f"%(tag, stats.mean(), stats.gsl_stats_wsd())) if params.tag is None: tag = "" else: tag = "%s "%params.tag if params.show_plots: # Plot the results detector_plot_dict(self.params, detectors[0], refl_counts, u"%sN reflections"%tag, u"%6d", show=False) #detector_plot_dict(self.params, detectors[0], delta_normals, u"%sAngle between normal vectors (\N{DEGREE SIGN})"%tag, u"%.2f\N{DEGREE SIGN}", show=False) detector_plot_dict(self.params, detectors[0], z_angles, u"%sZ rotation angle between panels (\N{DEGREE SIGN})"%tag, u"%.2f\N{DEGREE SIGN}", show=False) detector_plot_dict(self.params, detectors[0], f_deltas, u"%sFast displacements between panels (microns)"%tag, u"%4.1f", show=False) detector_plot_dict(self.params, detectors[0], s_deltas, u"%sSlow displacements between panels (microns)"%tag, u"%4.1f", show=False) detector_plot_dict(self.params, detectors[0], z_offsets_d, u"%sZ offsets along detector normal (microns)"%tag, u"%4.1f", show=False) detector_plot_dict(self.params, detectors[0], z_deltas, u"%sZ displacements between panels (microns)"%tag, u"%4.1f", show=False) detector_plot_dict(self.params, detectors[0], o_deltas, u"%sOverall displacements between panels (microns)"%tag, u"%4.1f", show=False) plt.show()
def run(self): """Parse the options.""" # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) self.params = params assert (len(params.input.experiments) == len(params.input.reflections) == 1), "Provide one experiment list and one reflection table" assert params.method == "summed_intensity" experiments = params.input.experiments[0].data reflections = params.input.reflections[0].data # Find the aveage unit cell for the crystals in the experiments provided weighted_relfs = flex.reflection_table() all_uc = [flex.double() for i in xrange(6)] space_group = None for expt_id, experiment in enumerate(experiments): refls = reflections.select(reflections["id"] == expt_id) unit_cell = experiment.crystal.get_unit_cell() for i in xrange(6): all_uc[i].append(unit_cell.parameters()[i]) if space_group is None: space_group = experiment.crystal.get_space_group() else: assert (space_group.type().lookup_symbol() == experiment. crystal.get_space_group().type().lookup_symbol()) # Compute the average unit cell and build a miller array with it unit_cell = uctbx.unit_cell([flex.mean(data) for data in all_uc]) cs = crystal_symmetry(unit_cell, space_group.type().lookup_symbol()) ms = miller_set(cs, reflections["miller_index"], anomalous_flag=False) ma = ms.array(reflections["intensity.sum.value"] / flex.sqrt(reflections["intensity.sum.variance"])) ma.setup_binner(n_bins=10) binner = ma.binner() mean_i = flex.double() reflections["delpsical.weights"] = flex.double(len(reflections), 0) # Iterate through the bins and compute the Wilson plot, then use it compute the weights for i in binner.range_all(): sel = binner.selection(i) if sel.count(True) == 0: mean_i.append(0) continue mean_i.append( flex.mean(reflections["intensity.sum.value"].select(sel))) reflections["delpsical.weights"].set_selected( sel, reflections["intensity.sum.value"].select(sel) * (params.summed_intensity.scale_factor / mean_i[i]), ) if params.show_weight_plots: fig = plt.figure() plt.title(str(i)) plt.hist(reflections["delpsical.weights"].select(sel)) # Show unit cell distribution and mean I print("Average uc +/- std. deviation") labels = [ "% 6s" % l for l in ["a", "b", "c", "alpha", "beta", "gamma"] ] for label, data in zip(labels, all_uc): stats = flex.mean_and_variance(data) print("%s % 6.1f +/- %6.1f" % (label, stats.mean(), stats.unweighted_sample_standard_deviation())) print("Mean I over all data") binner.show_data(mean_i, data_fmt="%.1f", show_unused=False) easy_pickle.dump(params.output.reflections, reflections) if params.show_weight_plots: plt.show()