def __init__(self, experiments, profile_fitter, reflections, num_folds): ''' Create the integration report :param experiments: The experiment list :param profile_model: The profile model :param reflections: The reflection table ''' from collections import OrderedDict # Initialise the report class super(ProfileValidationReport, self).__init__() # Create the table table = Table() # Set the title table.name = 'validation.summary' table.title = 'Summary of profile validation ' # Add the columns table.cols.append(('id', 'ID')) table.cols.append(('subsample', 'Sub-sample')) table.cols.append(('n_valid', '# validated')) table.cols.append(('cc', '<CC>')) table.cols.append(('nrmsd', '<NRMSD>')) # Split the reflections reflection_tables = reflections.split_by_experiment_id() assert len(reflection_tables) == len(experiments) assert len(profile_fitter) == num_folds # Create the summary for each profile model for i in range(len(reflection_tables)): reflection_table = reflection_tables[i] reflection_table = reflection_table.select( reflection_table.get_flags( reflection_table.flags.integrated_prf)) index = reflection_table['profile.index'] cc = reflection_table['profile.correlation'] nrmsd = reflection_table['profile.rmsd'] for j in range(num_folds): mask = index == j num_validated = mask.count(True) if num_validated == 0: mean_cc = 0 mean_nrmsd = 0 else: mean_cc = flex.mean(cc.select(mask)) mean_nrmsd = flex.mean(nrmsd.select(mask)) table.rows.append([ '%d' % i, '%d' % j, '%d' % num_validated, '%.2f' % mean_cc, '%.2f' % mean_nrmsd]) # Add the table self.add_table(table)
def tst_for_dataset(self, creator, filename): from dials.array_family import flex from dials.algorithms.shoebox import MaskCode print filename rlist = flex.reflection_table.from_pickle(filename) shoebox = rlist['shoebox'] background = [sb.background.deep_copy() for sb in shoebox] success = creator(shoebox) assert(success.count(True) == len(success)) diff = [] for i in range(len(rlist)): mask = flex.bool([(m & MaskCode.Foreground) != 0 for m in shoebox[i].mask]) px1 = background[i].select(mask) px2 = shoebox[i].background.select(mask) den = max([flex.mean(px1), 1.0]) diff.append(flex.mean(px2 - px1) / den) diff = flex.double(diff) mv = flex.mean_and_variance(flex.double(diff)) mean = mv.mean() sdev = mv.unweighted_sample_standard_deviation() try: assert(abs(mean) < 0.01) except Exception: print "Mean: %f, Sdev: %f", mean, sdev from matplotlib import pylab pylab.hist(diff) pylab.show() raise
def overall_report(data): # Start by adding some overall numbers report = OrderedDict() report['n'] = len(reflections) report['n_full'] = data['full'].count(True) report['n_partial'] = data['full'].count(False) report['n_overload'] = data['over'].count(True) report['n_ice'] = data['ice'].count(True) report['n_summed'] = data['sum'].count(True) report['n_fitted'] = data['prf'].count(True) report['n_integated'] = data['int'].count(True) report['n_invalid_bg'] = data['ninvbg'].count(True) report['n_invalid_fg'] = data['ninvfg'].count(True) report['n_failed_background'] = data['fbgd'].count(True) report['n_failed_summation'] = data['fsum'].count(True) report['n_failed_fitting'] = data['fprf'].count(True) # Compute mean background try: report['mean_background'] = flex.mean( data['background.mean'].select(data['int'])) except Exception: report['mean_background'] = 0.0 # Compute mean I/Sigma summation try: report['ios_sum'] = flex.mean( data['intensity.sum.ios'].select(data['sum'])) except Exception: report['ios_sum'] = 0.0 # Compute mean I/Sigma profile fitting try: report['ios_prf'] = flex.mean( data['intensity.prf.ios'].select(data['prf'])) except Exception: report['ios_prf'] = 0.0 # Compute the mean profile correlation try: report['cc_prf'] = flex.mean( data['profile.correlation'].select(data['prf'])) except Exception: report['cc_prf'] = 0.0 # Compute the correlations between summation and profile fitting try: mask = data['sum'] & data['prf'] Isum = data['intensity.sum.value'].select(mask) Iprf = data['intensity.prf.value'].select(mask) report['cc_pearson_sum_prf'] = pearson_correlation_coefficient(Isum, Iprf) report['cc_spearman_sum_prf'] = spearman_correlation_coefficient(Isum, Iprf) except Exception: report['cc_pearson_sum_prf'] = 0.0 report['cc_spearman_sum_prf'] = 0.0 # Return the overall report return report
def ini_contrast(self): if not self.contrast_initiated: try: n_of_imgs = len(self.my_sweep.indices()) logger.debug("n_of_imgs(ini_contrast) = %s", n_of_imgs) img_arr_n0 = self.my_sweep.get_raw_data(0)[0] img_arr_n1 = self.my_sweep.get_raw_data(1)[0] img_arr_n2 = self.my_sweep.get_raw_data(2)[0] tst_sample = ( img_arr_n0[0:25, 0:25].as_double() + img_arr_n1[0:25, 0:25].as_double() + img_arr_n2[0:25, 0:25].as_double() ) / 3.0 logger.debug("tst_sample = %s", tst_sample) i_mean = flex.mean(tst_sample) tst_new_max = (i_mean + 1) * 25 logger.debug("flex.mean(tst_sample) = %s", i_mean) logger.debug("tst_new_max = %s", tst_new_max) self.try_change_max(tst_new_max) self.try_change_min(-3) self.contrast_initiated = True except BaseException as e: # We don't want to catch bare exceptions but don't know # what this was supposed to catch. Log it. logger.error( "Caught unknown exception type %s: %s", type(e).__name__, e ) logger.debug("Unable to calculate mean and adjust contrast")
def plot_histograms(self, reflections, panel = None, ax = None, bounds = None): data = reflections['difference_vector_norms'] colors = ['b-', 'g-', 'g--', 'r-', 'b-', 'b--'] n_slots = 20 if self.params.residuals.histogram_max is None: h = flex.histogram(data, n_slots=n_slots) else: h = flex.histogram(data.select(data <= self.params.residuals.histogram_max), n_slots=n_slots) n = len(reflections) rmsd_obs = math.sqrt((reflections['xyzcal.mm']-reflections['xyzobs.mm.value']).sum_sq()/n) sigma = mode = h.slot_centers()[list(h.slots()).index(flex.max(h.slots()))] mean_obs = flex.mean(data) median = flex.median(data) mean_rayleigh = math.sqrt(math.pi/2)*sigma rmsd_rayleigh = math.sqrt(2)*sigma data = flex.vec2_double([(i,j) for i, j in zip(h.slot_centers(), h.slots())]) n = len(data) for i in [mean_obs, mean_rayleigh, mode, rmsd_obs, rmsd_rayleigh]: data.extend(flex.vec2_double([(i, 0), (i, flex.max(h.slots()))])) data = self.get_bounded_data(data, bounds) tmp = [data[:n]] for i in xrange(len(colors)): tmp.append(data[n+(i*2):n+((i+1)*2)]) data = tmp for d, c in zip(data, colors): ax.plot(d.parts()[0], d.parts()[1], c) if ax.get_legend() is None: ax.legend([r"$\Delta$XY", "MeanObs", "MeanRayl", "Mode", "RMSDObs", "RMSDRayl"])
def compute_functional_and_gradients_test_code(self): values = self.parameterization(self.x) assert -150. < values.BFACTOR < 150. # limits on the exponent, please self.func = self.refinery.fvec_callable(values) functional = flex.sum(self.func*self.func) self.f = functional jacobian = self.refinery.jacobian_callable(values) self.gg_0 = flex.sum(2. * self.func * jacobian[0]) self.gg_1 = flex.sum(2. * self.func * jacobian[1]) self.gg_3 = flex.sum(2. * self.func * jacobian[3]) self.gg_4 = flex.sum(2. * self.func * jacobian[4]) DELTA = 1.E-7 self.g = flex.double() for x in xrange(self.n): templist = list(self.x) templist[x]+=DELTA dvalues = flex.double(templist) dfunc = self.refinery.fvec_callable(self.parameterization(dvalues)) dfunctional = flex.sum(dfunc*dfunc) #calculate by finite_difference self.g.append( ( dfunctional-functional )/DELTA ) self.g[2]=0. print >> self.out, "rms %10.3f"%math.sqrt(flex.mean(self.func*self.func)), values.show(self.out) print >>self.out, "derivatives--> %15.5f %15.5f %9.7f %5.2f %5.2f"%tuple(self.g) print >>self.out, " analytical-> %15.5f %15.5f %5.2f %5.2f"%( self.gg_0,self.gg_1, self.gg_3,self.gg_4) self.g[0]=self.gg_0 self.g[1]=self.gg_1 self.g[3]=self.gg_3 self.g[4]=self.gg_4 return self.f, self.g
def outlier_rejection(reflections): # http://scripts.iucr.org/cgi-bin/paper?ba0032 if len(reflections) == 1: return reflections intensities = reflections['intensity.sum.value'] variances = reflections['intensity.sum.variance'] i_max = flex.max_index(intensities) sel = flex.bool(len(reflections), True) sel[i_max] = False i_test = intensities[i_max] var_test = variances[i_max] intensities_subset = intensities.select(sel) var_subset = variances.select(sel) var_prior = var_test + 1/flex.sum(1/var_subset) p_prior = 1/math.sqrt(2*math.pi * var_prior) * math.exp( -(i_test - flex.mean(intensities_subset))**2/(2 * var_prior)) #print p_prior if p_prior > 1e-10: return reflections return outlier_rejection(reflections.select(sel))
def calc_2D_rmsd_and_displacements(reflections): displacements = flex.vec2_double(reflections['xyzobs.px.value'].parts()[0], reflections['xyzobs.px.value'].parts()[1]) - \ flex.vec2_double(reflections['xyzcal.px'].parts()[0], reflections['xyzcal.px'].parts()[1]) rmsd = math.sqrt(flex.mean(displacements.dot( displacements ))) return rmsd,displacements
def build_up(pfh, objective_only=False): values = pfh.parameterization(pfh.x) assert 0. < values.G , "G-scale value out of range ( < 0 ) within LevMar build_up" # XXX revisit these limits. Seems like an ad hoc approach to have to set these limits # However, the assertions are necessary to avoid floating point exceptions at the C++ level # Regardless, these tests throw out ~30% of LM14 data, thus search for another approach assert -150. < values.BFACTOR < 150. ,"B-factor out of range (+/-150) within LevMar build_up" assert -0.5 < 180.*values.thetax/math.pi < 0.5 , "thetax out of range ( |rotx|>.5 degrees ) within LevMar build_up" assert -0.5 < 180.*values.thetay/math.pi < 0.5 , "thetay out of range ( |roty|>.5 degrees ) within LevMar build_up" assert 0.000001 < values.RS , "RLP size out of range (<0.000001) within LevMar build_up" assert values.RS < 0.001 , "RLP size out of range (>0.001) within LevMar build_up" residuals = pfh.refinery.fvec_callable(values) pfh.reset() if objective_only: pfh.add_residuals(residuals, weights=pfh.refinery.WEIGHTS) else: grad_r = pfh.refinery.jacobian_callable(values) jacobian = flex.double( flex.grid(len(pfh.refinery.MILLER), pfh.n_parameters)) for j, der_r in enumerate(grad_r): jacobian.matrix_paste_column_in_place(der_r,j) #print >> pfh.out, "COL",j, list(der_r) pfh.add_equations(residuals, jacobian, weights=pfh.refinery.WEIGHTS) print >> pfh.out, "rms %10.3f"%math.sqrt(flex.mean(pfh.refinery.WEIGHTS*residuals*residuals)), values.show(pfh.out)
def wrapper(experiment): from dials.algorithms.profile_model.gaussian_rs import GaussianRSProfileModeller from math import ceil # Return if no scan or gonio if (experiment.scan is None or experiment.goniometer is None or experiment.scan.get_oscillation()[1] == 0): return None # Compute the scan step phi0, phi1 = experiment.scan.get_oscillation_range(deg=True) assert(phi1 > phi0) phi_range = phi1 - phi0 num_scan_points = int(ceil(phi_range / self.params.gaussian_rs.fitting.scan_step)) assert(num_scan_points > 0) # Create the grid method GridMethod = GaussianRSProfileModeller.GridMethod FitMethod = GaussianRSProfileModeller.FitMethod grid_method = int(GridMethod.names[self.params.gaussian_rs.fitting.grid_method].real) fit_method = int(FitMethod.names[self.params.gaussian_rs.fitting.fit_method].real) if self._scan_varying: sigma_b = flex.mean(self.sigma_b(deg=False)) sigma_m = flex.mean(self.sigma_m(deg=False)) else: sigma_b = self.sigma_b(deg=False) sigma_m = self.sigma_m(deg=False) # Create the modeller return GaussianRSProfileModeller( experiment.beam, experiment.detector, experiment.goniometer, experiment.scan, sigma_b, sigma_m, self.n_sigma(), self.params.gaussian_rs.fitting.grid_size, num_scan_points, self.params.gaussian_rs.fitting.threshold, grid_method, fit_method)
def compute_functional_and_gradients(self): values = self.parameterization(self.x) assert -150. < values.BFACTOR < 150. # limits on the exponent, please self.func = self.refinery.fvec_callable(values) functional = flex.sum(self.func*self.func) self.f = functional jacobian = self.refinery.jacobian_callable(values) self.g = flex.double(self.n) for ix in xrange(self.n): self.g[ix] = flex.sum(2. * self.func * jacobian[ix]) print >> self.out, "rms %10.3f"%math.sqrt(flex.mean(self.func*self.func)), values.show(self.out) return self.f, self.g
def compute(self, reflections): from dials.algorithms.background.gmodel import Fitter from dials.array_family import flex assert self.finalized() fitter = Fitter(self.background) scale = fitter(reflections['shoebox']) success = scale >= 0 mean = flex.double([ flex.mean(sbox.background) for sbox in reflections['shoebox'] ]) reflections['background.mean'] = mean reflections['background.scale'] = scale reflections.set_flags(success != True, reflections.flags.dont_integrate) return success
def histogram(self, reflections, title): data = reflections['difference_vector_norms'] n_slots = 100 if self.params.residuals.histogram_max is None: h = flex.histogram(data, n_slots=n_slots) else: h = flex.histogram(data.select(data <= self.params.residuals.histogram_max), n_slots=n_slots) n = len(reflections) rmsd = math.sqrt((reflections['xyzcal.mm']-reflections['xyzobs.mm.value']).sum_sq()/n) sigma = mode = h.slot_centers()[list(h.slots()).index(flex.max(h.slots()))] mean = flex.mean(data) median = flex.median(data) print "RMSD (microns)", rmsd * 1000 print "Histogram mode (microns):", mode * 1000 print "Overall mean (microns):", mean * 1000 print "Overall median (microns):", median * 1000 mean2 = math.sqrt(math.pi/2)*sigma rmsd2 = math.sqrt(2)*sigma print "Rayleigh Mean (microns)", mean2 * 1000 print "Rayleigh RMSD (microns)", rmsd2 * 1000 r = reflections['radial_displacements'] t = reflections['transverse_displacements'] print "Overall radial RMSD (microns)", math.sqrt(flex.sum_sq(r)/len(r)) * 1000 print "Overall transverse RMSD (microns)", math.sqrt(flex.sum_sq(t)/len(t)) * 1000 fig = plt.figure() ax = fig.add_subplot(111) ax.plot(h.slot_centers().as_numpy_array(), h.slots().as_numpy_array(), '-') vmax = self.params.residuals.plot_max if self.params.residuals.histogram_xmax is not None: ax.set_xlim((0,self.params.residuals.histogram_xmax)) if self.params.residuals.histogram_ymax is not None: ax.set_ylim((0,self.params.residuals.histogram_ymax)) plt.title(title) ax.plot((mean, mean), (0, flex.max(h.slots())), 'g-') ax.plot((mean2, mean2), (0, flex.max(h.slots())), 'g--') ax.plot((mode, mode), (0, flex.max(h.slots())), 'r-') ax.plot((rmsd, rmsd), (0, flex.max(h.slots())), 'b-') ax.plot((rmsd2, rmsd2), (0, flex.max(h.slots())), 'b--') ax.legend([r"$\Delta$XY", "MeanObs", "MeanRayl", "Mode", "RMSDObs", "RMSDRayl"]) ax.set_xlabel("(mm)") ax.set_ylabel("Count")
def compute_functional_and_gradients(self): values = self.parameterization(self.x) assert -150. < values.BFACTOR < 150. # limits on the exponent, please self.func = self.refinery.fvec_callable(values) functional = flex.sum(self.func*self.func) self.f = functional DELTA = 1.E-7 self.g = flex.double() for x in xrange(self.n): templist = list(self.x) templist[x]+=DELTA dvalues = flex.double(templist) dfunc = self.refinery.fvec_callable(self.parameterization(dvalues)) dfunctional = flex.sum(dfunc*dfunc) #calculate by finite_difference self.g.append( ( dfunctional-functional )/DELTA ) self.g[2]=0. print >> self.out, "rms %10.3f"%math.sqrt(flex.mean(self.func*self.func)), values.show(self.out) return self.f, self.g
def wilson_outliers(reflections, ice_sel=None, p_cutoff=1e-2): # http://scripts.iucr.org/cgi-bin/paper?ba0032 if ice_sel is None: ice_sel = flex.bool(len(reflections), False) E_cutoff = math.sqrt(-math.log(p_cutoff)) intensities = reflections['intensity.sum.value'] variances = reflections['intensity.sum.variance'] Sigma_n = flex.mean(intensities.select(~ice_sel)) normalised_amplitudes = flex.sqrt(intensities)/math.sqrt(Sigma_n) outliers = normalised_amplitudes >= E_cutoff if outliers.count(True): # iterative outlier rejection inliers = ~outliers outliers.set_selected( inliers, wilson_outliers( reflections.select(inliers), ice_sel.select(inliers))) return outliers
def wilson_outliers(reflections, ice_sel=None, p_cutoff=1e-2): # http://scripts.iucr.org/cgi-bin/paper?ba0032 if ice_sel is None: ice_sel = flex.bool(len(reflections), False) E_cutoff = math.sqrt(-math.log(p_cutoff)) intensities = reflections["intensity.sum.value"] Sigma_n = flex.mean(intensities.select(~ice_sel)) normalised_amplitudes = flex.sqrt(intensities) / math.sqrt(Sigma_n) outliers = normalised_amplitudes >= E_cutoff if outliers.count(True): # iterative outlier rejection inliers = ~outliers outliers.set_selected( inliers, wilson_outliers(reflections.select(inliers), ice_sel.select(inliers)), ) return outliers
def compute_functional_and_gradients(self): values = self.parameterization(self.x) assert -150. < values.BFACTOR < 150. # limits on the exponent, please self.func = self.refinery.fvec_callable(values) functional = flex.sum(self.func * self.func) self.f = functional DELTA = 1.E-7 self.g = flex.double() for x in range(self.n): templist = list(self.x) templist[x] += DELTA dvalues = flex.double(templist) dfunc = self.refinery.fvec_callable(self.parameterization(dvalues)) dfunctional = flex.sum(dfunc * dfunc) #calculate by finite_difference self.g.append((dfunctional - functional) / DELTA) self.g[2] = 0. print("rms %10.3f" % math.sqrt(flex.mean(self.func * self.func)), end=' ', file=self.out) values.show(self.out) return self.f, self.g
def setup_work_arrays(self, reflections): '''Select multiply-measured HKLs. Calculate and cache reflection deltas, deltas squared, and HKL means for every reflection''' self.deltas = flex.double() self.work_table = flex.reflection_table() delta_sq = flex.double() mean = flex.double() # mean = <I'_hj> biased_mean = flex.double() # biased_mean = <I_h>, so dont leave out any reflection var = flex.double() all_biased_mean = flex.double() for refls in reflection_table_utils.get_next_hkl_reflection_table(reflections): number_of_measurements = refls.size() if number_of_measurements == 0: # if the returned "refls" list is empty, it's the end of the input "reflections" list break refls_biased_mean = flex.double(len(refls), flex.mean(refls['intensity.sum.value'])) all_biased_mean.extend(refls_biased_mean) if number_of_measurements > self.params.merging.minimum_multiplicity: nn_factor_sqrt = math.sqrt((number_of_measurements - 1) / number_of_measurements) i_sum = flex.double(number_of_measurements, flex.sum(refls['intensity.sum.value'])) i_sum_minus_val = i_sum - refls['intensity.sum.value'] mean_without_val = i_sum_minus_val/(number_of_measurements-1) delta = nn_factor_sqrt * (refls['intensity.sum.value'] - mean_without_val) self.deltas.extend(delta/flex.sqrt(refls['intensity.sum.variance'])) # Please be careful about where to put the var delta_sq.extend(delta**2) mean.extend(mean_without_val) biased_mean.extend(refls_biased_mean) var.extend(refls['intensity.sum.variance']) self.work_table["delta_sq"] = delta_sq self.work_table["mean"] = mean self.work_table["biased_mean"] = biased_mean self.work_table["var"] = var reflections['biased_mean'] = all_biased_mean self.logger.log("Number of work reflections selected: %d"%self.deltas.size()) return reflections
def compute_functional_and_gradients_test_code(self): values = self.parameterization(self.x) assert -150. < values.BFACTOR < 150. # limits on the exponent, please self.func = self.refinery.fvec_callable(values) functional = flex.sum(self.func * self.func) self.f = functional jacobian = self.refinery.jacobian_callable(values) self.gg_0 = flex.sum(2. * self.func * jacobian[0]) self.gg_1 = flex.sum(2. * self.func * jacobian[1]) self.gg_3 = flex.sum(2. * self.func * jacobian[3]) self.gg_4 = flex.sum(2. * self.func * jacobian[4]) DELTA = 1.E-7 self.g = flex.double() for x in xrange(self.n): templist = list(self.x) templist[x] += DELTA dvalues = flex.double(templist) dfunc = self.refinery.fvec_callable(self.parameterization(dvalues)) dfunctional = flex.sum(dfunc * dfunc) #calculate by finite_difference self.g.append((dfunctional - functional) / DELTA) self.g[2] = 0. print >> self.out, "rms %10.3f" % math.sqrt( flex.mean(self.func * self.func)), values.show(self.out) print >> self.out, "derivatives--> %15.5f %15.5f %9.7f %5.2f %5.2f" % tuple( self.g) print >> self.out, " analytical-> %15.5f %15.5f %5.2f %5.2f" % ( self.gg_0, self.gg_1, self.gg_3, self.gg_4) self.g[0] = self.gg_0 self.g[1] = self.gg_1 self.g[3] = self.gg_3 self.g[4] = self.gg_4 return self.f, self.g
def __call__(self): """Determine optimal mosaicity and domain size model (monochromatic)""" if self.refinery is None: RR = self.reflections else: RR = self.refinery.predict_for_reflection_table(self.reflections) all_crystals = [] self.nv_acceptance_flags = flex.bool(len(self.reflections["id"])) from dxtbx.model import MosaicCrystalSauter2014 for iid, experiment in enumerate(self.experiments): excursion_rad = RR["delpsical.rad"].select(RR["id"] == iid) delta_psi_deg = excursion_rad * 180.0 / math.pi logger.info("") logger.info("%s %s", flex.max(delta_psi_deg), flex.min(delta_psi_deg)) mean_excursion = flex.mean(delta_psi_deg) logger.info( "The mean excursion is %7.3f degrees, r.m.s.d %7.3f", mean_excursion, math.sqrt(flex.mean(RR["delpsical2"].select(RR["id"] == iid))), ) crystal = MosaicCrystalSauter2014(self.experiments[iid].crystal) self.experiments[iid].crystal = crystal beam = self.experiments[iid].beam miller_indices = self.reflections["miller_index"].select( self.reflections["id"] == iid) # FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection two_thetas = crystal.get_unit_cell().two_theta( miller_indices, beam.get_wavelength(), deg=True) dspacings = crystal.get_unit_cell().d(miller_indices) # First -- try to get a reasonable envelope for the observed excursions. # minimum of three regions; maximum of 50 measurements in each bin logger.info("fitting parameters on %d spots", len(excursion_rad)) n_bins = min(max(3, len(excursion_rad) // 25), 50) bin_sz = len(excursion_rad) // n_bins logger.info("nbins %s bin_sz %s", n_bins, bin_sz) order = flex.sort_permutation(two_thetas) two_thetas_env = flex.double() dspacings_env = flex.double() excursion_rads_env = flex.double() for x in range(0, n_bins): subset = order[x * bin_sz:(x + 1) * bin_sz] two_thetas_env.append(flex.mean(two_thetas.select(subset))) dspacings_env.append(flex.mean(dspacings.select(subset))) excursion_rads_env.append( flex.max(flex.abs(excursion_rad.select(subset)))) # Second -- parameter fit # solve the normal equations sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env) sum_inv_u = flex.sum(dspacings_env) sum_te_u = flex.sum(dspacings_env * excursion_rads_env) sum_te = flex.sum(excursion_rads_env) Normal_Mat = sqr( (sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env))) Vector = col((sum_te_u, sum_te)) solution = Normal_Mat.inverse() * Vector s_ang = 1.0 / (2 * solution[0]) logger.info("Best LSQ fit Scheerer domain size is %9.2f ang", s_ang) k_degrees = solution[1] * 180.0 / math.pi logger.info( "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f", 2 * k_degrees, k_degrees, ) from xfel.mono_simulation.max_like import minimizer # coerce the estimates to be positive for max-likelihood lower_limit_domain_size = ( math.pow(crystal.get_unit_cell().volume(), 1.0 / 3.0) * 3 ) # params.refinement.domain_size_lower_limit d_estimate = max(s_ang, lower_limit_domain_size) M = minimizer( d_i=dspacings, psi_i=excursion_rad, eta_rad=abs(2.0 * solution[1]), Deff=d_estimate, ) logger.info( "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots", M.x[1] * 180.0 / math.pi, 2.0 / M.x[0], len(two_thetas), ) tan_phi_rad_ML = dspacings / (2.0 / M.x[0]) tan_phi_deg_ML = tan_phi_rad_ML * 180.0 / math.pi tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180.0 / math.pi # Only set the flags for those reflections that were indexed for this lattice self.nv_acceptance_flags.set_selected( self.reflections["id"] == iid, flex.abs(delta_psi_deg) < tan_outer_deg_ML, ) if ( self.graph_verbose ): # params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit AD1TF7B_MAX2T = 30.0 AD1TF7B_MAXDP = 1.0 from matplotlib import pyplot as plt plt.plot(two_thetas, delta_psi_deg, "bo") minplot = flex.min(two_thetas) plt.plot([0, minplot], [mean_excursion, mean_excursion], "k-") LR = flex.linear_regression(two_thetas, delta_psi_deg) model_y = LR.slope() * two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") plt.title( "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" % (M.x[1] * 180.0 / math.pi, 2.0 / M.x[0], len(two_thetas))) plt.plot(two_thetas, tan_phi_deg_ML, "r.") plt.plot(two_thetas, -tan_phi_deg_ML, "r.") plt.plot(two_thetas, tan_outer_deg_ML, "g.") plt.plot(two_thetas, -tan_outer_deg_ML, "g.") plt.xlim([0, AD1TF7B_MAX2T]) plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP]) plt.show() plt.close() from xfel.mono_simulation.util import green_curve_area self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML) logger.info("The green curve area is %s", self.green_curve_area) crystal.set_half_mosaicity_deg(M.x[1] * 180.0 / (2.0 * math.pi)) crystal.set_domain_size_ang(2.0 / M.x[0]) self._ML_full_mosaicity_rad = M.x[1] self._ML_domain_size_ang = 2.0 / M.x[0] # params.refinement.mosaic.model_expansion_factor """The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes as close as possible to # of observed reflections input, in the last integration call. Determine this by inspecting the output log file interactively. Do not exceed the bare minimum threshold needed. The intention is to find an optimal value, global for a given dataset.""" model_expansion_factor = 1.4 crystal.set_half_mosaicity_deg(crystal.get_half_mosaicity_deg() * model_expansion_factor) crystal.set_domain_size_ang(crystal.get_domain_size_ang() / model_expansion_factor) if (self.ewald_proximal_volume(iid) > self.params.indexing.stills.ewald_proximal_volume_max): raise DialsIndexError("Ewald proximity volume too high, %f" % self.ewald_proximal_volume(iid)) all_crystals.append(crystal) return all_crystals
def __init__(self,measurements_orig, params, i_model, miller_set, result, out): measurements = measurements_orig.deep_copy() # Now manipulate the data to conform to unit cell, asu, and space group # of reference. The resolution will be cut later. # Only works if there is NOT an indexing ambiguity! observations = measurements.customized_copy( anomalous_flag=not params.merge_anomalous, crystal_symmetry=miller_set.crystal_symmetry() ).map_to_asu() observations_original_index = measurements.customized_copy( anomalous_flag=not params.merge_anomalous, crystal_symmetry=miller_set.crystal_symmetry() ) # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. assert len(i_model.indices()) == len(miller_set.indices()) \ and (i_model.indices() == miller_set.indices()).count(False) == 0 matches = miller.match_multi_indices( miller_indices_unique=miller_set.indices(), miller_indices=observations.indices()) pair1 = flex.int([pair[1] for pair in matches.pairs()]) pair0 = flex.int([pair[0] for pair in matches.pairs()]) # narrow things down to the set that matches, only observations_pair1_selected = observations.customized_copy( indices = flex.miller_index([observations.indices()[p] for p in pair1]), data = flex.double([observations.data()[p] for p in pair1]), sigmas = flex.double([observations.sigmas()[p] for p in pair1]), ) observations_original_index_pair1_selected = observations_original_index.customized_copy( indices = flex.miller_index([observations_original_index.indices()[p] for p in pair1]), data = flex.double([observations_original_index.data()[p] for p in pair1]), sigmas = flex.double([observations_original_index.sigmas()[p] for p in pair1]), ) ################### I_observed = observations_pair1_selected.data() chosen = chosen_weights(observations_pair1_selected, params) MILLER = observations_original_index_pair1_selected.indices() ORI = result["current_orientation"][0] Astar = matrix.sqr(ORI.reciprocal_matrix()) WAVE = result["wavelength"] BEAM = matrix.col((0.0,0.0,-1./WAVE)) BFACTOR = 0. #calculation of correlation here I_reference = flex.double([i_model.data()[pair[0]] for pair in matches.pairs()]) use_weights = False # New facility for getting variance-weighted correlation if use_weights: #variance weighting I_weight = flex.double( [1./(observations_pair1_selected.sigmas()[pair[1]])**2 for pair in matches.pairs()]) else: I_weight = flex.double(len(observations_pair1_selected.sigmas()), 1.) """Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement: include_negatives = True + and - reflections both used for Rh distribution for initial estimate of RS parameter + and - reflections both used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) include_negatives = False + and - reflections both used for Rh distribution for initial estimate of RS parameter + reflections only used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) """ if params.include_negatives: SWC = simple_weighted_correlation(I_weight, I_reference, I_observed) else: non_positive = ( observations_pair1_selected.data() <= 0 ) SWC = simple_weighted_correlation(I_weight.select(~non_positive), I_reference.select(~non_positive), I_observed.select(~non_positive)) print >> out, "Old correlation is", SWC.corr assert params.postrefinement.algorithm=="rs_hybrid" Rhall = flex.double() for mill in MILLER: H = matrix.col(mill) Xhkl = Astar*H Rh = ( Xhkl + BEAM ).length() - (1./WAVE) Rhall.append(Rh) Rs = math.sqrt(flex.mean(Rhall*Rhall)) RS = 1./10000. # reciprocal effective domain size of 1 micron RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value self.rs2_current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.]) self.rs2_parameterization_class = rs_parameterization self.rs2_refinery = rs2_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC = I_reference, IOBSVEC = I_observed, WEIGHTS = chosen) self.rs2_refinery.set_profile_shape(params.postrefinement.lineshape) self.nave1_refinery = nave1_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC = I_reference, IOBSVEC = I_observed, WEIGHTS = chosen) self.nave1_refinery.set_profile_shape(params.postrefinement.lineshape) self.out=out; self.params = params; self.miller_set = miller_set self.observations_pair1_selected = observations_pair1_selected; self.observations_original_index_pair1_selected = observations_original_index_pair1_selected self.i_model = i_model
def run(self, experiments, reflections): self.logger.log_step_time("POLARIZATION_CORRECTION") result = flex.reflection_table() for experiment in experiments: refls = reflections.select(reflections['exp_id'] == experiment.identifier) beam = experiment.beam # Remove the need for pixel size within cxi.merge. Allows multipanel detector with dissimilar panels. # Relies on new frame extractor code called by dials.stills_process that writes s0, s1 and polarization normal # vectors all to the integration pickle. Future path (IE THIS CODE): use dials json and reflection file. s0_vec = matrix.col(beam.get_s0()).normalize() s0_polar_norm = beam.get_polarization_normal() s1_vec = refls['s1'] Ns1 = len(s1_vec) # project the s1_vector onto the plane normal to s0. Get result by subtracting the # projection of s1 onto s0, which is (s1.dot.s0_norm)s0_norm s0_norm = flex.vec3_double(Ns1,s0_vec) s1_proj = (s1_vec.dot(s0_norm))*s0_norm s1_in_normal_plane = s1_vec - s1_proj # Now want the polar angle between the projected s1 and the polarization normal s0_polar_norms = flex.vec3_double(Ns1,s0_polar_norm) dotprod = (s1_in_normal_plane.dot(s0_polar_norms)) costheta = dotprod/(s1_in_normal_plane.norms()) theta = flex.acos(costheta) cos_two_polar_angle = flex.cos(2.0*theta) # gives same as old answer to ~1% but not exact. Not sure why, should not matter. tt_vec = experiment.crystal.get_unit_cell().two_theta(miller_indices = refls['miller_index'], wavelength = beam.get_wavelength()) cos_tt_vec = flex.cos(tt_vec) sin_tt_vec = flex.sin(tt_vec) cos_sq_tt_vec = cos_tt_vec * cos_tt_vec sin_sq_tt_vec = sin_tt_vec * sin_tt_vec P_nought_vec = 0.5 * (1. + cos_sq_tt_vec) F_prime = -1.0 # Hard-coded value defines the incident polarization axis P_prime = 0.5 * F_prime * cos_two_polar_angle * sin_sq_tt_vec # added as a diagnostic #prange=P_nought_vec - P_prime #other_F_prime = 1.0 #otherP_prime = 0.5 * other_F_prime * cos_two_polar_angle * sin_sq_tt_vec #otherprange=P_nought_vec - otherP_prime #diff2 = flex.abs(prange - otherprange) #print >> out, "mean diff is",flex.mean(diff2), "range",flex.min(diff2), flex.max(diff2) # done correction = 1 / ( P_nought_vec - P_prime ) refls['intensity.sum.value'] = refls['intensity.sum.value'] * correction refls['intensity.sum.variance'] = refls['intensity.sum.variance'] * correction**2 # propagated error # This corrects observations for polarization assuming 100% polarization on # one axis (thus the F_prime = -1.0 rather than the perpendicular axis, 1.0) # Polarization model as described by Kahn, Fourme, Gadet, Janin, Dumas & Andre # (1982) J. Appl. Cryst. 15, 330-337, equations 13 - 15. result.extend(refls) if len(reflections) > 0: self.logger.log("Applied polarization correction. Mean intensity changed from %.2f to %.2f"%(flex.mean(reflections['intensity.sum.value']), flex.mean(result['intensity.sum.value']))) self.logger.log_step_time("POLARIZATION_CORRECTION", True) return experiments, result
def run(self): ''' Parse the options. ''' from dials.util.options import flatten_experiments, flatten_reflections # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) self.params = params experiments = flatten_experiments(params.input.experiments) # Find all detector objects detectors = experiments.detectors() # Verify inputs if len(params.input.reflections) == len(detectors) and len(detectors) > 1: # case for passing in multiple images on the command line assert len(params.input.reflections) == len(detectors) reflections = flex.reflection_table() for expt_id in xrange(len(detectors)): subset = params.input.reflections[expt_id].data subset['id'] = flex.int(len(subset), expt_id) reflections.extend(subset) else: # case for passing in combined experiments and reflections reflections = flatten_reflections(params.input.reflections)[0] detector = detectors[0] #from dials.algorithms.refinement.prediction import ExperimentsPredictor #ref_predictor = ExperimentsPredictor(experiments, force_stills=experiments.all_stills()) print "N reflections total:", len(reflections) if params.residuals.exclude_outliers: reflections = reflections.select(reflections.get_flags(reflections.flags.used_in_refinement)) print "N reflections used in refinement:", len(reflections) print "Reporting only on those reflections used in refinement" if self.params.residuals.i_sigi_cutoff is not None: sel = (reflections['intensity.sum.value']/flex.sqrt(reflections['intensity.sum.variance'])) >= self.params.residuals.i_sigi_cutoff reflections = reflections.select(sel) print "After filtering by I/sigi cutoff of %f, there are %d reflections left"%(self.params.residuals.i_sigi_cutoff,len(reflections)) reflections['difference_vector_norms'] = (reflections['xyzcal.mm']-reflections['xyzobs.mm.value']).norms() n = len(reflections) rmsd = self.get_weighted_rmsd(reflections) print "Dataset RMSD (microns)", rmsd * 1000 if params.tag is None: tag = '' else: tag = '%s '%params.tag # set up delta-psi ratio heatmap p = flex.int() # positive n = flex.int() # negative for i in set(reflections['id']): exprefls = reflections.select(reflections['id']==i) p.append(len(exprefls.select(exprefls['delpsical.rad']>0))) n.append(len(exprefls.select(exprefls['delpsical.rad']<0))) plt.hist2d(p, n, bins=30) cb = plt.colorbar() cb.set_label("N images") plt.title(r"%s2D histogram of pos vs. neg $\Delta\Psi$ per image"%tag) plt.xlabel(r"N reflections with $\Delta\Psi$ > 0") plt.ylabel(r"N reflections with $\Delta\Psi$ < 0") self.delta_scalar = 50 # Iterate through the detectors, computing detector statistics at the per-panel level (IE one statistic per panel) # Per panel dictionaries rmsds = {} refl_counts = {} transverse_rmsds = {} radial_rmsds = {} ttdpcorr = {} pg_bc_dists = {} mean_delta_two_theta = {} # per panelgroup flex arrays pg_rmsds = flex.double() pg_r_rmsds = flex.double() pg_t_rmsds = flex.double() pg_refls_count = flex.int() pg_refls_count_d = {} table_header = ["PG id", "RMSD","Radial", "Transverse", "N refls"] table_header2 = ["","(um)","RMSD (um)","RMSD (um)",""] table_data = [] table_data.append(table_header) table_data.append(table_header2) # Compute a set of radial and transverse displacements for each reflection print "Setting up stats..." tmp = flex.reflection_table() # Need to construct a variety of vectors for panel_id, panel in enumerate(detector): panel_refls = reflections.select(reflections['panel'] == panel_id) bcl = flex.vec3_double() tto = flex.double() ttc = flex.double() # Compute the beam center in lab space (a vector pointing from the origin to where the beam would intersect # the panel, if it did intersect the panel) for expt_id in set(panel_refls['id']): beam = experiments[expt_id].beam s0 = beam.get_s0() expt_refls = panel_refls.select(panel_refls['id'] == expt_id) beam_centre = panel.get_beam_centre_lab(s0) bcl.extend(flex.vec3_double(len(expt_refls), beam_centre)) obs_x, obs_y, _ = expt_refls['xyzobs.px.value'].parts() cal_x, cal_y, _ = expt_refls['xyzcal.px'].parts() tto.extend(flex.double([panel.get_two_theta_at_pixel(s0, (obs_x[i], obs_y[i])) for i in xrange(len(expt_refls))])) ttc.extend(flex.double([panel.get_two_theta_at_pixel(s0, (cal_x[i], cal_y[i])) for i in xrange(len(expt_refls))])) panel_refls['beam_centre_lab'] = bcl panel_refls['two_theta_obs'] = tto * (180/math.pi) panel_refls['two_theta_cal'] = ttc * (180/math.pi) #+ (0.5*panel_refls['delpsical.rad']*panel_refls['two_theta_obs']) # Compute obs in lab space x, y, _ = panel_refls['xyzobs.mm.value'].parts() c = flex.vec2_double(x, y) panel_refls['obs_lab_coords'] = panel.get_lab_coord(c) # Compute deltaXY in panel space. This vector is relative to the panel origin x, y, _ = (panel_refls['xyzcal.mm'] - panel_refls['xyzobs.mm.value']).parts() # Convert deltaXY to lab space, subtracting off of the panel origin panel_refls['delta_lab_coords'] = panel.get_lab_coord(flex.vec2_double(x,y)) - panel.get_origin() tmp.extend(panel_refls) reflections = tmp # The radial vector points from the center of the reflection to the beam center radial_vectors = (reflections['obs_lab_coords'] - reflections['beam_centre_lab']).each_normalize() # The transverse vector is orthogonal to the radial vector and the beam vector transverse_vectors = radial_vectors.cross(reflections['beam_centre_lab']).each_normalize() # Compute the raidal and transverse components of each deltaXY reflections['radial_displacements'] = reflections['delta_lab_coords'].dot(radial_vectors) reflections['transverse_displacements'] = reflections['delta_lab_coords'].dot(transverse_vectors) # Iterate through the detector at the specified hierarchy level for pg_id, pg in enumerate(iterate_detector_at_level(detector.hierarchy(), 0, params.hierarchy_level)): pg_msd_sum = 0 pg_r_msd_sum = 0 pg_t_msd_sum = 0 pg_refls = 0 pg_delpsi = flex.double() pg_deltwotheta = flex.double() for p in iterate_panels(pg): panel_id = id_from_name(detector, p.get_name()) panel_refls = reflections.select(reflections['panel'] == panel_id) n = len(panel_refls) pg_refls += n delta_x = panel_refls['xyzcal.mm'].parts()[0] - panel_refls['xyzobs.mm.value'].parts()[0] delta_y = panel_refls['xyzcal.mm'].parts()[1] - panel_refls['xyzobs.mm.value'].parts()[1] tmp = flex.sum((delta_x**2)+(delta_y**2)) pg_msd_sum += tmp r = panel_refls['radial_displacements'] t = panel_refls['transverse_displacements'] pg_r_msd_sum += flex.sum_sq(r) pg_t_msd_sum += flex.sum_sq(t) pg_delpsi.extend(panel_refls['delpsical.rad']*180/math.pi) pg_deltwotheta.extend(panel_refls['two_theta_obs'] - panel_refls['two_theta_cal']) bc = col(pg.get_beam_centre_lab(s0)) ori = get_center(pg) pg_bc_dists[pg.get_name()] = (ori-bc).length() if len(pg_deltwotheta) > 0: mean_delta_two_theta[pg.get_name()] = flex.mean(pg_deltwotheta) else: mean_delta_two_theta[pg.get_name()] = 0 if pg_refls == 0: pg_rmsd = pg_r_rmsd = pg_t_rmsd = 0 else: pg_rmsd = math.sqrt(pg_msd_sum/pg_refls) * 1000 pg_r_rmsd = math.sqrt(pg_r_msd_sum/pg_refls) * 1000 pg_t_rmsd = math.sqrt(pg_t_msd_sum/pg_refls) * 1000 pg_rmsds.append(pg_rmsd) pg_r_rmsds.append(pg_r_rmsd) pg_t_rmsds.append(pg_t_rmsd) pg_refls_count.append(pg_refls) pg_refls_count_d[pg.get_name()] = pg_refls table_data.append(["%d"%pg_id, "%.1f"%pg_rmsd, "%.1f"%pg_r_rmsd, "%.1f"%pg_t_rmsd, "%6d"%pg_refls]) refl_counts[pg.get_name()] = pg_refls if pg_refls == 0: rmsds[p.get_name()] = -1 radial_rmsds[p.get_name()] = -1 transverse_rmsds[p.get_name()] = -1 ttdpcorr[pg.get_name()] = -1 else: rmsds[pg.get_name()] = pg_rmsd radial_rmsds[pg.get_name()] = pg_r_rmsd transverse_rmsds[pg.get_name()] = pg_t_rmsd lc = flex.linear_correlation(pg_delpsi, pg_deltwotheta) ttdpcorr[pg.get_name()] = lc.coefficient() r1 = ["Weighted mean"] r2 = ["Weighted stddev"] if len(pg_rmsds) > 1: stats = flex.mean_and_variance(pg_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(pg_r_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) stats = flex.mean_and_variance(pg_t_rmsds, pg_refls_count.as_double()) r1.append("%.1f"%stats.mean()) r2.append("%.1f"%stats.gsl_stats_wsd()) else: r1.extend([""]*3) r2.extend([""]*3) r1.append("") r2.append("") table_data.append(r1) table_data.append(r2) table_data.append(["Mean", "", "", "", "%8.1f"%flex.mean(pg_refls_count.as_double())]) from libtbx import table_utils print "Detector statistics. Angles in degrees, RMSDs in microns" print table_utils.format(table_data,has_header=2,justify='center',delim=" ") self.histogram(reflections, '%sDifference vector norms (mm)'%tag) if params.show_plots: if self.params.tag is None: t = "" else: t = "%s "%self.params.tag self.image_rmsd_histogram(reflections, tag) # Plots! these are plots with callbacks to draw on individual panels self.detector_plot_refls(detector, reflections, '%sOverall positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltas) self.detector_plot_refls(detector, reflections, '%sRadial positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_radial_deltas) self.detector_plot_refls(detector, reflections, '%sTransverse positional displacements (mm)'%tag, show=False, plot_callback=self.plot_obs_colored_by_transverse_deltas) self.detector_plot_refls(detector, reflections, r'%s$\Delta\Psi$'%tag, show=False, plot_callback=self.plot_obs_colored_by_deltapsi, colorbar_units=r"$\circ$") self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY*%s'%(tag, self.delta_scalar), show=False, plot_callback=self.plot_deltas) self.detector_plot_refls(detector, reflections, '%sSP Manual CDF'%tag, show=False, plot_callback=self.plot_cdf_manually) self.detector_plot_refls(detector, reflections, r'%s$\Delta$XY Histograms'%tag, show=False, plot_callback=self.plot_histograms) self.detector_plot_refls(detector, reflections, r'%sRadial displacements vs. $\Delta\Psi$, colored by $\Delta$XY'%tag, show=False, plot_callback=self.plot_radial_displacements_vs_deltapsi) self.detector_plot_refls(detector, reflections, r'%sDistance vector norms'%tag, show=False, plot_callback=self.plot_difference_vector_norms_histograms) # Plot intensity vs. radial_displacement fig = plt.figure() panel_id = 15 panel_refls = reflections.select(reflections['panel'] == panel_id) a = panel_refls['radial_displacements'] b = panel_refls['intensity.sum.value'] sel = (a > -0.2) & (a < 0.2) & (b < 50000) plt.hist2d(a.select(sel), b.select(sel), bins=100) plt.title("%s2D histogram of intensity vs. radial displacement for panel %d"%(tag, panel_id)) plt.xlabel("Radial displacement (mm)") plt.ylabel("Intensity") ax = plt.colorbar() ax.set_label("Counts") # Plot delta 2theta vs. deltapsi n_bins = 10 bin_size = len(reflections)//n_bins bin_low = [] bin_high = [] data = flex.sorted(reflections['two_theta_obs']) for i in xrange(n_bins): bin_low = data[i*bin_size] if (i+1)*bin_size >= len(reflections): bin_high = data[-1] else: bin_high = data[(i+1)*bin_size] refls = reflections.select((reflections['two_theta_obs'] >= bin_low) & (reflections['two_theta_obs'] <= bin_high)) a = refls['delpsical.rad']*180/math.pi b = refls['two_theta_obs'] - refls['two_theta_cal'] fig = plt.figure() sel = (a > -0.2) & (a < 0.2) & (b > -0.05) & (b < 0.05) plt.hist2d(a.select(sel), b.select(sel), bins=50, range = [[-0.2, 0.2], [-0.05, 0.05]]) cb = plt.colorbar() cb.set_label("N reflections") plt.title(r'%sBin %d (%.02f, %.02f 2$\Theta$) $\Delta2\Theta$ vs. $\Delta\Psi$. Showing %d of %d refls'%(tag,i,bin_low,bin_high,len(a.select(sel)),len(a))) plt.xlabel(r'$\Delta\Psi \circ$') plt.ylabel(r'$\Delta2\Theta \circ$') # Plot delta 2theta vs. 2theta a = reflections['two_theta_obs']#[:71610] b = reflections['two_theta_obs'] - reflections['two_theta_cal'] fig = plt.figure() limits = -0.05, 0.05 sel = (b > limits[0]) & (b < limits[1]) plt.hist2d(a.select(sel), b.select(sel), bins=100, range=((0,50), limits)) plt.clim((0,100)) cb = plt.colorbar() cb.set_label("N reflections") plt.title(r'%s$\Delta2\Theta$ vs. 2$\Theta$. Showing %d of %d refls'%(tag,len(a.select(sel)),len(a))) plt.xlabel(r'2$\Theta \circ$') plt.ylabel(r'$\Delta2\Theta \circ$') # calc the trendline z = np.polyfit(a.select(sel), b.select(sel), 1) print 'y=%.7fx+(%.7f)'%(z[0],z[1]) # Plots with single values per panel self.detector_plot_dict(detector, refl_counts, u"%s N reflections"%t, u"%6d", show=False) self.detector_plot_dict(detector, rmsds, "%s Positional RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, radial_rmsds, "%s Radial RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, transverse_rmsds, "%s Transverse RMSDs (microns)"%t, u"%4.1f", show=False) self.detector_plot_dict(detector, ttdpcorr, r"%s $\Delta2\Theta$ vs. $\Delta\Psi$ CC"%t, u"%5.3f", show=False) self.plot_unitcells(experiments) self.plot_data_by_two_theta(reflections, tag) # Plot data by panel group sorted_values = sorted(pg_bc_dists.values()) vdict = {} for k in pg_bc_dists: vdict[pg_bc_dists[k]] = k sorted_keys = [vdict[v] for v in sorted_values if vdict[v] in rmsds] x = [sorted_values[i] for i in xrange(len(sorted_values)) if pg_bc_dists.keys()[i] in rmsds] self.plot_multi_data(x, [[pg_refls_count_d[k] for k in sorted_keys], ([rmsds[k] for k in sorted_keys], [radial_rmsds[k] for k in sorted_keys], [transverse_rmsds[k] for k in sorted_keys]), [radial_rmsds[k]/transverse_rmsds[k] for k in sorted_keys], [mean_delta_two_theta[k] for k in sorted_keys]], "Panel group distance from beam center (mm)", ["N reflections", ("Overall RMSD", "Radial RMSD", "Transverse RMSD"), "R/T RMSD ratio", "Delta two theta"], ["N reflections", "RMSD (microns)", "R/T RMSD ratio", "Delta two theta (degrees)"], "%sData by panelgroup"%tag) if self.params.save_pdf: pp = PdfPages('residuals_%s.pdf'%(tag.strip())) for i in plt.get_fignums(): pp.savefig(plt.figure(i)) pp.close() else: plt.show()
def test_contraints_manager_simple_test(): x = flex.random_double(10) # constrain parameters 2 and 4 and 6, 7 and 8 c1 = EqualShiftConstraint([1, 3], x) c2 = EqualShiftConstraint([5, 6, 7], x) cm = ConstraintManager([c1, c2], len(x)) constrained_x = cm.constrain_parameters(x) # check the constrained parameters are as expected assert len(constrained_x) == 7 assert constrained_x[5] == flex.mean(x.select([1, 3])) assert constrained_x[6] == flex.mean(x[5:8]) # minimiser would modify the constrained parameters mod_constrained_x = constrained_x + 10.0 # check the expanded parameters are as expected expanded = cm.expand_parameters(mod_constrained_x) assert x + 10.0 == expanded # make a matrix to exercise jacobian compaction j = flex.random_double(20 * 10) j.reshape(flex.grid(20, 10)) # for constrained columns, elements that are non-zero in one column are # zero in the other columns. Enforce that in this example mask2 = flex.bool([True] * 10 + [False] * 10) mask4 = ~mask2 col2 = j.matrix_copy_column(1) col2.set_selected(mask2, 0) j.matrix_paste_column_in_place(col2, 1) col4 = j.matrix_copy_column(3) col4.set_selected(mask4, 0) j.matrix_paste_column_in_place(col4, 3) mask6 = flex.bool([False] * 7 + [True] * 13) mask7 = mask6.reversed() mask8 = ~(mask6 & mask7) col6 = j.matrix_copy_column(5) col6.set_selected(mask6, 0) j.matrix_paste_column_in_place(col6, 5) col7 = j.matrix_copy_column(6) col7.set_selected(mask7, 0) j.matrix_paste_column_in_place(col7, 6) col8 = j.matrix_copy_column(7) col8.set_selected(mask8, 0) j.matrix_paste_column_in_place(col8, 7) cj = cm.constrain_jacobian(j) # check expected dimensions assert cj.all() == (20, 7) # check that the constrained columns are equal to sums of the relevant # columns in the original Jacobian tmp = j.matrix_copy_column(1) + j.matrix_copy_column(3) assert (cj.matrix_copy_column(5) == tmp).all_eq(True) tmp = j.matrix_copy_column(5) + j.matrix_copy_column( 6) + j.matrix_copy_column(7) assert (cj.matrix_copy_column(6) == tmp).all_eq(True) # convert to a sparse matrix to exercise the sparse Jacobian compaction j2 = sparse.matrix(20, 10) mask = flex.bool(20, True) for i, c in enumerate(j2.cols()): c.set_selected(mask, j.matrix_copy_column(i)) assert (j2.as_dense_matrix() == j).all_eq(True) cm2 = SparseConstraintManager([c1, c2], len(x)) cj2 = cm2.constrain_jacobian(j2) # ensure dense and sparse calculations give the same result assert (cj2.as_dense_matrix() == cj).all_eq(True) # construct derivatives of the objective dL/dp from the Jacobian to test # constrain_gradient_vector. Here assume unit weights dL_dp = [sum(col.as_dense_vector()) for col in j2.cols()] constr_dL_dp = cm.constrain_gradient_vector(dL_dp) # check constrained values are equal to sums of relevant elements in the # original gradient vector assert constr_dL_dp[5] == dL_dp[1] + dL_dp[3] assert constr_dL_dp[6] == dL_dp[5] + dL_dp[6] + dL_dp[7]
def __del__(self): values = self.parameterization(self.x) print >> self.out, "FINALMODEL", print >> self.out, "rms %10.3f"%math.sqrt(flex.mean(self.func*self.func)), values.show(self.out)
def overall_report(data): # Start by adding some overall numbers report = collections.OrderedDict() report["n"] = len(reflections) report["n_full"] = data["full"].count(True) report["n_partial"] = data["full"].count(False) report["n_overload"] = data["over"].count(True) report["n_ice"] = data["ice"].count(True) report["n_summed"] = data["sum"].count(True) report["n_fitted"] = data["prf"].count(True) report["n_integated"] = data["int"].count(True) report["n_invalid_bg"] = data["ninvbg"].count(True) report["n_invalid_fg"] = data["ninvfg"].count(True) report["n_failed_background"] = data["fbgd"].count(True) report["n_failed_summation"] = data["fsum"].count(True) report["n_failed_fitting"] = data["fprf"].count(True) # Compute mean background try: report["mean_background"] = flex.mean( data["background.mean"].select(data["int"])) except Exception: report["mean_background"] = 0.0 # Compute mean I/Sigma summation try: report["ios_sum"] = flex.mean(data["intensity.sum.ios"].select( data["sum"])) except Exception: report["ios_sum"] = 0.0 # Compute mean I/Sigma profile fitting try: report["ios_prf"] = flex.mean(data["intensity.prf.ios"].select( data["prf"])) except Exception: report["ios_prf"] = 0.0 # Compute the mean profile correlation try: report["cc_prf"] = flex.mean(data["profile.correlation"].select( data["prf"])) except Exception: report["cc_prf"] = 0.0 # Compute the correlations between summation and profile fitting try: mask = data["sum"] & data["prf"] Isum = data["intensity.sum.value"].select(mask) Iprf = data["intensity.prf.value"].select(mask) report["cc_pearson_sum_prf"] = pearson_correlation_coefficient( Isum, Iprf) report["cc_spearman_sum_prf"] = spearman_correlation_coefficient( Isum, Iprf) except Exception: report["cc_pearson_sum_prf"] = 0.0 report["cc_spearman_sum_prf"] = 0.0 # Return the overall report return report
def integration_concept_detail(self, experiments, reflections, spots,image_number,cb_op_to_primitive,**kwargs): detector = experiments[0].detector crystal = experiments[0].crystal from cctbx.crystal import symmetry c_symmetry = symmetry(space_group = crystal.get_space_group(), unit_cell = crystal.get_unit_cell()) self.image_number = image_number NEAR = 10 pxlsz = detector[0].get_pixel_size() Predicted = self.get_predictions_accounting_for_centering(experiments,reflections,cb_op_to_primitive,**kwargs) FWMOSAICITY = self.inputai.getMosaicity() self.DOMAIN_SZ_ANG = kwargs.get("domain_size_ang", self.__dict__.get("actual",0) ) refineflag = {True:0,False:1}[kwargs.get("domain_size_ang",0)==0] c_symmetry.show_summary(prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f "%(refineflag,FWMOSAICITY,self.DOMAIN_SZ_ANG)) from annlib_ext import AnnAdaptor self.cell = c_symmetry.unit_cell() query = flex.double() print len(self.predicted) for pred in self.predicted: # predicted spot coord in pixels query.append(pred[0]/pxlsz[0]) query.append(pred[1]/pxlsz[1]) self.reserve_hkllist_for_signal_search = self.hkllist reference = flex.double() assert self.length>NEAR# Can't do spot/pred matching with too few spots for spot in spots: reference.append(spot.ctr_mass_x()) reference.append(spot.ctr_mass_y()) IS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR) IS_adapt.query(query) idx_cutoff = float(min(self.mask_focus[image_number])) from rstbx.apps.slip_helpers import slip_callbacks cache_refinement_spots = getattr(slip_callbacks.slip_callback,"requires_refinement_spots",False) indexed_pairs_provisional = [] correction_vectors_provisional = [] c_v_p_flex = flex.vec3_double() this_setting_matched_indices = reflections["miller_index"] for j,item in enumerate(this_setting_matched_indices): this_setting_index = self.hkllist.first_index(item) if this_setting_index: Match = dict(spot=j,pred=this_setting_index) indexed_pairs_provisional.append(Match) vector = matrix.col( [reflections["xyzobs.px.value"][j][0] - self.predicted[Match["pred"]][0]/pxlsz[0], reflections["xyzobs.px.value"][j][1] - self.predicted[Match["pred"]][1]/pxlsz[1]]) correction_vectors_provisional.append(vector) c_v_p_flex.append((vector[0],vector[1],0.)) self.N_correction_vectors = len(correction_vectors_provisional) self.rmsd_px = math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))) print "... %d provisional matches"%self.N_correction_vectors, print "r.m.s.d. in pixels: %6.3f"%(self.rmsd_px) if self.horizons_phil.integration.enable_residual_scatter: from matplotlib import pyplot as plt fig = plt.figure() for cv in correction_vectors_provisional: plt.plot([cv[1]],[-cv[0]],"r.") plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt,fig,"res") plt.close() if self.horizons_phil.integration.enable_residual_map: from matplotlib import pyplot as plt PX = reflections["xyzobs.px.value"] fig = plt.figure() for match,cv in zip(indexed_pairs_provisional,correction_vectors_provisional): plt.plot([PX[match["spot"]][1]],[-PX[match["spot"]][0]],"r.") plt.plot([self.predicted[match["pred"]][1]/pxlsz[1]],[-self.predicted[match["pred"]][0]/pxlsz[0]],"g.") plt.plot([PX[match["spot"]][1], PX[match["spot"]][1] + 10.*cv[1]], [-PX[match["spot"]][0], -PX[match["spot"]][0] - 10.*cv[0]],'r-') if kwargs.get("user-reentrant") != None and self.horizons_phil.integration.spot_prediction == "dials" \ and self.horizons_phil.integration.enable_residual_map_deltapsi: from rstbx.apps.stills.util import residual_map_special_deltapsi_add_on residual_map_special_deltapsi_add_on( reflections = self.dials_spot_prediction, matches = indexed_pairs_provisional, experiments=experiments, hkllist = self.hkllist, predicted = self.predicted, plot=plt, eta_deg=FWMOSAICITY, deff=self.DOMAIN_SZ_ANG ) plt.xlim([0,detector[0].get_image_size()[1]]) plt.ylim([-detector[0].get_image_size()[0],0]) plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt,fig,"map") plt.close() indexed_pairs = indexed_pairs_provisional correction_vectors = correction_vectors_provisional ########### skip outlier rejection for this derived class ### However must retain the ability to write out correction vectiors. if True: # at Aaron's request; test later correction_lengths = flex.double([v.length() for v in correction_vectors_provisional]) clorder = flex.sort_permutation(correction_lengths) sorted_cl = correction_lengths.select(clorder) indexed_pairs = [] correction_vectors = [] self.correction_vectors = [] for icand in xrange(len(sorted_cl)): # somewhat arbitrary sigma = 1.0 cutoff for outliers indexed_pairs.append(indexed_pairs_provisional[clorder[icand]]) correction_vectors.append(correction_vectors_provisional[clorder[icand]]) if cache_refinement_spots: self.spotfinder.images[self.frame_numbers[self.image_number]]["refinement_spots"].append( spots[reflections[indexed_pairs[-1]["spot"]]['spotfinder_lookup']]) if kwargs.get("verbose_cv")==True: print "CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f"%( float(self.inputpd["size1"])/2.,float(self.inputpd["size2"])/2., self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1]), print "OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f"%( reflections[indexed_pairs[-1]["spot"]]['xyzobs.px.value'][0], reflections[indexed_pairs[-1]["spot"]]['xyzobs.px.value'][1], self.predicted[indexed_pairs[-1]["pred"]][0]/pxlsz[0], self.predicted[indexed_pairs[-1]["pred"]][1]/pxlsz[1]), the_hkl = self.hkllist[indexed_pairs[-1]["pred"]] print "HKL %4d %4d %4d"%the_hkl,"%2d"%self.setting_id, radial, azimuthal = spots[indexed_pairs[-1]["spot"]].get_radial_and_azimuthal_size( self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1]) print "RADIALpx %5.3f AZIMUTpx %5.3f"%(radial,azimuthal) # Store a list of correction vectors in self. radial, azimuthal = spots[indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size( self.inputai.xbeam()/pxlsz[0], self.inputai.ybeam()/pxlsz[1]) self.correction_vectors.append( dict(obscenter=(float(self.inputpd['size1']) / 2, float(self.inputpd['size2']) / 2), refinedcenter=(self.inputai.xbeam() / pxlsz[0], self.inputai.ybeam() / pxlsz[1]), obsspot=(reflections[indexed_pairs[-1]['spot']]['xyzobs.px.value'][0], reflections[indexed_pairs[-1]['spot']]['xyzobs.px.value'][1]), predspot=(self.predicted[indexed_pairs[-1]['pred']][0] / pxlsz[0], self.predicted[indexed_pairs[-1]['pred']][1] / pxlsz[1]), hkl=(self.hkllist[indexed_pairs[-1]['pred']][0], self.hkllist[indexed_pairs[-1]['pred']][1], self.hkllist[indexed_pairs[-1]['pred']][2]), setting_id=self.setting_id, radial=radial, azimuthal=azimuthal)) self.inputpd["symmetry"] = c_symmetry self.inputpd["symmetry"].show_summary(prefix="SETTING ") if self.horizons_phil.integration.model == "user_supplied": # Not certain of whether the reentrant_* dictionary keys create a memory leak if kwargs.get("user-reentrant",None)==None: kwargs["reentrant_experiments"] = experiments kwargs["reentrant_reflections"] = reflections from cxi_user import post_outlier_rejection self.indexed_pairs = indexed_pairs self.spots = spots post_outlier_rejection(self,image_number,cb_op_to_primitive,self.horizons_phil,kwargs) return ########### finished with user-supplied code correction_lengths=flex.double([v.length() for v in correction_vectors]) self.r_residual = pxlsz[0]*flex.mean(correction_lengths) #assert len(indexed_pairs)>NEAR # must have enough indexed spots if (len(indexed_pairs) <= NEAR): raise Sorry("Not enough indexed spots, only found %d, need %d" % (len(indexed_pairs), NEAR)) reference = flex.double() for item in indexed_pairs: reference.append(spots[item["spot"]].ctr_mass_x()) reference.append(spots[item["spot"]].ctr_mass_y()) PS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR) PS_adapt.query(query) self.BSmasks = [] # do not use null: self.null_correction_mapping( predicted=self.predicted, self.positional_correction_mapping( predicted=self.predicted, correction_vectors = correction_vectors, PS_adapt = PS_adapt, IS_adapt = IS_adapt, spots = spots) # which spots are close enough to interfere with background? MAXOVER=6 OS_adapt = AnnAdaptor(data=query,dim=2,k=MAXOVER) #six near nbrs OS_adapt.query(query) if self.mask_focus[image_number] is None: raise Sorry("No observed/predicted spot agreement; no Spotfinder masks; skip integration") nbr_cutoff = 2.0* max(self.mask_focus[image_number]) FRAME = int(nbr_cutoff/2) #print "The overlap cutoff is %d pixels"%nbr_cutoff nbr_cutoff_sq = nbr_cutoff * nbr_cutoff #print "Optimized C++ section...", self.set_frame(FRAME) self.set_background_factor(kwargs["background_factor"]) self.set_nbr_cutoff_sq(nbr_cutoff_sq) self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq) self.set_detector_gain(self.horizons_phil.integration.detector_gain) flex_sorted = flex.int() for item in self.sorted: flex_sorted.append(item[0]);flex_sorted.append(item[1]); if self.horizons_phil.integration.mask_pixel_value is not None: self.set_mask_pixel_val(self.horizons_phil.integration.mask_pixel_value) image_obj = self.imagefiles.imageindex(self.frame_numbers[self.image_number]) image_obj.read() rawdata = image_obj.linearintdata # assume image #1 if self.inputai.active_areas != None: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted, tiles=self.inputai.active_areas.IT, tile_id=self.inputai.active_areas.tile_id); else: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted); for i in xrange(len(self.predicted)): # loop over predicteds B_S_mask = {} keys = self.get_bsmask(i) for k in xrange(0,len(keys),2): B_S_mask[(keys[k],keys[k+1])]=True self.BSmasks.append(B_S_mask) #print "Done" return
def __call__(self): """Determine optimal mosaicity and domain size model (monochromatic)""" RR = self.refinery.predict_for_reflection_table(self.reflections) excursion_rad = RR["delpsical.rad"] delta_psi_deg = excursion_rad * 180. / math.pi print print flex.max(delta_psi_deg), flex.min(delta_psi_deg) mean_excursion = flex.mean(delta_psi_deg) print "The mean excursion is %7.3f degrees, r.m.s.d %7.3f" % ( mean_excursion, math.sqrt(flex.mean(RR["delpsical2"]))) crystal = self.experiments[0].crystal beam = self.experiments[0].beam miller_indices = self.reflections["miller_index"] # FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection two_thetas = crystal.get_unit_cell().two_theta(miller_indices, beam.get_wavelength(), deg=True) dspacings = crystal.get_unit_cell().d(miller_indices) dspace_sq = dspacings * dspacings # First -- try to get a reasonable envelope for the observed excursions. ## minimum of three regions; maximum of 50 measurements in each bin print "fitting parameters on %d spots" % len(excursion_rad) n_bins = min(max(3, len(excursion_rad) // 25), 50) bin_sz = len(excursion_rad) // n_bins print "nbins", n_bins, "bin_sz", bin_sz order = flex.sort_permutation(two_thetas) two_thetas_env = flex.double() dspacings_env = flex.double() excursion_rads_env = flex.double() for x in xrange(0, n_bins): subset = order[x * bin_sz:(x + 1) * bin_sz] two_thetas_env.append(flex.mean(two_thetas.select(subset))) dspacings_env.append(flex.mean(dspacings.select(subset))) excursion_rads_env.append( flex.max(flex.abs(excursion_rad.select(subset)))) # Second -- parameter fit ## solve the normal equations sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env) sum_inv_u = flex.sum(dspacings_env) sum_te_u = flex.sum(dspacings_env * excursion_rads_env) sum_te = flex.sum(excursion_rads_env) Normal_Mat = sqr( (sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env))) Vector = col((sum_te_u, sum_te)) solution = Normal_Mat.inverse() * Vector s_ang = 1. / (2 * solution[0]) print "Best LSQ fit Scheerer domain size is %9.2f ang" % (s_ang) tan_phi_rad = dspacings / (2. * s_ang) tan_phi_deg = tan_phi_rad * 180. / math.pi k_degrees = solution[1] * 180. / math.pi print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f" % ( 2 * k_degrees, k_degrees) tan_outer_deg = tan_phi_deg + k_degrees from xfel.mono_simulation.max_like import minimizer # coerce the estimates to be positive for max-likelihood lower_limit_domain_size = math.pow( crystal.get_unit_cell().volume(), 1. / 3.) * 3 # params.refinement.domain_size_lower_limit d_estimate = max(s_ang, lower_limit_domain_size) M = minimizer(d_i=dspacings, psi_i=excursion_rad, eta_rad=abs(2. * solution[1]), Deff=d_estimate) print "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" % ( M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas)) tan_phi_rad_ML = dspacings / (2. / M.x[0]) tan_phi_deg_ML = tan_phi_rad_ML * 180. / math.pi tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180. / math.pi self.nv_acceptance_flags = flex.abs(delta_psi_deg) < tan_outer_deg_ML if self.graph_verbose: #params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit AD1TF7B_MAX2T = 30. AD1TF7B_MAXDP = 1. from matplotlib import pyplot as plt plt.plot(two_thetas, delta_psi_deg, "bo") minplot = flex.min(two_thetas) plt.plot([0, minplot], [mean_excursion, mean_excursion], "k-") LR = flex.linear_regression(two_thetas, delta_psi_deg) model_y = LR.slope() * two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" % (M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas))) plt.plot(two_thetas, tan_phi_deg_ML, "r.") plt.plot(two_thetas, -tan_phi_deg_ML, "r.") plt.plot(two_thetas, tan_outer_deg_ML, "g.") plt.plot(two_thetas, -tan_outer_deg_ML, "g.") plt.xlim([0, AD1TF7B_MAX2T]) plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP]) plt.show() plt.close() from xfel.mono_simulation.util import green_curve_area self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML) print "The green curve area is ", self.green_curve_area crystal._ML_half_mosaicity_deg = M.x[1] * 180. / (2. * math.pi) crystal._ML_domain_size_ang = 2. / M.x[0] self._ML_full_mosaicity_rad = M.x[1] self._ML_domain_size_ang = 2. / M.x[0] #params.refinement.mosaic.model_expansion_factor """The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes as close as possible to # of observed reflections input, in the last integration call. Determine this by inspecting the output log file interactively. Do not exceed the bare minimum threshold needed. The intention is to find an optimal value, global for a given dataset.""" model_expansion_factor = 1.4 crystal._ML_half_mosaicity_deg *= model_expansion_factor crystal._ML_domain_size_ang /= model_expansion_factor return crystal
def abs_bounding_lines_in_mm(self, detector): """Return bounding lines of kapton""" # first get bounding directions from detector: detz = flex.mean(flex.double([panel.get_origin()[2] for panel in detector])) edges = [] for ii, panel in enumerate(detector): f_size, s_size = panel.get_image_size() for point in [(0, 0), (0, s_size), (f_size, 0), (f_size, s_size)]: x, y = panel.get_pixel_lab_coord(point)[0:2] edges.append((x, y, detz)) # Use the idea that the corners of the detector are end points of the diagonal and will be the # top 2 max dimension among all end points dlist = flex.double() dlist_idx = [] n_edges = len(edges) for ii in range(n_edges - 1): for jj in range(ii + 1, n_edges): pt_1 = col(edges[ii]) pt_2 = col(edges[jj]) distance = (pt_1 - pt_2).length() dlist.append(distance) dlist_idx.append((ii, jj)) sorted_idx = flex.sort_permutation(dlist, reverse=True) edge_pts = [ edges[dlist_idx[sorted_idx[0]][0]], edges[dlist_idx[sorted_idx[1]][0]], edges[dlist_idx[sorted_idx[0]][1]], edges[dlist_idx[sorted_idx[1]][1]], ] self.detector_edges = edge_pts # Now get the maximum extent of the intersection of the rays with the detector all_ints = [] kapton_path_list = [] for ii, edge_point in enumerate(self.edge_points): s1 = edge_point.normalize() kapton_path_mm = self.get_kapton_path_mm(s1) for panel in detector: try: x_int, y_int = panel.get_lab_coord(panel.get_ray_intersection(s1))[ 0:2 ] except RuntimeError: pass int_point = (x_int, y_int, detz) # Arbitrary tolerance of couple of pixels otherwise these points were getting clustered together tolerance = min(panel.get_pixel_size()) * 2.0 if ( sum( (col(trial_pt) - col(int_point)).length() <= tolerance for trial_pt in all_ints ) == 0 ): all_ints.append(int_point) kapton_path_list.append(kapton_path_mm) # Use the idea that the extreme edges of the intersection points are end points of the diagonal and will be the # top 2 max dimension among all end points dlist = flex.double() dlist_idx = [] n_edges = len(all_ints) for ii in range(n_edges - 1): pt_1 = col(all_ints[ii]) for jj in range(ii + 1, n_edges): pt_2 = col(all_ints[jj]) distance = (pt_1 - pt_2).length() dlist.append(distance) dlist_idx.append((ii, jj)) sorted_idx = flex.sort_permutation(dlist, reverse=True) int_edge_pts = [ all_ints[dlist_idx[sorted_idx[0]][0]], all_ints[dlist_idx[sorted_idx[1]][0]], all_ints[dlist_idx[sorted_idx[0]][1]], all_ints[dlist_idx[sorted_idx[1]][1]], ] # Sort out the edge points and the int_edge_points which are on the same side kapton_edge_1 = (col(int_edge_pts[0]) - col(int_edge_pts[1])).normalize() kapton_edge_2 = (col(int_edge_pts[2]) - col(int_edge_pts[3])).normalize() min_loss_func = -999.9 edge_idx = None for edge_idx_combo in [(0, 1, 2, 3), (0, 3, 1, 2)]: side_1 = ( col(edge_pts[edge_idx_combo[0]]) - col(edge_pts[edge_idx_combo[1]]) ).normalize() side_2 = ( col(edge_pts[edge_idx_combo[2]]) - col(edge_pts[edge_idx_combo[3]]) ).normalize() loss_func = abs(kapton_edge_1.dot(side_1)) + abs(kapton_edge_2.dot(side_2)) if loss_func > min_loss_func: edge_idx = edge_idx_combo min_loss_func = loss_func # Make sure the edges of the detector and the kapton are in the same orientation # first for kapton edge 1 side_1 = (col(edge_pts[edge_idx[0]]) - col(edge_pts[edge_idx[1]])).normalize() side_2 = (col(edge_pts[edge_idx[2]]) - col(edge_pts[edge_idx[3]])).normalize() v1 = kapton_edge_1.dot(side_1) v2 = kapton_edge_2.dot(side_2) if v1 < 0.0: edge_idx = (edge_idx[1], edge_idx[0], edge_idx[2], edge_idx[3]) if v2 < 0.0: edge_idx = (edge_idx[0], edge_idx[1], edge_idx[3], edge_idx[2]) # Now make sure the edges and the kapton lines are on the right side (i.e not swapped). # Let's look at edge_idx[0:2] i,e the first edge of detector parallel to the kapton pt1 = edge_pts[edge_idx[0]] pt2 = edge_pts[edge_idx[1]] # Now find the distance between each of these points and the kapton lines. d1_kapton_1 = self.distance_of_point_from_line( pt1, int_edge_pts[0], int_edge_pts[1] ) d1_kapton_2 = self.distance_of_point_from_line( pt1, int_edge_pts[2], int_edge_pts[3] ) d2_kapton_1 = self.distance_of_point_from_line( pt2, int_edge_pts[0], int_edge_pts[1] ) d2_kapton_2 = self.distance_of_point_from_line( pt2, int_edge_pts[2], int_edge_pts[3] ) if d1_kapton_1 < d1_kapton_2: # closer to max than edge assert ( d2_kapton_1 < d2_kapton_2 ), "Distance mismatch. Edge of detector might be on wrong side of kapton tape ... please check" pair_values = [ ( edge_pts[edge_idx[0]], edge_pts[edge_idx[1]], edge_pts[edge_idx[2]], edge_pts[edge_idx[3]], ), (int_edge_pts[0], int_edge_pts[1], int_edge_pts[2], int_edge_pts[3]), ] else: pair_values = [ ( edge_pts[edge_idx[0]], edge_pts[edge_idx[1]], edge_pts[edge_idx[2]], edge_pts[edge_idx[3]], ), (int_edge_pts[2], int_edge_pts[3], int_edge_pts[0], int_edge_pts[1]), ] return pair_values
def __call__(self, index): """ Extract strong pixels from an image :param index: The index of the image """ # Get the frame number if isinstance(self.imageset, ImageSequence): frame = self.imageset.get_array_range()[0] + index else: ind = self.imageset.indices() if len(ind) > 1: assert all(i1 + 1 == i2 for i1, i2 in zip(ind[0:-1], ind[1:-1])) frame = ind[index] # Create the list of pixel lists pixel_list = [] # Get the image and mask image = self.imageset.get_corrected_data(index) mask = self.imageset.get_mask(index) # Set the mask if self.mask is not None: assert len(self.mask) == len(mask) mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask)) logger.debug( "Number of masked pixels for image %i: %i", index, sum(m.count(False) for m in mask), ) # Add the images to the pixel lists num_strong = 0 average_background = 0 for im, mk in zip(image, mask): if self.region_of_interest is not None: x0, x1, y0, y1 = self.region_of_interest height, width = im.all() assert x0 < x1, "x0 < x1" assert y0 < y1, "y0 < y1" assert x0 >= 0, "x0 >= 0" assert y0 >= 0, "y0 >= 0" assert x1 <= width, "x1 <= width" assert y1 <= height, "y1 <= height" im_roi = im[y0:y1, x0:x1] mk_roi = mk[y0:y1, x0:x1] tm_roi = self.threshold_function.compute_threshold( im_roi, mk_roi) threshold_mask = flex.bool(im.accessor(), False) threshold_mask[y0:y1, x0:x1] = tm_roi else: threshold_mask = self.threshold_function.compute_threshold( im, mk) # Add the pixel list plist = PixelList(frame, im, threshold_mask) pixel_list.append(plist) # Get average background if self.compute_mean_background: background = im.as_1d().select((mk & ~threshold_mask).as_1d()) average_background += flex.mean(background) # Add to the spot count num_strong += len(plist) # Make average background average_background /= len(image) # Check total number of strong pixels if self.max_strong_pixel_fraction < 1: num_image = 0 for im in image: num_image += len(im) max_strong = int( math.ceil(self.max_strong_pixel_fraction * num_image)) if num_strong > max_strong: raise RuntimeError(f""" The number of strong pixels found ({num_strong}) is greater than the maximum allowed ({max_strong}). Try changing spot finding parameters """) # Print some info if self.compute_mean_background: logger.info( "Found %d strong pixels on image %d with average background %f", num_strong, frame + 1, average_background, ) else: logger.info("Found %d strong pixels on image %d", num_strong, frame + 1) # Return the result return pixel_list
def plot_one_model(self,nrow,out): fig = plt.subplot(self.gs[nrow*self.ncols]) two_thetas = self.reduction.get_two_theta_deg() degrees = self.reduction.get_delta_psi_deg() if self.color_encoding=="conventional": positive = (self.reduction.i_sigi>=0.) fig.plot(two_thetas.select(positive), degrees.select(positive), "bo") fig.plot(two_thetas.select(~positive), degrees.select(~positive), "r+") elif self.color_encoding=="I/sigma": positive = (self.reduction.i_sigi>=0.) tt_selected = two_thetas.select(positive) dp_selected = degrees.select(positive) i_sigi_select = self.reduction.i_sigi.select(positive) order = flex.sort_permutation(i_sigi_select) tt_selected = tt_selected.select(order) dp_selected = dp_selected.select(order) i_sigi_selected = i_sigi_select.select(order) from matplotlib.colors import Normalize dnorm = Normalize() dcolors = i_sigi_selected.as_numpy_array() dnorm.autoscale(dcolors) N = len(dcolors) CMAP = plt.get_cmap("rainbow") if self.refined.get("partiality_array",None) is None: for n in xrange(N): fig.plot([tt_selected[n]],[dp_selected[n]], color=CMAP(dnorm(dcolors[n])),marker=".", markersize=10) else: partials = self.refined.get("partiality_array") partials_select = partials.select(positive) partials_selected = partials_select.select(order) assert len(partials)==len(positive) for n in xrange(N): fig.plot([tt_selected[n]],[dp_selected[n]], color=CMAP(dnorm(dcolors[n])),marker=".", markersize=20*partials_selected[n]) # change the markersize to indicate partiality. negative = (self.reduction.i_sigi<0.) fig.plot(two_thetas.select(negative), degrees.select(negative), "r+", linewidth=1) else: strong = (self.reduction.i_sigi>=10.) positive = ((~strong) & (self.reduction.i_sigi>=0.)) negative = (self.reduction.i_sigi<0.) assert (strong.count(True)+positive.count(True)+negative.count(True) == len(self.reduction.i_sigi)) fig.plot(two_thetas.select(positive), degrees.select(positive), "bo") fig.plot(two_thetas.select(strong), degrees.select(strong), marker='.',linestyle='None', markerfacecolor='#00ee00', markersize=10) fig.plot(two_thetas.select(negative), degrees.select(negative), "r+") # indicate the imposed resolution filter wavelength = self.reduction.experiment.beam.get_wavelength() imposed_res_filter = self.reduction.get_imposed_res_filter(out) resolution_markers = [ a for a in [imposed_res_filter,self.reduction.measurements.d_min()] if a is not None] for RM in resolution_markers: two_th = (180./math.pi)*2.*math.asin(wavelength/(2.*RM)) plt.plot([two_th, two_th],[self.AD1TF7B_MAXDP*-0.8,self.AD1TF7B_MAXDP*0.8],'k-') plt.text(two_th,self.AD1TF7B_MAXDP*-0.9,"%4.2f"%RM) #indicate the linefit mean = flex.mean(degrees) minplot = flex.min(two_thetas) plt.plot([0,minplot],[mean,mean],"k-") LR = flex.linear_regression(two_thetas, degrees) model_y = LR.slope()*two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") #Now let's take care of the red and green lines. half_mosaic_rotation_deg = self.refined["half_mosaic_rotation_deg"] mosaic_domain_size_ang = self.refined["mosaic_domain_size_ang"] red_curve_domain_size_ang = self.refined.get("red_curve_domain_size_ang",mosaic_domain_size_ang) a_step = self.AD1TF7B_MAX2T / 50. a_range = flex.double([a_step*x for x in xrange(1,50)]) # domain two-theta array #Bragg law [d=L/2sinTH] d_spacing = (wavelength/(2.*flex.sin(math.pi*a_range/360.))) # convert two_theta to a delta-psi. Formula for Deffective [Dpsi=d/2Deff] inner_phi_deg = flex.asin((d_spacing / (2.*red_curve_domain_size_ang)) )*(180./math.pi) outer_phi_deg = flex.asin((d_spacing / (2.*mosaic_domain_size_ang)) + \ half_mosaic_rotation_deg*math.pi/180. )*(180./math.pi) plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots\n%s"%( 2.*half_mosaic_rotation_deg, mosaic_domain_size_ang, len(two_thetas), os.path.basename(self.reduction.filename))) plt.plot(a_range, inner_phi_deg, "r-") plt.plot(a_range,-inner_phi_deg, "r-") plt.plot(a_range, outer_phi_deg, "g-") plt.plot(a_range, -outer_phi_deg, "g-") plt.xlim([0,self.AD1TF7B_MAX2T]) plt.ylim([-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP]) #second plot shows histogram fig = plt.subplot(self.gs[1+nrow*self.ncols]) plt.xlim([-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP]) nbins = 50 n,bins,patches = plt.hist(dp_selected, nbins, range=(-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP), weights=self.reduction.i_sigi.select(positive), normed=0, facecolor="orange", alpha=0.75) #ersatz determine the median i_sigi point: isi_positive = self.reduction.i_sigi.select(positive) isi_order = flex.sort_permutation(isi_positive) reordered = isi_positive.select(isi_order) isi_median = reordered[int(len(isi_positive)*0.9)] isi_top_half_selection = (isi_positive>isi_median) n,bins,patches = plt.hist(dp_selected.select(isi_top_half_selection), nbins, range=(-self.AD1TF7B_MAXDP,self.AD1TF7B_MAXDP), weights=isi_positive.select(isi_top_half_selection), normed=0, facecolor="#ff0000", alpha=0.75) plt.xlabel("(degrees)") plt.title("Weighted histogram of Delta-psi")
def integration_concept_detail(self, experiments, reflections, spots, image_number, cb_op_to_primitive, **kwargs): detector = experiments[0].detector crystal = experiments[0].crystal from cctbx.crystal import symmetry c_symmetry = symmetry(space_group=crystal.get_space_group(), unit_cell=crystal.get_unit_cell()) self.image_number = image_number NEAR = 10 pxlsz = detector[0].get_pixel_size() Predicted = self.get_predictions_accounting_for_centering( experiments, reflections, cb_op_to_primitive, **kwargs) FWMOSAICITY = self.inputai.getMosaicity() self.DOMAIN_SZ_ANG = kwargs.get("domain_size_ang", self.__dict__.get("actual", 0)) refineflag = {True: 0, False: 1}[kwargs.get("domain_size_ang", 0) == 0] c_symmetry.show_summary( prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f " % (refineflag, FWMOSAICITY, self.DOMAIN_SZ_ANG)) from annlib_ext import AnnAdaptor self.cell = c_symmetry.unit_cell() query = flex.double() print len(self.predicted) for pred in self.predicted: # predicted spot coord in pixels query.append(pred[0] / pxlsz[0]) query.append(pred[1] / pxlsz[1]) self.reserve_hkllist_for_signal_search = self.hkllist reference = flex.double() assert self.length > NEAR # Can't do spot/pred matching with too few spots for spot in spots: reference.append(spot.ctr_mass_x()) reference.append(spot.ctr_mass_y()) IS_adapt = AnnAdaptor(data=reference, dim=2, k=NEAR) IS_adapt.query(query) idx_cutoff = float(min(self.mask_focus[image_number])) from rstbx.apps.slip_helpers import slip_callbacks cache_refinement_spots = getattr(slip_callbacks.slip_callback, "requires_refinement_spots", False) indexed_pairs_provisional = [] correction_vectors_provisional = [] c_v_p_flex = flex.vec3_double() this_setting_matched_indices = reflections["miller_index"] for j, item in enumerate(this_setting_matched_indices): this_setting_index = self.hkllist.first_index(item) if this_setting_index: Match = dict(spot=j, pred=this_setting_index) indexed_pairs_provisional.append(Match) vector = matrix.col([ reflections["xyzobs.px.value"][j][0] - self.predicted[Match["pred"]][0] / pxlsz[0], reflections["xyzobs.px.value"][j][1] - self.predicted[Match["pred"]][1] / pxlsz[1] ]) correction_vectors_provisional.append(vector) c_v_p_flex.append((vector[0], vector[1], 0.)) self.N_correction_vectors = len(correction_vectors_provisional) self.rmsd_px = math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))) print "... %d provisional matches" % self.N_correction_vectors, print "r.m.s.d. in pixels: %6.3f" % (self.rmsd_px) if self.horizons_phil.integration.enable_residual_scatter: from matplotlib import pyplot as plt fig = plt.figure() for cv in correction_vectors_provisional: plt.plot([cv[1]], [-cv[0]], "r.") plt.title(" %d matches, r.m.s.d. %5.2f pixels" % (len(correction_vectors_provisional), math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt, fig, "res") plt.close() if self.horizons_phil.integration.enable_residual_map: from matplotlib import pyplot as plt PX = reflections["xyzobs.px.value"] fig = plt.figure() for match, cv in zip(indexed_pairs_provisional, correction_vectors_provisional): plt.plot([PX[match["spot"]][1]], [-PX[match["spot"]][0]], "r.") plt.plot([self.predicted[match["pred"]][1] / pxlsz[1]], [-self.predicted[match["pred"]][0] / pxlsz[0]], "g.") plt.plot( [PX[match["spot"]][1], PX[match["spot"]][1] + 10. * cv[1]], [ -PX[match["spot"]][0], -PX[match["spot"]][0] - 10. * cv[0] ], 'r-') if kwargs.get("user-reentrant") != None and self.horizons_phil.integration.spot_prediction == "dials" \ and self.horizons_phil.integration.enable_residual_map_deltapsi: from rstbx.apps.stills.util import residual_map_special_deltapsi_add_on residual_map_special_deltapsi_add_on( reflections=self.dials_spot_prediction, matches=indexed_pairs_provisional, experiments=experiments, hkllist=self.hkllist, predicted=self.predicted, plot=plt, eta_deg=FWMOSAICITY, deff=self.DOMAIN_SZ_ANG) plt.xlim([0, detector[0].get_image_size()[1]]) plt.ylim([-detector[0].get_image_size()[0], 0]) plt.title(" %d matches, r.m.s.d. %5.2f pixels" % (len(correction_vectors_provisional), math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt, fig, "map") plt.close() indexed_pairs = indexed_pairs_provisional correction_vectors = correction_vectors_provisional ########### skip outlier rejection for this derived class ### However must retain the ability to write out correction vectiors. if True: # at Aaron's request; test later correction_lengths = flex.double( [v.length() for v in correction_vectors_provisional]) clorder = flex.sort_permutation(correction_lengths) sorted_cl = correction_lengths.select(clorder) indexed_pairs = [] correction_vectors = [] self.correction_vectors = [] for icand in xrange(len(sorted_cl)): # somewhat arbitrary sigma = 1.0 cutoff for outliers indexed_pairs.append(indexed_pairs_provisional[clorder[icand]]) correction_vectors.append( correction_vectors_provisional[clorder[icand]]) if cache_refinement_spots: self.spotfinder.images[self.frame_numbers[ self.image_number]]["refinement_spots"].append( spots[reflections[indexed_pairs[-1]["spot"]] ['spotfinder_lookup']]) if kwargs.get("verbose_cv") == True: print "CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f" % ( float(self.inputpd["size1"]) / 2., float(self.inputpd["size2"]) / 2., self.inputai.xbeam() / pxlsz[0], self.inputai.ybeam() / pxlsz[1]), print "OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f" % ( reflections[indexed_pairs[-1]["spot"]] ['xyzobs.px.value'][0], reflections[ indexed_pairs[-1]["spot"]]['xyzobs.px.value'][1], self.predicted[indexed_pairs[-1]["pred"]][0] / pxlsz[0], self.predicted[indexed_pairs[-1]["pred"]][1] / pxlsz[1]), the_hkl = self.hkllist[indexed_pairs[-1]["pred"]] print "HKL %4d %4d %4d" % the_hkl, "%2d" % self.setting_id, radial, azimuthal = spots[indexed_pairs[-1][ "spot"]].get_radial_and_azimuthal_size( self.inputai.xbeam() / pxlsz[0], self.inputai.ybeam() / pxlsz[1]) print "RADIALpx %5.3f AZIMUTpx %5.3f" % (radial, azimuthal) # Store a list of correction vectors in self. radial, azimuthal = spots[ indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size( self.inputai.xbeam() / pxlsz[0], self.inputai.ybeam() / pxlsz[1]) self.correction_vectors.append( dict(obscenter=(float(self.inputpd['size1']) / 2, float(self.inputpd['size2']) / 2), refinedcenter=(self.inputai.xbeam() / pxlsz[0], self.inputai.ybeam() / pxlsz[1]), obsspot=(reflections[indexed_pairs[-1] ['spot']]['xyzobs.px.value'][0], reflections[indexed_pairs[-1] ['spot']]['xyzobs.px.value'][1]), predspot=( self.predicted[indexed_pairs[-1]['pred']][0] / pxlsz[0], self.predicted[indexed_pairs[-1]['pred']][1] / pxlsz[1]), hkl=(self.hkllist[indexed_pairs[-1]['pred']][0], self.hkllist[indexed_pairs[-1]['pred']][1], self.hkllist[indexed_pairs[-1]['pred']][2]), setting_id=self.setting_id, radial=radial, azimuthal=azimuthal)) self.inputpd["symmetry"] = c_symmetry self.inputpd["symmetry"].show_summary(prefix="SETTING ") if self.horizons_phil.integration.model == "user_supplied": # Not certain of whether the reentrant_* dictionary keys create a memory leak if kwargs.get("user-reentrant", None) == None: kwargs["reentrant_experiments"] = experiments kwargs["reentrant_reflections"] = reflections from cxi_user import post_outlier_rejection self.indexed_pairs = indexed_pairs self.spots = spots post_outlier_rejection(self, image_number, cb_op_to_primitive, self.horizons_phil, kwargs) return ########### finished with user-supplied code correction_lengths = flex.double( [v.length() for v in correction_vectors]) self.r_residual = pxlsz[0] * flex.mean(correction_lengths) #assert len(indexed_pairs)>NEAR # must have enough indexed spots if (len(indexed_pairs) <= NEAR): raise Sorry("Not enough indexed spots, only found %d, need %d" % (len(indexed_pairs), NEAR)) reference = flex.double() for item in indexed_pairs: reference.append(spots[item["spot"]].ctr_mass_x()) reference.append(spots[item["spot"]].ctr_mass_y()) PS_adapt = AnnAdaptor(data=reference, dim=2, k=NEAR) PS_adapt.query(query) self.BSmasks = [] # do not use null: self.null_correction_mapping( predicted=self.predicted, self.positional_correction_mapping( predicted=self.predicted, correction_vectors=correction_vectors, PS_adapt=PS_adapt, IS_adapt=IS_adapt, spots=spots) # which spots are close enough to interfere with background? MAXOVER = 6 OS_adapt = AnnAdaptor(data=query, dim=2, k=MAXOVER) #six near nbrs OS_adapt.query(query) if self.mask_focus[image_number] is None: raise Sorry( "No observed/predicted spot agreement; no Spotfinder masks; skip integration" ) nbr_cutoff = 2.0 * max(self.mask_focus[image_number]) FRAME = int(nbr_cutoff / 2) #print "The overlap cutoff is %d pixels"%nbr_cutoff nbr_cutoff_sq = nbr_cutoff * nbr_cutoff #print "Optimized C++ section...", self.set_frame(FRAME) self.set_background_factor(kwargs["background_factor"]) self.set_nbr_cutoff_sq(nbr_cutoff_sq) self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq) self.set_detector_gain(self.horizons_phil.integration.detector_gain) flex_sorted = flex.int() for item in self.sorted: flex_sorted.append(item[0]) flex_sorted.append(item[1]) if self.horizons_phil.integration.mask_pixel_value is not None: self.set_mask_pixel_val( self.horizons_phil.integration.mask_pixel_value) image_obj = self.imagefiles.imageindex( self.frame_numbers[self.image_number]) image_obj.read() rawdata = image_obj.linearintdata # assume image #1 if self.inputai.active_areas != None: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted, tiles=self.inputai.active_areas.IT, tile_id=self.inputai.active_areas.tile_id) else: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted) for i in xrange(len(self.predicted)): # loop over predicteds B_S_mask = {} keys = self.get_bsmask(i) for k in xrange(0, len(keys), 2): B_S_mask[(keys[k], keys[k + 1])] = True self.BSmasks.append(B_S_mask) #print "Done" return
def __call__(self, index): ''' Extract strong pixels from an image :param index: The index of the image ''' from dials.model.data import PixelList from dxtbx.imageset import ImageSweep from dials.array_family import flex from math import ceil # Parallel reading of HDF5 from the same handle is not allowed. Python # multiprocessing is a bit messed up and used fork on linux so need to # close and reopen file. if self.first: from dxtbx.imageset import SingleFileReader if isinstance(self.imageset.reader(), SingleFileReader): self.imageset.reader().nullify_format_instance() self.first = False # Get the frame number if isinstance(self.imageset, ImageSweep): frame = self.imageset.get_array_range()[0] + index else: ind = self.imageset.indices() if len(ind) > 1: assert(all(i1+1 == i2 for i1, i2 in zip(ind[0:-1], ind[1:-1]))) frame = ind[index] # Create the list of pixel lists pixel_list = [] # Get the image and mask image = self.imageset.get_corrected_data(index) mask = self.imageset.get_mask(index) # Set the mask if self.mask is not None: assert(len(self.mask) == len(mask)) mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask)) logger.debug("Number of masked pixels for image %i: %i" % (index, sum(m.count(False) for m in mask))) # Add the images to the pixel lists num_strong = 0 average_background = 0 for im, mk in zip(image, mask): if self.region_of_interest is not None: x0, x1, y0, y1 = self.region_of_interest height, width = im.all() assert x0 < x1, "x0 < x1" assert y0 < y1, "y0 < y1" assert x0 >= 0, "x0 >= 0" assert y0 >= 0, "y0 >= 0" assert x1 <= width, "x1 <= width" assert y1 <= height, "y1 <= height" im_roi = im[y0:y1,x0:x1] mk_roi = mk[y0:y1,x0:x1] tm_roi = self.threshold_function.compute_threshold(im_roi, mk_roi) threshold_mask = flex.bool(im.accessor(),False) threshold_mask[y0:y1,x0:x1] = tm_roi else: threshold_mask = self.threshold_function.compute_threshold(im, mk) # Add the pixel list plist = PixelList(frame, im, threshold_mask) pixel_list.append(plist) # Get average background if self.compute_mean_background: background = im.as_1d().select((mk & ~threshold_mask).as_1d()) average_background += flex.mean(background) # Add to the spot count num_strong += len(plist) # Make average background average_background /= len(image) # Check total number of strong pixels if self.max_strong_pixel_fraction < 1: num_image = 0 for im in image: num_image += len(im) max_strong = int(ceil(self.max_strong_pixel_fraction * num_image)) if num_strong > max_strong: raise RuntimeError( ''' The number of strong pixels found (%d) is greater than the maximum allowed (%d). Try changing spot finding parameters ''' % (num_strong, max_strong)) # Print some info if self.compute_mean_background: logger.info("Found %d strong pixels on image %d with average background %f" % (num_strong, frame+1, average_background)) else: logger.info("Found %d strong pixels on image %d" % (num_strong, frame+1)) # Return the result return Result(pixel_list)
def run(self, experiments, reflections): self.logger.log_step_time("POSTREFINEMENT") if not self.params.postrefinement.enable: self.logger.log("Postrefinement was not done") if self.mpi_helper.rank == 0: self.logger.main_log("Postrefinement was not done") return experiments, reflections target_symm = symmetry( unit_cell=self.params.scaling.unit_cell, space_group_info=self.params.scaling.space_group) i_model = self.params.scaling.i_model miller_set = self.params.scaling.miller_set # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. assert len(i_model.indices()) == len(miller_set.indices()) assert (i_model.indices() == miller_set.indices()).count(False) == 0 new_experiments = ExperimentList() new_reflections = flex.reflection_table() experiments_rejected_by_reason = {} # reason:how_many_rejected for experiment in experiments: exp_reflections = reflections.select( reflections['exp_id'] == experiment.identifier) # Build a miller array for the experiment reflections with original miller indexes exp_miller_indices_original = miller.set( target_symm, exp_reflections['miller_index'], True) observations_original_index = miller.array( exp_miller_indices_original, exp_reflections['intensity.sum.value'], flex.double( flex.sqrt(exp_reflections['intensity.sum.variance']))) assert exp_reflections.size() == exp_miller_indices_original.size() assert observations_original_index.size( ) == exp_miller_indices_original.size() # Build a miller array for the experiment reflections with asu miller indexes exp_miller_indices_asu = miller.set( target_symm, exp_reflections['miller_index_asymmetric'], True) observations = miller.array( exp_miller_indices_asu, exp_reflections['intensity.sum.value'], flex.double( flex.sqrt(exp_reflections['intensity.sum.variance']))) matches = miller.match_multi_indices( miller_indices_unique=miller_set.indices(), miller_indices=observations.indices()) pair1 = flex.int([pair[1] for pair in matches.pairs() ]) # refers to the observations pair0 = flex.int([pair[0] for pair in matches.pairs() ]) # refers to the model assert exp_reflections.size() == exp_miller_indices_original.size() assert observations_original_index.size( ) == exp_miller_indices_original.size() # narrow things down to the set that matches, only observations_pair1_selected = observations.customized_copy( indices=flex.miller_index( [observations.indices()[p] for p in pair1]), data=flex.double([observations.data()[p] for p in pair1]), sigmas=flex.double([observations.sigmas()[p] for p in pair1])) observations_original_index_pair1_selected = observations_original_index.customized_copy( indices=flex.miller_index( [observations_original_index.indices()[p] for p in pair1]), data=flex.double( [observations_original_index.data()[p] for p in pair1]), sigmas=flex.double( [observations_original_index.sigmas()[p] for p in pair1])) I_observed = observations_pair1_selected.data() MILLER = observations_original_index_pair1_selected.indices() ORI = crystal_orientation(experiment.crystal.get_A(), basis_type.reciprocal) Astar = matrix.sqr(ORI.reciprocal_matrix()) Astar_from_experiment = matrix.sqr(experiment.crystal.get_A()) assert Astar == Astar_from_experiment WAVE = experiment.beam.get_wavelength() BEAM = matrix.col((0.0, 0.0, -1. / WAVE)) BFACTOR = 0. MOSAICITY_DEG = experiment.crystal.get_half_mosaicity_deg() DOMAIN_SIZE_A = experiment.crystal.get_domain_size_ang() # calculation of correlation here I_reference = flex.double( [i_model.data()[pair[0]] for pair in matches.pairs()]) I_invalid = flex.bool( [i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()]) use_weights = False # New facility for getting variance-weighted correlation if use_weights: # variance weighting I_weight = flex.double([ 1. / (observations_pair1_selected.sigmas()[pair[1]])**2 for pair in matches.pairs() ]) else: I_weight = flex.double( len(observations_pair1_selected.sigmas()), 1.) I_weight.set_selected(I_invalid, 0.) """Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement: include_negatives = True + and - reflections both used for Rh distribution for initial estimate of RS parameter + and - reflections both used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) include_negatives = False + and - reflections both used for Rh distribution for initial estimate of RS parameter + reflections only used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) """ # RB: By design, for MPI-Merge "include negatives" is implicitly True SWC = simple_weighted_correlation(I_weight, I_reference, I_observed) if self.params.output.log_level == 0: self.logger.log("Old correlation is: %f" % SWC.corr) if self.params.postrefinement.algorithm == "rs": Rhall = flex.double() for mill in MILLER: H = matrix.col(mill) Xhkl = Astar * H Rh = (Xhkl + BEAM).length() - (1. / WAVE) Rhall.append(Rh) Rs = math.sqrt(flex.mean(Rhall * Rhall)) RS = 1. / 10000. # reciprocal effective domain size of 1 micron RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.]) parameterization_class = rs_parameterization refinery = rs_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) elif self.params.postrefinement.algorithm == "eta_deff": eta_init = 2. * MOSAICITY_DEG * math.pi / 180. D_eff_init = 2. * DOMAIN_SIZE_A current = flex.double( [SWC.slope, BFACTOR, eta_init, 0., 0., D_eff_init]) parameterization_class = eta_deff_parameterization refinery = eta_deff_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) func = refinery.fvec_callable(parameterization_class(current)) functional = flex.sum(func * func) if self.params.output.log_level == 0: self.logger.log("functional: %f" % functional) self.current = current self.parameterization_class = parameterization_class self.refinery = refinery self.observations_pair1_selected = observations_pair1_selected self.observations_original_index_pair1_selected = observations_original_index_pair1_selected error_detected = False try: self.run_plain() result_observations_original_index, result_observations, result_matches = self.result_for_cxi_merge( ) assert result_observations_original_index.size( ) == result_observations.size() assert result_matches.pairs().size( ) == result_observations_original_index.size() except (AssertionError, ValueError, RuntimeError) as e: error_detected = True reason = repr(e) if not reason: reason = "Unknown error" if not reason in experiments_rejected_by_reason: experiments_rejected_by_reason[reason] = 1 else: experiments_rejected_by_reason[reason] += 1 if not error_detected: new_experiments.append(experiment) new_exp_reflections = flex.reflection_table() new_exp_reflections[ 'miller_index_asymmetric'] = flex.miller_index( result_observations.indices()) new_exp_reflections['intensity.sum.value'] = flex.double( result_observations.data()) new_exp_reflections['intensity.sum.variance'] = flex.double( flex.pow(result_observations.sigmas(), 2)) new_exp_reflections['exp_id'] = flex.std_string( len(new_exp_reflections), experiment.identifier) new_reflections.extend(new_exp_reflections) ''' # debugging elif reason.startswith("ValueError"): self.logger.log("Rejected b/c of value error exp id: %s; unit cell: %s"%(exp_id, str(experiment.crystal.get_unit_cell())) ) ''' # report rejected experiments, reflections experiments_rejected_by_postrefinement = len(experiments) - len( new_experiments) reflections_rejected_by_postrefinement = reflections.size( ) - new_reflections.size() self.logger.log("Experiments rejected by post-refinement: %d" % experiments_rejected_by_postrefinement) self.logger.log("Reflections rejected by post-refinement: %d" % reflections_rejected_by_postrefinement) all_reasons = [] for reason, count in experiments_rejected_by_reason.iteritems(): self.logger.log("Experiments rejected due to %s: %d" % (reason, count)) all_reasons.append(reason) comm = self.mpi_helper.comm MPI = self.mpi_helper.MPI # Collect all rejection reasons from all ranks. Use allreduce to let each rank have all reasons. all_reasons = comm.allreduce(all_reasons, MPI.SUM) all_reasons = set(all_reasons) # Now that each rank has all reasons from all ranks, we can treat the reasons in a uniform way. total_experiments_rejected_by_reason = {} for reason in all_reasons: rejected_experiment_count = 0 if reason in experiments_rejected_by_reason: rejected_experiment_count = experiments_rejected_by_reason[ reason] total_experiments_rejected_by_reason[reason] = comm.reduce( rejected_experiment_count, MPI.SUM, 0) total_accepted_experiment_count = comm.reduce(len(new_experiments), MPI.SUM, 0) # how many reflections have we rejected due to post-refinement? rejected_reflections = len(reflections) - len(new_reflections) total_rejected_reflections = self.mpi_helper.sum(rejected_reflections) if self.mpi_helper.rank == 0: for reason, count in total_experiments_rejected_by_reason.iteritems( ): self.logger.main_log( "Total experiments rejected due to %s: %d" % (reason, count)) self.logger.main_log("Total experiments accepted: %d" % total_accepted_experiment_count) self.logger.main_log( "Total reflections rejected due to post-refinement: %d" % total_rejected_reflections) self.logger.log_step_time("POSTREFINEMENT", True) return new_experiments, new_reflections
def dano_over_sigdano(anomalous_amplitudes): """Calculate < |F(+) - F(-)| / sigma(F(+) - F(-))> i.e. <DANO/SIGDANO>.""" diff = anomalous_amplitudes.anomalous_differences() if not diff.data() or not diff.sigmas(): return 0.0 return flex.mean(flex.abs(diff.data()) / diff.sigmas())
def __init__(self, measurements_orig, params, i_model, miller_set, result, out): measurements = measurements_orig.deep_copy() # Now manipulate the data to conform to unit cell, asu, and space group # of reference. The resolution will be cut later. # Only works if there is NOT an indexing ambiguity! observations = measurements.customized_copy( anomalous_flag=not params.merge_anomalous, crystal_symmetry=miller_set.crystal_symmetry()).map_to_asu() observations_original_index = measurements.customized_copy( anomalous_flag=not params.merge_anomalous, crystal_symmetry=miller_set.crystal_symmetry()) # Ensure that match_multi_indices() will return identical results # when a frame's observations are matched against the # pre-generated Miller set, self.miller_set, and the reference # data set, self.i_model. The implication is that the same match # can be used to map Miller indices to array indices for intensity # accumulation, and for determination of the correlation # coefficient in the presence of a scaling reference. assert len(i_model.indices()) == len(miller_set.indices()) \ and (i_model.indices() == miller_set.indices()).count(False) == 0 matches = miller.match_multi_indices( miller_indices_unique=miller_set.indices(), miller_indices=observations.indices()) pair1 = flex.int([pair[1] for pair in matches.pairs()]) pair0 = flex.int([pair[0] for pair in matches.pairs()]) # narrow things down to the set that matches, only observations_pair1_selected = observations.customized_copy( indices=flex.miller_index( [observations.indices()[p] for p in pair1]), data=flex.double([observations.data()[p] for p in pair1]), sigmas=flex.double([observations.sigmas()[p] for p in pair1]), ) observations_original_index_pair1_selected = observations_original_index.customized_copy( indices=flex.miller_index( [observations_original_index.indices()[p] for p in pair1]), data=flex.double( [observations_original_index.data()[p] for p in pair1]), sigmas=flex.double( [observations_original_index.sigmas()[p] for p in pair1]), ) ################### I_observed = observations_pair1_selected.data() MILLER = observations_original_index_pair1_selected.indices() ORI = result["current_orientation"][0] Astar = matrix.sqr(ORI.reciprocal_matrix()) WAVE = result["wavelength"] BEAM = matrix.col((0.0, 0.0, -1. / WAVE)) BFACTOR = 0. #calculation of correlation here I_reference = flex.double( [i_model.data()[pair[0]] for pair in matches.pairs()]) I_invalid = flex.bool( [i_model.sigmas()[pair[0]] < 0. for pair in matches.pairs()]) use_weights = False # New facility for getting variance-weighted correlation if use_weights: #variance weighting I_weight = flex.double([ 1. / (observations_pair1_selected.sigmas()[pair[1]])**2 for pair in matches.pairs() ]) else: I_weight = flex.double(len(observations_pair1_selected.sigmas()), 1.) I_weight.set_selected(I_invalid, 0.) """Explanation of 'include_negatives' semantics as originally implemented in cxi.merge postrefinement: include_negatives = True + and - reflections both used for Rh distribution for initial estimate of RS parameter + and - reflections both used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) include_negatives = False + and - reflections both used for Rh distribution for initial estimate of RS parameter + reflections only used for calc/obs correlation slope for initial estimate of G parameter + and - reflections both passed to the refinery and used in the target function (makes sense if you look at it from a certain point of view) """ if params.include_negatives: SWC = simple_weighted_correlation(I_weight, I_reference, I_observed) else: non_positive = (observations_pair1_selected.data() <= 0) SWC = simple_weighted_correlation( I_weight.select(~non_positive), I_reference.select(~non_positive), I_observed.select(~non_positive)) print >> out, "Old correlation is", SWC.corr if params.postrefinement.algorithm == "rs": Rhall = flex.double() for mill in MILLER: H = matrix.col(mill) Xhkl = Astar * H Rh = (Xhkl + BEAM).length() - (1. / WAVE) Rhall.append(Rh) Rs = math.sqrt(flex.mean(Rhall * Rhall)) RS = 1. / 10000. # reciprocal effective domain size of 1 micron RS = Rs # try this empirically determined approximate, monochrome, a-mosaic value current = flex.double([SWC.slope, BFACTOR, RS, 0., 0.]) parameterization_class = rs_parameterization refinery = rs_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) elif params.postrefinement.algorithm == "eta_deff": eta_init = 2. * result["ML_half_mosaicity_deg"][0] * math.pi / 180. D_eff_init = 2. * result["ML_domain_size_ang"][0] current = flex.double([ SWC.slope, BFACTOR, eta_init, 0., 0., D_eff_init, ]) parameterization_class = eta_deff_parameterization refinery = eta_deff_refinery(ORI=ORI, MILLER=MILLER, BEAM=BEAM, WAVE=WAVE, ICALCVEC=I_reference, IOBSVEC=I_observed) func = refinery.fvec_callable(parameterization_class(current)) functional = flex.sum(func * func) print >> out, "functional", functional self.current = current self.parameterization_class = parameterization_class self.refinery = refinery self.out = out self.params = params self.miller_set = miller_set self.observations_pair1_selected = observations_pair1_selected self.observations_original_index_pair1_selected = observations_original_index_pair1_selected
def finalize(self, data, mask): ''' Finalize the model :param data: The data array :param mask: The mask array ''' from dials.algorithms.image.filter import median_filter, mean_filter from dials.algorithms.image.fill_holes import diffusion_fill from dials.algorithms.image.fill_holes import simple_fill from dials.array_family import flex # Print some image properties sub_data = data.as_1d().select(mask.as_1d()) logger.info('Raw image statistics:') logger.info(' min: %d' % int(flex.min(sub_data))) logger.info(' max: %d' % int(flex.max(sub_data))) logger.info(' mean: %d' % int(flex.mean(sub_data))) logger.info('') # Transform to polar logger.info('Transforming image data to polar grid') result = self.transform.to_polar(data, mask) data = result.data() mask = result.mask() sub_data = data.as_1d().select(mask.as_1d()) logger.info('Polar image statistics:') logger.info(' min: %d' % int(flex.min(sub_data))) logger.info(' max: %d' % int(flex.max(sub_data))) logger.info(' mean: %d' % int(flex.mean(sub_data))) logger.info('') # Filter the image to remove noise if self.kernel_size > 0: if self.filter_type == 'median': logger.info('Applying median filter') data = median_filter(data, mask, (self.kernel_size, 0)) sub_data = data.as_1d().select(mask.as_1d()) logger.info('Median polar image statistics:') logger.info(' min: %d' % int(flex.min(sub_data))) logger.info(' max: %d' % int(flex.max(sub_data))) logger.info(' mean: %d' % int(flex.mean(sub_data))) logger.info('') elif self.filter_type == 'mean': logger.info('Applying mean filter') mask_as_int = mask.as_1d().as_int() mask_as_int.reshape(mask.accessor()) data = mean_filter(data, mask_as_int, (self.kernel_size, 0), 1) sub_data = data.as_1d().select(mask.as_1d()) logger.info('Mean polar image statistics:') logger.info(' min: %d' % int(flex.min(sub_data))) logger.info(' max: %d' % int(flex.max(sub_data))) logger.info(' mean: %d' % int(flex.mean(sub_data))) logger.info('') else: raise RuntimeError('Unknown filter_type: %s' % self.filter_type) # Fill any remaining holes logger.info("Filling holes") data = simple_fill(data, mask) data = diffusion_fill(data, mask, self.niter) mask = flex.bool(data.accessor(), True) sub_data = data.as_1d().select(mask.as_1d()) logger.info('Filled polar image statistics:') logger.info(' min: %d' % int(flex.min(sub_data))) logger.info(' max: %d' % int(flex.max(sub_data))) logger.info(' mean: %d' % int(flex.mean(sub_data))) logger.info('') # Transform back logger.info('Transforming image data from polar grid') result = self.transform.from_polar(data, mask) data = result.data() mask = result.mask() sub_data = data.as_1d().select(mask.as_1d()) logger.info('Final image statistics:') logger.info(' min: %d' % int(flex.min(sub_data))) logger.info(' max: %d' % int(flex.max(sub_data))) logger.info(' mean: %d' % int(flex.mean(sub_data))) logger.info('') # Fill in any discontinuities # FIXME NEED TO HANDLE DISCONTINUITY # mask = ~self.transform.discontinuity()[:-1,:-1] # data = diffusion_fill(data, mask, self.niter) # Get and apply the mask mask = self.experiment.imageset.get_mask(0)[0] mask = mask.as_1d().as_int().as_double() mask.reshape(data.accessor()) data *= mask # Return the result return data
def finalize(self, data, mask): """ Finalize the model :param data: The data array :param mask: The mask array """ from dials.algorithms.image.filter import median_filter, mean_filter from dials.algorithms.image.fill_holes import diffusion_fill from dials.algorithms.image.fill_holes import simple_fill from dials.array_family import flex # Print some image properties sub_data = data.as_1d().select(mask.as_1d()) logger.info("Raw image statistics:") logger.info(" min: %d" % int(flex.min(sub_data))) logger.info(" max: %d" % int(flex.max(sub_data))) logger.info(" mean: %d" % int(flex.mean(sub_data))) logger.info("") # Transform to polar logger.info("Transforming image data to polar grid") result = self.transform.to_polar(data, mask) data = result.data() mask = result.mask() sub_data = data.as_1d().select(mask.as_1d()) logger.info("Polar image statistics:") logger.info(" min: %d" % int(flex.min(sub_data))) logger.info(" max: %d" % int(flex.max(sub_data))) logger.info(" mean: %d" % int(flex.mean(sub_data))) logger.info("") # Filter the image to remove noise if self.kernel_size > 0: if self.filter_type == "median": logger.info("Applying median filter") data = median_filter(data, mask, (self.kernel_size, 0), periodic=True) sub_data = data.as_1d().select(mask.as_1d()) logger.info("Median polar image statistics:") logger.info(" min: %d" % int(flex.min(sub_data))) logger.info(" max: %d" % int(flex.max(sub_data))) logger.info(" mean: %d" % int(flex.mean(sub_data))) logger.info("") elif self.filter_type == "mean": logger.info("Applying mean filter") mask_as_int = mask.as_1d().as_int() mask_as_int.reshape(mask.accessor()) data = mean_filter(data, mask_as_int, (self.kernel_size, 0), 1) sub_data = data.as_1d().select(mask.as_1d()) logger.info("Mean polar image statistics:") logger.info(" min: %d" % int(flex.min(sub_data))) logger.info(" max: %d" % int(flex.max(sub_data))) logger.info(" mean: %d" % int(flex.mean(sub_data))) logger.info("") else: raise RuntimeError("Unknown filter_type: %s" % self.filter_type) # Fill any remaining holes logger.info("Filling holes") data = simple_fill(data, mask) data = diffusion_fill(data, mask, self.niter) mask = flex.bool(data.accessor(), True) sub_data = data.as_1d().select(mask.as_1d()) logger.info("Filled polar image statistics:") logger.info(" min: %d" % int(flex.min(sub_data))) logger.info(" max: %d" % int(flex.max(sub_data))) logger.info(" mean: %d" % int(flex.mean(sub_data))) logger.info("") # Transform back logger.info("Transforming image data from polar grid") result = self.transform.from_polar(data, mask) data = result.data() mask = result.mask() sub_data = data.as_1d().select(mask.as_1d()) logger.info("Final image statistics:") logger.info(" min: %d" % int(flex.min(sub_data))) logger.info(" max: %d" % int(flex.max(sub_data))) logger.info(" mean: %d" % int(flex.mean(sub_data))) logger.info("") # Fill in any discontinuities mask = ~self.transform.discontinuity()[:-1, :-1] data = diffusion_fill(data, mask, self.niter) # Get and apply the mask mask = self.experiment.imageset.get_mask(0)[0] mask = mask.as_1d().as_int().as_double() mask.reshape(data.accessor()) data *= mask # Return the result return data
def __call__(self, experiments, reflections): results = flex.reflection_table() table_header = ["","","","I","IsigI","N >","RMSD","Cutoff"] table_header2 = ["Bin","Resolution Range","Completeness","","","cutoff","(um)",""] for exp_id in xrange(len(experiments)): print("*"*80) print("Significance filtering experiment", exp_id) table_data = [] table_data.append(table_header) table_data.append(table_header2) experiment = experiments[exp_id] # Find the bins for this experiment crystal = experiment.crystal refls = reflections.select(reflections['id'] == exp_id) sym = symmetry(unit_cell = crystal.get_unit_cell(), space_group = crystal.get_space_group()) d = crystal.get_unit_cell().d(refls['miller_index']) mset = sym.miller_set(indices = refls['miller_index'], anomalous_flag=False) binner = mset.setup_binner(n_bins=self.params.n_bins) acceptable_resolution_bins = [] # Iterate through the bins, examining I/sigI at each bin for i in binner.range_used(): d_max, d_min = binner.bin_d_range(i) sel = (d <= d_max) & (d > d_min) sel &= refls['intensity.sum.value'] > 0 bin_refls = refls.select(sel) n_refls = len(bin_refls) avg_i = flex.mean(bin_refls['intensity.sum.value']) if n_refls > 0 else 0 avg_i_sigi = flex.mean(bin_refls['intensity.sum.value'] / flex.sqrt(bin_refls['intensity.sum.variance'])) if n_refls > 0 else 0 acceptable_resolution_bins.append(avg_i_sigi >= self.params.isigi_cutoff) bright_refls = bin_refls.select((bin_refls['intensity.sum.value']/flex.sqrt(bin_refls['intensity.sum.variance'])) >= self.params.isigi_cutoff) n_bright = len(bright_refls) rmsd_obs = 1000*math.sqrt((bright_refls['xyzcal.mm']-bright_refls['xyzobs.mm.value']).sum_sq()/n_bright) if n_bright > 0 else 0 table_row = [] table_row.append("%3d"%i) table_row.append("%-13s"%binner.bin_legend(i_bin=i,show_bin_number=False,show_bin_range=False, show_d_range=True, show_counts=False)) table_row.append("%13s"%binner.bin_legend(i_bin=i,show_bin_number=False,show_bin_range=False, show_d_range=False, show_counts=True)) table_row.append("%.1f"%(avg_i)) table_row.append("%.1f"%(avg_i_sigi)) table_row.append("%3d"%n_bright) table_row.append("%.1f"%(rmsd_obs)) table_data.append(table_row) # Throw out bins that go back above the cutoff after the first non-passing bin is found acceptable_resolution_bins = [acceptable_resolution_bins[i] for i in xrange(len(acceptable_resolution_bins)) if False not in acceptable_resolution_bins[:i+1]] for b, row in zip(acceptable_resolution_bins, table_data[2:]): if b: row.append("X") print(table_utils.format(table_data,has_header=2,justify='center',delim=" ")) # Save the results if any(acceptable_resolution_bins): best_index = acceptable_resolution_bins.count(True)-1 best_row = table_data[best_index+2] d_min = binner.bin_d_range(binner.range_used()[best_index])[1] print("best row:", " ".join(best_row)) if self.params.enable: results.extend(refls.select(d >= d_min)) else: print("Data didn't pass cutoff") if self.params.enable: return results else: return reflections
def blank_integrated_analysis(reflections, scan, phi_step, fractional_loss): prf_sel = reflections.get_flags(reflections.flags.integrated_prf) if prf_sel.count(True) > 0: reflections = reflections.select(prf_sel) intensities = reflections["intensity.prf.value"] variances = reflections["intensity.prf.variance"] else: sum_sel = reflections.get_flags(reflections.flags.integrated_sum) reflections = reflections.select(sum_sel) intensities = reflections["intensity.sum.value"] variances = reflections["intensity.sum.variance"] i_sigi = intensities / flex.sqrt(variances) xyz_px = reflections["xyzobs.px.value"] x_px, y_px, z_px = xyz_px.parts() phi = scan.get_angle_from_array_index(z_px) osc = scan.get_oscillation()[1] n_images_per_step = iceil(phi_step / osc) phi_step = n_images_per_step * osc array_range = scan.get_array_range() phi_min = flex.min(phi) phi_max = flex.max(phi) n_steps = int(round((phi_max - phi_min) / phi_step)) hist = flex.histogram( z_px, data_min=array_range[0], data_max=array_range[1], n_slots=n_steps ) logger.debug("Histogram:") logger.debug(hist.as_str()) mean_i_sigi = flex.double() for i, slot_info in enumerate(hist.slot_infos()): sel = (z_px >= slot_info.low_cutoff) & (z_px < slot_info.high_cutoff) if sel.count(True) == 0: mean_i_sigi.append(0) else: mean_i_sigi.append(flex.mean(i_sigi.select(sel))) potential_blank_sel = mean_i_sigi <= (fractional_loss * flex.max(mean_i_sigi)) xmin, xmax = zip( *[ (slot_info.low_cutoff, slot_info.high_cutoff) for slot_info in hist.slot_infos() ] ) d = { "data": [ { "x": list(hist.slot_centers()), "y": list(mean_i_sigi), "xlow": xmin, "xhigh": xmax, "blank": list(potential_blank_sel), "type": "bar", "name": "blank_counts_analysis", } ], "layout": { "xaxis": {"title": "z observed (images)"}, "yaxis": {"title": "Number of reflections"}, "bargap": 0, }, } blank_regions = blank_regions_from_sel(d["data"][0]) d["blank_regions"] = blank_regions return d
def centroid_mean_diff_vs_phi(self, rlist, threshold): from os.path import join import math I = rlist['intensity.sum.value'] I_sig = flex.sqrt(rlist['intensity.sum.variance']) I_over_S = I / I_sig mask = I_over_S > threshold rlist = rlist.select(mask) assert(len(rlist) > 0) xc, yc, zc = rlist['xyzcal.mm'].parts() xo, yo, zo = rlist['xyzobs.mm.value'].parts() dx = xc - xo dy = yc - yo dphi = (zc - zo) * RAD2DEG mean_residuals_x = flex.double() mean_residuals_y = flex.double() mean_residuals_phi = flex.double() rmsd_x = flex.double() rmsd_y = flex.double() rmsd_phi = flex.double() frame = [] phi_obs_deg = RAD2DEG * zo phi = [] for i_phi in range(int(math.floor(flex.min(phi_obs_deg))), int(math.ceil(flex.max(phi_obs_deg)))): sel = (phi_obs_deg >= i_phi) & (phi_obs_deg < (i_phi+1)) if sel.count(True) == 0: continue mean_residuals_x.append(flex.mean(dx.select(sel))) mean_residuals_y.append(flex.mean(dy.select(sel))) mean_residuals_phi.append(flex.mean(dphi.select(sel))) rmsd_x.append(math.sqrt(flex.mean_sq(dx.select(sel)))) rmsd_y.append(math.sqrt(flex.mean_sq(dy.select(sel)))) rmsd_phi.append(math.sqrt(flex.mean_sq(dphi.select(sel)))) phi.append(i_phi) from matplotlib import pyplot fig = pyplot.figure() ax = fig.add_subplot(311) #fig.subplots_adjust(hspace=0.5) pyplot.axhline(0, color='grey') ax.scatter(phi, mean_residuals_x) ax.set_xlabel('phi (deg)') ax.set_ylabel('mean $\Delta$ x (mm)') ax = fig.add_subplot(312) pyplot.axhline(0, color='grey') ax.scatter(phi, mean_residuals_y) ax.set_xlabel('phi (deg)') ax.set_ylabel('mean $\Delta$ y (mm)') ax = fig.add_subplot(313) pyplot.axhline(0, color='grey') ax.scatter(phi, mean_residuals_phi) ax.set_xlabel('phi (deg)') ax.set_ylabel('mean $\Delta$ phi (deg)') pyplot.savefig(join(self.directory, "centroid_mean_diff_vs_phi.png")) pyplot.close() fig = pyplot.figure() ax = fig.add_subplot(311) #fig.subplots_adjust(hspace=0.5) pyplot.axhline(flex.mean(rmsd_x), color='grey') ax.scatter(phi, rmsd_x) ax.set_xlabel('phi (deg)') ax.set_ylabel('rmsd x (mm)') ax = fig.add_subplot(312) pyplot.axhline(flex.mean(rmsd_y), color='grey') ax.scatter(phi, rmsd_y) ax.set_xlabel('phi (deg)') ax.set_ylabel('rmsd y (mm)') ax = fig.add_subplot(313) pyplot.axhline(flex.mean(rmsd_phi), color='grey') ax.scatter(phi, rmsd_phi) ax.set_xlabel('phi (deg)') ax.set_ylabel('rmsd phi (deg)') pyplot.savefig(join(self.directory, "centroid_rmsd_vs_phi.png")) pyplot.close()
def __call__(self, index): """ Extract strong pixels from an image :param index: The index of the image """ from dials.model.data import PixelList from dxtbx.imageset import ImageSequence # Parallel reading of HDF5 from the same handle is not allowed. Python # multiprocessing is a bit messed up and used fork on linux so need to # close and reopen file. if self.first: if self.imageset.reader().is_single_file_reader(): self.imageset.reader().nullify_format_instance() self.first = False # Get the frame number if isinstance(self.imageset, ImageSequence): frame = self.imageset.get_array_range()[0] + index else: ind = self.imageset.indices() if len(ind) > 1: assert all(i1 + 1 == i2 for i1, i2 in zip(ind[0:-1], ind[1:-1])) frame = ind[index] # Create the list of pixel lists pixel_list = [] # Get the image and mask image = self.imageset.get_corrected_data(index) mask = self.imageset.get_mask(index) # Set the mask if self.mask is not None: assert len(self.mask) == len(mask) mask = tuple(m1 & m2 for m1, m2 in zip(mask, self.mask)) logger.debug("Number of masked pixels for image %i: %i" % (index, sum(m.count(False) for m in mask))) # Add the images to the pixel lists num_strong = 0 average_background = 0 for im, mk in zip(image, mask): if self.region_of_interest is not None: x0, x1, y0, y1 = self.region_of_interest height, width = im.all() assert x0 < x1, "x0 < x1" assert y0 < y1, "y0 < y1" assert x0 >= 0, "x0 >= 0" assert y0 >= 0, "y0 >= 0" assert x1 <= width, "x1 <= width" assert y1 <= height, "y1 <= height" im_roi = im[y0:y1, x0:x1] mk_roi = mk[y0:y1, x0:x1] tm_roi = self.threshold_function.compute_threshold( im_roi, mk_roi) threshold_mask = flex.bool(im.accessor(), False) threshold_mask[y0:y1, x0:x1] = tm_roi else: threshold_mask = self.threshold_function.compute_threshold( im, mk) # Add the pixel list plist = PixelList(frame, im, threshold_mask) pixel_list.append(plist) # Get average background if self.compute_mean_background: background = im.as_1d().select((mk & ~threshold_mask).as_1d()) average_background += flex.mean(background) # Add to the spot count num_strong += len(plist) # Make average background average_background /= len(image) # Check total number of strong pixels if self.max_strong_pixel_fraction < 1: num_image = 0 for im in image: num_image += len(im) max_strong = int( math.ceil(self.max_strong_pixel_fraction * num_image)) if num_strong > max_strong: raise RuntimeError(""" The number of strong pixels found (%d) is greater than the maximum allowed (%d). Try changing spot finding parameters """ % (num_strong, max_strong)) # Print some info if self.compute_mean_background: logger.info( "Found %d strong pixels on image %d with average background %f" % (num_strong, frame + 1, average_background)) else: logger.info("Found %d strong pixels on image %d" % (num_strong, frame + 1)) # Return the result return Result(pixel_list)
def overall_report(data): # Start by adding some overall numbers report = OrderedDict() report['n'] = len(reflections) report['n_full'] = data['full'].count(True) report['n_partial'] = data['full'].count(False) report['n_overload'] = data['over'].count(True) report['n_ice'] = data['ice'].count(True) report['n_summed'] = data['sum'].count(True) report['n_fitted'] = data['prf'].count(True) report['n_integated'] = data['int'].count(True) report['n_invalid_bg'] = data['ninvbg'].count(True) report['n_invalid_fg'] = data['ninvfg'].count(True) report['n_failed_background'] = data['fbgd'].count(True) report['n_failed_summation'] = data['fsum'].count(True) report['n_failed_fitting'] = data['fprf'].count(True) # Compute mean background try: report['mean_background'] = flex.mean( data['background.mean'].select(data['int'])) except Exception: report['mean_background'] = 0.0 # Compute mean I/Sigma summation try: report['ios_sum'] = flex.mean(data['intensity.sum.ios'].select( data['sum'])) except Exception: report['ios_sum'] = 0.0 # Compute mean I/Sigma profile fitting try: report['ios_prf'] = flex.mean(data['intensity.prf.ios'].select( data['prf'])) except Exception: report['ios_prf'] = 0.0 # Compute the mean profile correlation try: report['cc_prf'] = flex.mean(data['profile.correlation'].select( data['prf'])) except Exception: report['cc_prf'] = 0.0 # Compute the correlations between summation and profile fitting try: mask = data['sum'] & data['prf'] Isum = data['intensity.sum.value'].select(mask) Iprf = data['intensity.prf.value'].select(mask) report['cc_pearson_sum_prf'] = pearson_correlation_coefficient( Isum, Iprf) report['cc_spearman_sum_prf'] = spearman_correlation_coefficient( Isum, Iprf) except Exception: report['cc_pearson_sum_prf'] = 0.0 report['cc_spearman_sum_prf'] = 0.0 # Return the overall report return report
def run(params, root, common_set=None): iterable = [] iterable2 = [] debug_root = os.path.join(root, 'debug') print('start appending') for filename in os.listdir(debug_root): if os.path.splitext(filename)[1] != ".txt": continue iterable.append(filename) for filename in os.listdir(root): if 'refined_experiments' not in os.path.splitext( filename)[0] or os.path.splitext(filename)[1] != ".json": continue iterable2.append(filename) print('done appending') #if command_line.options.mpi: if params.mpi: try: from mpi4py import MPI except ImportError: raise Sorry("MPI not found") comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() print(rank, size) # get hits and indexing info iterable = [ iterable[i] for i in range(len(iterable)) if (i + rank) % size == 0 ] results = get_hits_and_indexing_stats(iterable, debug_root, rank) results = comm.gather(results, root=0) # Now get uc and rmsd info iterable2 = [ iterable2[i] for i in range(len(iterable2)) if (i + rank) % size == 0 ] results2 = get_uc_and_rmsd_stats(iterable2, root, rank=rank, common_set=common_set) results2 = comm.gather(results2, root=0) if rank != 0: return else: results = [get_hits_and_indexing_stats(iterable, debug_root)] results2 = [ get_uc_and_rmsd_stats(iterable2, root, rank=0, common_set=common_set) ] # Now evaulate summary statistics print('Now evaluating summary statistics') n_hits = 0 n_idx = 0 t_idx = 0 t_idx_success = 0 all_idx_cutoff_time_exceeded_event = [] total_xray_events = 0 total_images_analyzed = 0 for ii, r in enumerate(results): n_hits += r[0] n_idx += r[1] t_idx += r[2] t_idx_success += r[3] all_idx_cutoff_time_exceeded_event.extend(r[4]) total_xray_events += r[5] total_images_analyzed += r[6] # Write out the cutoff time exceeded events in a format that xtc_process.py can interpret for skipping events if indexing_time_cutoff is not None: fts = open('timestamps_to_skip.dat', 'w') for evt in all_idx_cutoff_time_exceeded_event: fts.write('psanagpu999,%s,%s,fail\n' % (evt, evt)) fts.close() node_hours = None core_hours = None if out_logfile is not None and wall_time is None: total_time = [] run_number = int(os.path.abspath(root).strip().split('/')[-3][1:]) print(run_number) with open(out_logfile, 'r') as flog: for line in flog: if string_to_search_for in line: ax = line.split() if int(ax[-1]) == run_number: total_time.append(float(ax[1])) node_hours = max(total_time) * num_nodes / 3600.0 if num_cores is not None: core_hours = max(total_time) * num_cores / 3600.0 else: core_hours = max( total_time) * num_nodes * num_cores_per_node / 3600.0 if wall_time is not None: node_hours = wall_time * num_nodes / 3600.0 if num_cores is not None: core_hours = wall_time * num_cores / 3600.0 else: core_hours = wall_time * num_nodes * num_cores_per_node / 3600.0 all_uc_a = flex.double() all_uc_b = flex.double() all_uc_c = flex.double() all_uc_alpha = flex.double() all_uc_beta = flex.double() all_uc_gamma = flex.double() dR = flex.double() info_list = [] info = [] for ii, r in enumerate(results2): all_uc_a.extend(r[0]) all_uc_b.extend(r[1]) all_uc_c.extend(r[2]) all_uc_alpha.extend(r[3]) all_uc_beta.extend(r[4]) all_uc_gamma.extend(r[5]) dR.extend(r[6]) if show_plot: for jj, aa in enumerate(r[0]): info.append({ 'a': r[0][jj], 'b': r[1][jj], 'c': r[2][jj], 'alpha': r[3][jj], 'beta': r[4][jj], 'gamma': r[5][jj], 'n_img': 0 }) info_list.append(info) n_lattices = len(all_uc_a) # Now print out all relevant statistics if True: print('-' * 80) print('|' + ' ' * 80 + '|\n' + '|' + ' ' * 20 + 'Analytics Package for Indexing' + ' ' * 30 + '|\n|' + ' ' * 80 + '|') print('-' * 80) print('Getting stats for data in : ', root) print( '====================== Indexing and Timing Statistics ============================' ) print('Total number of X-ray events = ', total_xray_events) print('Total number of images analyzed = ', total_images_analyzed) print('Number of Hits = ', n_hits) print('Number of images successfully indexed = ', n_idx) if common_set is None: print('Number of lattices = ', n_lattices) else: print('Number of common lattices', n_lattices) print('Total time spent in indexing (hrs) = ', t_idx) print('Time spent in indexing successfully (core-hrs) = ', t_idx_success) print('Average time spent indexing (core-secs) = ', 3600 * t_idx / n_hits) print('Average time spent indexing successfully (core-secs) = ', 3600 * t_idx_success / n_idx) if node_hours is not None: print('Total Node-hours with %d nodes = %.2f (hrs)' % (num_nodes, node_hours)) print( '% core utilization i.e (total indexing time)/(total core-hrs) = ', 100.0 * t_idx / core_hours) if common_set is None: print( '====================== Unit Cell & RMSD Statistics ============================' ) else: print( '====================== Unit Cell & RMSD Statistics from Common Set ============================' ) print('a-edge (A) : %.2f +/- %.2f' % (flex.mean(all_uc_a), flex.mean_and_variance( all_uc_a).unweighted_sample_standard_deviation())) print('b-edge (A) : %.2f +/- %.2f' % (flex.mean(all_uc_b), flex.mean_and_variance( all_uc_b).unweighted_sample_standard_deviation())) print('c-edge (A) : %.2f +/- %.2f' % (flex.mean(all_uc_c), flex.mean_and_variance( all_uc_c).unweighted_sample_standard_deviation())) print('alpha (deg) : %.2f +/- %.2f' % (flex.mean(all_uc_alpha), flex.mean_and_variance( all_uc_alpha).unweighted_sample_standard_deviation())) print('beta (deg) : %.2f +/- %.2f' % (flex.mean(all_uc_beta), flex.mean_and_variance( all_uc_beta).unweighted_sample_standard_deviation())) print('gamma (deg) : %.2f +/- %.2f' % (flex.mean(all_uc_gamma), flex.mean_and_variance( all_uc_gamma).unweighted_sample_standard_deviation())) print('Total RMSD i.e calc - obs for Bragg spots (um) = ', 1000.0 * math.sqrt(dR.dot(dR) / len(dR))) print('-' * 80) if show_plot: import xfel.ui.components.xfel_gui_plotter as pltr plotter = pltr.PopUpCharts() plotter.plot_uc_histogram(info_list=info_list, legend_list=['combined'], iqr_ratio=None) plotter.plt.show()
def __init__(self, experiments, profile_fitter, reflections, num_folds): ''' Create the integration report :param experiments: The experiment list :param profile_model: The profile model :param reflections: The reflection table ''' from collections import OrderedDict # Initialise the report class super(ProfileValidationReport, self).__init__() # Create the table table = Table() # Set the title table.name = 'validation.summary' table.title = 'Summary of profile validation ' # Add the columns table.cols.append(('id', 'ID')) table.cols.append(('subsample', 'Sub-sample')) table.cols.append(('n_valid', '# validated')) table.cols.append(('cc', '<CC>')) table.cols.append(('nrmsd', '<NRMSD>')) # Split the reflections reflection_tables = reflections.split_by_experiment_id() assert len(reflection_tables) == len(experiments) assert len(profile_fitter) == num_folds # Create the summary for each profile model for i in range(len(reflection_tables)): reflection_table = reflection_tables[i] reflection_table = reflection_table.select( reflection_table.get_flags( reflection_table.flags.integrated_prf)) index = reflection_table['profile.index'] cc = reflection_table['profile.correlation'] nrmsd = reflection_table['profile.rmsd'] for j in range(num_folds): mask = index == j num_validated = mask.count(True) if num_validated == 0: mean_cc = 0 mean_nrmsd = 0 else: mean_cc = flex.mean(cc.select(mask)) mean_nrmsd = flex.mean(nrmsd.select(mask)) table.rows.append([ '%d' % i, '%d' % j, '%d' % num_validated, '%.2f' % mean_cc, '%.2f' % mean_nrmsd ]) # Add the table self.add_table(table)
def per_sensor_analysis(self): # hardcoded Jungfrau 16M geometry for isensor in range(32): print ("Panel Sensor <Δx>(μm) <Δy>(μm) Nrefl RMS Δx(μm) RMS Δy(μm) ") if len(self.cumCALC[isensor]) < 2: continue for ipanel in range(8*isensor, 8*(1+isensor)): if len(self.panel_deltax[ipanel])<2: continue Sx = flex.mean_and_variance(1000.*self.panel_deltax[ipanel]) Sy = flex.mean_and_variance(1000.*self.panel_deltay[ipanel]) RMSDx = 1000.*math.sqrt(flex.mean(self.panel_deltax[ipanel]*self.panel_deltax[ipanel])) RMSDy = 1000.*math.sqrt(flex.mean(self.panel_deltay[ipanel]*self.panel_deltay[ipanel])) print("%3d %3d"%(ipanel,ipanel//8),"%7.2f±%6.2f %7.2f±%6.2f %6d"%(Sx.mean(),Sx.unweighted_standard_error_of_mean(), Sy.mean(),Sy.unweighted_standard_error_of_mean(), len(self.panel_deltax[ipanel])), " %5.1f %5.1f"%(RMSDx,RMSDy), ) print("") cumD = (self.cumCALC[isensor]-self.cumOBS[isensor]).parts() print ( "All %3d %7.2f %7.2f %6d"%(isensor,1000.*flex.mean(cumD[0]), 1000.*flex.mean(cumD[1]), len(cumD[0]))) print("") # Now we'll do a linear least squares refinement over sensors: #Method 1. Simple rectilinear translation. if self.params.verbose: veclength = len(self.cumCALC[isensor]) correction = flex.vec3_double( veclength, (flex.mean(cumD[0]), flex.mean(cumD[1]), flex.mean(cumD[2])) ) new_delta = (self.cumCALC[isensor]-correction ) -self.cumOBS[isensor] for ipanel in range(8*isensor, 8*(1+isensor)): panel_delta = new_delta.select(self.cumPANNO[isensor]==ipanel) if len(panel_delta)<2: continue deltax_part, deltay_part = panel_delta.parts()[0:2] RMSDx = 1000.*math.sqrt( flex.mean(deltax_part * deltax_part) ) RMSDy = 1000.*math.sqrt( flex.mean(deltay_part * deltay_part) ) Sx = flex.mean_and_variance(1000.*deltax_part) Sy = flex.mean_and_variance(1000.*deltay_part) print("%3d %3d"%(ipanel,ipanel//8),"%7.2f±%6.2f %7.2f±%6.2f %6d"%(Sx.mean(),Sx.unweighted_standard_error_of_mean(), Sy.mean(),Sy.unweighted_standard_error_of_mean(), len(deltax_part)), " %5.1f %5.1f"%(RMSDx,RMSDy), ) print() # Method 2. Translation + rotation. src = [] dst = [] for icoord in range(len(self.cumCALC[isensor])): src.append(self.cumCALC[isensor][icoord][0:2]) dst.append(self.cumOBS[isensor][icoord][0:2]) src = np.array(src) dst = np.array(dst) # estimate affine transform model using all coordinates model = SimilarityTransform() model.estimate(src, dst) # robustly estimate affine transform model with RANSAC model_robust, inliers = ransac((src, dst), SimilarityTransform, min_samples=3, residual_threshold=2, max_trials=10) outliers = flex.bool(inliers == False) # compare "true" and estimated transform parameters if self.params.verbose: print("Similarity transform:") print("%2d"%isensor, "Scale: %.5f,"%(model.scale), "Translation(μm): (%7.2f,"%(1000.*model.translation[0]), "%7.2f),"%(1000.*model.translation[1]), "Rotation (°): %7.4f"%((180./math.pi)*model.rotation)) print("RANSAC:") print("%2d"%isensor, "Scale: %.5f,"%(model_robust.scale), "Translation(μm): (%7.2f,"%(1000.*model_robust.translation[0]), "%7.2f),"%(1000.*model_robust.translation[1]), "Rotation (°): %7.4f,"%((180./math.pi)*model_robust.rotation), "Outliers:",outliers.count(True) ) """from documentation: X = a0 * x - b0 * y + a1 = s * x * cos(rotation) - s * y * sin(rotation) + a1 Y = b0 * x + a0 * y + b1 = s * x * sin(rotation) + s * y * cos(rotation) + b1""" oldCALC = self.cumCALC[isensor].parts() ransacCALC = flex.vec3_double( (float(model_robust.scale) * oldCALC[0] * math.cos(model_robust.rotation) - float(model_robust.scale) * oldCALC[1] * math.sin(model_robust.rotation) + float(model_robust.translation[0])), (float(model_robust.scale) * oldCALC[0] * math.sin(model_robust.rotation) + float(model_robust.scale) * oldCALC[1] * math.cos(model_robust.rotation) + float(model_robust.translation[1])), oldCALC[2] ) new_delta = ransacCALC - self.cumOBS[isensor] inlier_delta = new_delta.select(~outliers) inlier_panno = self.cumPANNO[isensor].select(~outliers) for ipanel in range(8*isensor, 8*(1+isensor)): panel_delta = inlier_delta.select(inlier_panno==ipanel) if len(panel_delta)<2: continue deltax_part, deltay_part = panel_delta.parts()[0:2] RMSDx = 1000.*math.sqrt( flex.mean(deltax_part * deltax_part) ) RMSDy = 1000.*math.sqrt( flex.mean(deltay_part * deltay_part) ) Sx = flex.mean_and_variance(1000.*deltax_part) Sy = flex.mean_and_variance(1000.*deltay_part) print("%3d %3d"%(ipanel,ipanel//8),"%7.2f±%6.2f %7.2f±%6.2f %6d"%(Sx.mean(),Sx.unweighted_standard_error_of_mean(), Sy.mean(),Sy.unweighted_standard_error_of_mean(), len(deltax_part)), " %5.1f %5.1f"%(RMSDx,RMSDy), ) if self.params.verbose: print("") cumD = (inlier_delta).parts() print ( " %3d %7.2f %7.2f %6d\n"%(isensor,1000.*flex.mean(cumD[0]), 1000.*flex.mean(cumD[1]), len(cumD[0]))) print("----\n")
def run(self, method): from xia2.Handlers.Streams import Debug Debug.write('Running dials.index') self.clear_command_line() for f in self._sweep_filenames: self.add_command_line(f) for f in self._spot_filenames: self.add_command_line(f) self.add_command_line('indexing.method=%s' % method) nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc self.set_cpu_threads(nproc) self.add_command_line('indexing.nproc=%i' % nproc) if PhilIndex.params.xia2.settings.small_molecule == True: self.add_command_line('filter_ice=false') if self._reflections_per_degree is not None: self.add_command_line( 'reflections_per_degree=%i' %self._reflections_per_degree) if self._fft3d_n_points is not None: self.add_command_line( 'fft3d.reciprocal_space_grid.n_points=%i' %self._fft3d_n_points) if self._close_to_spindle_cutoff is not None: self.add_command_line( 'close_to_spindle_cutoff=%f' %self._close_to_spindle_cutoff) if self._outlier_algorithm: self.add_command_line('outlier.algorithm=%s' % self._outlier_algorithm) if self._max_cell: self.add_command_line('max_cell=%d' % self._max_cell) if self._min_cell: self.add_command_line('min_cell=%d' % self._min_cell) if self._histogram_binning is not None: self.add_command_line('max_cell_estimation.histogram_binning=%s' %self._histogram_binning) if self._d_min_start: self.add_command_line('d_min_start=%f' % self._d_min_start) if self._indxr_input_lattice is not None: from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number self._symm = lattice_to_spacegroup_number( self._indxr_input_lattice) self.add_command_line('known_symmetry.space_group=%s' % self._symm) if self._indxr_input_cell is not None: self.add_command_line( 'known_symmetry.unit_cell="%s,%s,%s,%s,%s,%s"' %self._indxr_input_cell) if self._maximum_spot_error: self.add_command_line('maximum_spot_error=%.f' % self._maximum_spot_error) if self._detector_fix: self.add_command_line('detector.fix=%s' % self._detector_fix) if self._beam_fix: self.add_command_line('beam.fix=%s' % self._beam_fix) if self._phil_file is not None: self.add_command_line("%s" %self._phil_file) self._experiment_filename = os.path.join( self.get_working_directory(), '%d_experiments.json' %self.get_xpid()) self._indexed_filename = os.path.join( self.get_working_directory(), '%d_indexed.pickle' %self.get_xpid()) self.add_command_line("output.experiments=%s" %self._experiment_filename) self.add_command_line("output.reflections=%s" %self._indexed_filename) self.start() self.close_wait() self.check_for_errors() from dials.array_family import flex from dxtbx.serialize import load self._experiment_list = load.experiment_list(self._experiment_filename) self._reflections = flex.reflection_table.from_pickle( self._indexed_filename) crystal = self._experiment_list.crystals()[0] self._p1_cell = crystal.get_unit_cell().parameters() refined_sel = self._reflections.get_flags(self._reflections.flags.used_in_refinement) refl = self._reflections.select(refined_sel) xc, yc, zc = refl['xyzcal.px'].parts() xo, yo, zo = refl['xyzobs.px.value'].parts() import math self._nref = refl.size() self._rmsd_x = math.sqrt(flex.mean(flex.pow2(xc - xo))) self._rmsd_y = math.sqrt(flex.mean(flex.pow2(yc - yo))) self._rmsd_z = math.sqrt(flex.mean(flex.pow2(zc - zo))) return
def integrate(experiment): from dials.algorithms.spot_prediction import PixelToMillerIndex from dials.array_family import flex from math import floor, sqrt from collections import defaultdict detector = experiment.detector assert len(detector) == 1 panel = detector[0] xsize, ysize = panel.get_image_size() transform = PixelToMillerIndex(experiment.beam, experiment.detector, experiment.crystal) data = experiment.imageset.get_raw_data(0)[0] mask = flex.bool(flex.grid(ysize, xsize), False) reflections = defaultdict(list) print("Doing pixel labelling") for j in range(ysize): for i in range(xsize): h = transform.h(0, i, j) h0 = tuple(map(lambda x: int(floor(x + 0.5)), h)) d = sqrt(sum(map(lambda x, y: (x - y)**2, h, h0))) # if not hasattr(reflections[h0], "xd"): # reflections[h0].xd = d # reflections[h0].xc = i # reflections[h0].yc = j # elif reflections[h0].xd > d: # reflections[h0].xd = d # reflections[h0].xc = i # reflections[h0].yc = j if d < 0.3: mask[j, i] = True reflections[h0].append((j, i)) # from matplotlib import pylab # #pylab.imshow(mask.as_numpy_array(), interpolation='none') # pylab.show() print("Integrating reflections") miller_index = flex.miller_index() intensity = flex.double() variance = flex.double() bbox = flex.int6() xyz = flex.vec3_double() for h, r in reflections.iteritems(): # xc = r.xc # yc = r.yc b_sum = 0 f_sum = 0 b_cnt = 0 f_cnt = 0 for i in range(len(r)): y, x = r[i] m = mask[y, x] if data[y, x] >= 0: if m: f_sum += data[y, x] f_cnt += 1 else: b_sum += data[y, x] b_cnt += 1 Y, X = zip(*r) x0, x1, y0, y1 = min(X), max(X), min(Y), max(Y) if f_cnt > 0 and b_cnt > 0: B = b_sum / b_cnt I = f_sum - B * f_cnt V = f_sum + B * (1 + f_cnt / b_cnt) miller_index.append(h) intensity.append(I) variance.append(V) bbox.append((x0, x1, y0, y1, 0, 1)) # xyz.append((xc, yc, 0)) print("Integrated %d reflections" % len(reflections)) print(flex.min(intensity), flex.max(intensity), flex.mean(intensity)) reflections = flex.reflection_table() reflections["miller_index"] = miller_index reflections["intensity.sum.value"] = intensity reflections["intensity.sum.variance"] = variance reflections["bbox"] = bbox reflections["panel"] = flex.size_t(len(reflections), 0) reflections["id"] = flex.size_t(len(reflections), 0) # reflections["xyzcal.px"] = xyz # reflections["xyzobs.px"] = xyz reflections.set_flags(flex.size_t(range(len(reflections))), reflections.flags.integrated_sum) return reflections
def run_indexing( reflections, experiment, working_directory, extra_args, expected_unit_cell, expected_rmsds, expected_hall_symbol, n_expected_lattices=1, relative_length_tolerance=0.005, absolute_angle_tolerance=0.5, ): commands = ["dials.index"] if isinstance(reflections, list): commands.extend(reflections) else: commands.append(reflections) if isinstance(experiment, list): commands.extend(experiment) else: commands.append(experiment) commands.extend(extra_args) result = procrunner.run(commands, working_directory=working_directory) assert not result.returncode and not result.stderr out_expts = working_directory.join("indexed.expt") out_refls = working_directory.join("indexed.refl") assert out_expts.check() assert out_refls.check() experiments_list = load.experiment_list(out_expts.strpath, check_format=False) assert len(experiments_list.crystals()) == n_expected_lattices indexed_reflections = flex.reflection_table.from_file(out_refls.strpath) indexed_reflections.assert_experiment_identifiers_are_consistent( experiments_list) rmsds = None for i, experiment in enumerate(experiments_list): assert unit_cells_are_similar( experiment.crystal.get_unit_cell(), expected_unit_cell, relative_length_tolerance=relative_length_tolerance, absolute_angle_tolerance=absolute_angle_tolerance, ), ( experiment.crystal.get_unit_cell().parameters(), expected_unit_cell.parameters(), ) sg = experiment.crystal.get_space_group() assert sg.type().hall_symbol() == expected_hall_symbol, ( sg.type().hall_symbol(), expected_hall_symbol, ) reflections = indexed_reflections.select( indexed_reflections["id"] == i) mi = reflections["miller_index"] assert (mi != (0, 0, 0)).count(False) == 0 reflections = reflections.select(mi != (0, 0, 0)) reflections = reflections.select( reflections.get_flags(reflections.flags.used_in_refinement)) assert len(reflections) > 0 obs_x, obs_y, obs_z = reflections["xyzobs.mm.value"].parts() calc_x, calc_y, calc_z = reflections["xyzcal.mm"].parts() rmsd_x = flex.mean(flex.pow2(obs_x - calc_x))**0.5 rmsd_y = flex.mean(flex.pow2(obs_y - calc_y))**0.5 rmsd_z = flex.mean(flex.pow2(obs_z - calc_z))**0.5 rmsds = (rmsd_x, rmsd_y, rmsd_z) for actual, expected in zip(rmsds, expected_rmsds): assert actual <= expected, "%s %s" % (rmsds, expected_rmsds) assert experiment.identifier != "" expt = ExperimentList() expt.append(experiment) reflections.assert_experiment_identifiers_are_consistent(expt) return _indexing_result(indexed_reflections, experiments_list, rmsds)
def __del__(self): values = self.parameterization(self.x) print >> self.out, "FINALMODEL", print >> self.out, "rms %10.3f" % math.sqrt( flex.mean(self.func * self.func)), values.show(self.out)
def plot_one_model(self, nrow, out): fig = plt.subplot(self.gs[nrow * self.ncols]) two_thetas = self.reduction.get_two_theta_deg() degrees = self.reduction.get_delta_psi_deg() if self.color_encoding == "conventional": positive = (self.reduction.i_sigi >= 0.) fig.plot(two_thetas.select(positive), degrees.select(positive), "bo") fig.plot(two_thetas.select(~positive), degrees.select(~positive), "r+") elif self.color_encoding == "I/sigma": positive = (self.reduction.i_sigi >= 0.) tt_selected = two_thetas.select(positive) dp_selected = degrees.select(positive) i_sigi_select = self.reduction.i_sigi.select(positive) order = flex.sort_permutation(i_sigi_select) tt_selected = tt_selected.select(order) dp_selected = dp_selected.select(order) i_sigi_selected = i_sigi_select.select(order) from matplotlib.colors import Normalize dnorm = Normalize() dcolors = i_sigi_selected.as_numpy_array() dnorm.autoscale(dcolors) N = len(dcolors) CMAP = plt.get_cmap("rainbow") if self.refined.get("partiality_array", None) is None: for n in xrange(N): fig.plot([tt_selected[n]], [dp_selected[n]], color=CMAP(dnorm(dcolors[n])), marker=".", markersize=10) else: partials = self.refined.get("partiality_array") partials_select = partials.select(positive) partials_selected = partials_select.select(order) assert len(partials) == len(positive) for n in xrange(N): fig.plot([tt_selected[n]], [dp_selected[n]], color=CMAP(dnorm(dcolors[n])), marker=".", markersize=20 * partials_selected[n]) # change the markersize to indicate partiality. negative = (self.reduction.i_sigi < 0.) fig.plot(two_thetas.select(negative), degrees.select(negative), "r+", linewidth=1) else: strong = (self.reduction.i_sigi >= 10.) positive = ((~strong) & (self.reduction.i_sigi >= 0.)) negative = (self.reduction.i_sigi < 0.) assert (strong.count(True) + positive.count(True) + negative.count(True) == len(self.reduction.i_sigi)) fig.plot(two_thetas.select(positive), degrees.select(positive), "bo") fig.plot(two_thetas.select(strong), degrees.select(strong), marker='.', linestyle='None', markerfacecolor='#00ee00', markersize=10) fig.plot(two_thetas.select(negative), degrees.select(negative), "r+") # indicate the imposed resolution filter wavelength = self.reduction.experiment.beam.get_wavelength() imposed_res_filter = self.reduction.get_imposed_res_filter(out) resolution_markers = [ a for a in [imposed_res_filter, self.reduction.measurements.d_min()] if a is not None ] for RM in resolution_markers: two_th = (180. / math.pi) * 2. * math.asin(wavelength / (2. * RM)) plt.plot([two_th, two_th], [self.AD1TF7B_MAXDP * -0.8, self.AD1TF7B_MAXDP * 0.8], 'k-') plt.text(two_th, self.AD1TF7B_MAXDP * -0.9, "%4.2f" % RM) #indicate the linefit mean = flex.mean(degrees) minplot = flex.min(two_thetas) plt.plot([0, minplot], [mean, mean], "k-") LR = flex.linear_regression(two_thetas, degrees) model_y = LR.slope() * two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") #Now let's take care of the red and green lines. half_mosaic_rotation_deg = self.refined["half_mosaic_rotation_deg"] mosaic_domain_size_ang = self.refined["mosaic_domain_size_ang"] red_curve_domain_size_ang = self.refined.get( "red_curve_domain_size_ang", mosaic_domain_size_ang) a_step = self.AD1TF7B_MAX2T / 50. a_range = flex.double([a_step * x for x in xrange(1, 50) ]) # domain two-theta array #Bragg law [d=L/2sinTH] d_spacing = (wavelength / (2. * flex.sin(math.pi * a_range / 360.))) # convert two_theta to a delta-psi. Formula for Deffective [Dpsi=d/2Deff] inner_phi_deg = flex.asin( (d_spacing / (2. * red_curve_domain_size_ang))) * (180. / math.pi) outer_phi_deg = flex.asin((d_spacing / (2.*mosaic_domain_size_ang)) + \ half_mosaic_rotation_deg*math.pi/180. )*(180./math.pi) plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots\n%s" % (2. * half_mosaic_rotation_deg, mosaic_domain_size_ang, len(two_thetas), os.path.basename(self.reduction.filename))) plt.plot(a_range, inner_phi_deg, "r-") plt.plot(a_range, -inner_phi_deg, "r-") plt.plot(a_range, outer_phi_deg, "g-") plt.plot(a_range, -outer_phi_deg, "g-") plt.xlim([0, self.AD1TF7B_MAX2T]) plt.ylim([-self.AD1TF7B_MAXDP, self.AD1TF7B_MAXDP]) #second plot shows histogram fig = plt.subplot(self.gs[1 + nrow * self.ncols]) plt.xlim([-self.AD1TF7B_MAXDP, self.AD1TF7B_MAXDP]) nbins = 50 n, bins, patches = plt.hist( dp_selected, nbins, range=(-self.AD1TF7B_MAXDP, self.AD1TF7B_MAXDP), weights=self.reduction.i_sigi.select(positive), normed=0, facecolor="orange", alpha=0.75) #ersatz determine the median i_sigi point: isi_positive = self.reduction.i_sigi.select(positive) isi_order = flex.sort_permutation(isi_positive) reordered = isi_positive.select(isi_order) isi_median = reordered[int(len(isi_positive) * 0.9)] isi_top_half_selection = (isi_positive > isi_median) n, bins, patches = plt.hist( dp_selected.select(isi_top_half_selection), nbins, range=(-self.AD1TF7B_MAXDP, self.AD1TF7B_MAXDP), weights=isi_positive.select(isi_top_half_selection), normed=0, facecolor="#ff0000", alpha=0.75) plt.xlabel("(degrees)") plt.title("Weighted histogram of Delta-psi")
def show_reflections( reflections, show_intensities=False, show_profile_fit=False, show_centroids=False, show_all_reflection_data=False, show_flags=False, max_reflections=None, show_identifiers=False, ): text = [] from orderedset import OrderedSet formats = { "miller_index": "%i, %i, %i", "d": "%.2f", "qe": "%.3f", "dqe": "%.3f", "id": "%i", "imageset_id": "%i", "panel": "%i", "flags": "%i", "background.mean": "%.1f", "background.dispersion": "%.1f", "background.mse": "%.1f", "background.sum.value": "%.1f", "background.sum.variance": "%.1f", "intensity.prf.value": "%.1f", "intensity.prf.variance": "%.1f", "intensity.sum.value": "%.1f", "intensity.sum.variance": "%.1f", "intensity.cor.value": "%.1f", "intensity.cor.variance": "%.1f", "intensity.scale.value": "%.1f", "intensity.scale.variance": "%.1f", "Ih_values": "%.1f", "lp": "%.3f", "num_pixels.background": "%i", "num_pixels.background_used": "%i", "num_pixels.foreground": "%i", "num_pixels.valid": "%i", "partial_id": "%i", "partiality": "%.4f", "profile.correlation": "%.3f", "profile.rmsd": "%.3f", "xyzcal.mm": "%.2f, %.2f, %.2f", "xyzcal.px": "%.2f, %.2f, %.2f", "delpsical.rad": "%.3f", "delpsical2": "%.3f", "delpsical.weights": "%.3f", "xyzobs.mm.value": "%.2f, %.2f, %.2f", "xyzobs.mm.variance": "%.4e, %.4e, %.4e", "xyzobs.px.value": "%.2f, %.2f, %.2f", "xyzobs.px.variance": "%.4f, %.4f, %.4f", "s1": "%.4f, %.4f, %.4f", "s2": "%.4f, %.4f, %.4f", "shoebox": "%.1f", "rlp": "%.4f, %.4f, %.4f", "zeta": "%.3f", "x_resid": "%.3f", "x_resid2": "%.3f", "y_resid": "%.3f", "y_resid2": "%.3f", "kapton_absorption_correction": "%.3f", "kapton_absorption_correction_sigmas": "%.3f", "inverse_scale_factor": "%.3f", "inverse_scale_factor_variance": "%.3f", } for rlist in reflections: from dials.algorithms.shoebox import MaskCode foreground_valid = MaskCode.Valid | MaskCode.Foreground text.append("") text.append(f"Reflection list contains {len(rlist)} reflections") if len(rlist) == 0: continue rows = [["Column", "min", "max", "mean"]] for k, col in rlist.cols(): if k in formats and "%" not in formats.get(k, "%s"): # Allow blanking out of entries that wouldn't make sense rows.append([ k, formats.get(k, "%s"), formats.get(k, "%s"), formats.get(k, "%s"), ]) elif type(col) in (flex.double, flex.int, flex.size_t): if type(col) in (flex.int, flex.size_t): col = col.as_double() rows.append([ k, formats.get(k, "%s") % flex.min(col), formats.get(k, "%s") % flex.max(col), formats.get(k, "%s") % flex.mean(col), ]) elif type(col) in (flex.vec3_double, flex.miller_index): if isinstance(col, flex.miller_index): col = col.as_vec3_double() rows.append([ k, formats.get(k, "%s") % col.min(), formats.get(k, "%s") % col.max(), formats.get(k, "%s") % col.mean(), ]) elif isinstance(col, flex.shoebox): rows.append([k, "", "", ""]) si = col.summed_intensity().observed_value() rows.append([ " summed I", formats.get(k, "%s") % flex.min(si), formats.get(k, "%s") % flex.max(si), formats.get(k, "%s") % flex.mean(si), ]) x1, x2, y1, y2, z1, z2 = col.bounding_boxes().parts() bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double() rows.append([ " N pix", formats.get(k, "%s") % flex.min(bbox_sizes), formats.get(k, "%s") % flex.max(bbox_sizes), formats.get(k, "%s") % flex.mean(bbox_sizes), ]) fore_valid = col.count_mask_values( foreground_valid).as_double() rows.append([ " N valid foreground pix", formats.get(k, "%s") % flex.min(fore_valid), formats.get(k, "%s") % flex.max(fore_valid), formats.get(k, "%s") % flex.mean(fore_valid), ]) text.append(tabulate(rows, headers="firstrow")) if show_flags: text.append(_create_flag_count_table(rlist)) if show_identifiers: if rlist.experiment_identifiers(): text.append( """Experiment identifiers id-map values:\n%s""" % ("\n".join( "id:" + str(k) + " -> experiment identifier:" + str(rlist.experiment_identifiers()[k]) for k in rlist.experiment_identifiers().keys()))) intensity_keys = ( "miller_index", "d", "intensity.prf.value", "intensity.prf.variance", "intensity.sum.value", "intensity.sum.variance", "background.mean", "profile.correlation", "profile.rmsd", ) profile_fit_keys = ("miller_index", "d") centroid_keys = ( "miller_index", "d", "xyzcal.mm", "xyzcal.px", "xyzobs.mm.value", "xyzobs.mm.variance", "xyzobs.px.value", "xyzobs.px.variance", ) keys_to_print = OrderedSet() if show_intensities: for k in intensity_keys: keys_to_print.add(k) if show_profile_fit: for k in profile_fit_keys: keys_to_print.add(k) if show_centroids: for k in centroid_keys: keys_to_print.add(k) if show_all_reflection_data: for k in formats: keys_to_print.add(k) def format_column(key, data, format_strings=None): if isinstance(data, flex.vec3_double): c_strings = [ c.as_string(format_strings[i].strip()) for i, c in enumerate(data.parts()) ] elif isinstance(data, flex.miller_index): c_strings = [ c.as_string(format_strings[i].strip()) for i, c in enumerate(data.as_vec3_double().parts()) ] elif isinstance(data, flex.size_t): c_strings = [data.as_int().as_string(format_strings[0].strip())] elif isinstance(data, flex.shoebox): x1, x2, y1, y2, z1, z2 = data.bounding_boxes().parts() bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double() c_strings = [bbox_sizes.as_string(format_strings[0].strip())] key += " (N pix)" else: c_strings = [data.as_string(format_strings[0].strip())] column = flex.std_string() max_element_lengths = [c.max_element_length() for c in c_strings] for i in range(len(c_strings[0])): column.append(f"%{len(key)}s" % ", ".join( ("%%%is" % max_element_lengths[j]) % c_strings[j][i] for j in range(len(c_strings)))) return column if keys_to_print: keys = [k for k in keys_to_print if k in rlist] if max_reflections is not None: max_reflections = min(len(rlist), max_reflections) else: max_reflections = len(rlist) columns = [] for k in keys: columns.append( format_column(k, rlist[k], format_strings=formats[k].split(","))) text.append("") text.append("Printing %i of %i reflections:" % (max_reflections, len(rlist))) line = [] for j in range(len(columns)): key = keys[j] if key == "shoebox": key += " (N pix)" width = max(len(key), columns[j].max_element_length()) line.append("%%%is" % width % key) text.append(" ".join(line)) for i in range(max_reflections): line = (c[i] for c in columns) text.append(" ".join(line)) return "\n".join(text)
def _create_summation_matrix(self): """ "Create a summation matrix to allow sums into intensity bins. This routine attempts to bin into bins equally spaced in log(intensity), to give a representative sample across all intensities. To avoid undersampling, it is required that there are at least 100 reflections per intensity bin unless there are very few reflections.""" n = self.Ih_table.size self.binning_info["n_reflections"] = n summation_matrix = sparse.matrix(n, self.n_bins) Ih = self.Ih_table.Ih_values * self.Ih_table.inverse_scale_factors size_order = flex.sort_permutation(Ih, reverse=True) Imax = max(Ih) Imin = max(1.0, min(Ih)) # avoid log issues spacing = (log(Imax) - log(Imin)) / float(self.n_bins) boundaries = [Imax] + [ exp(log(Imax) - (i * spacing)) for i in range(1, self.n_bins + 1) ] boundaries[-1] = min(Ih) - 0.01 self.binning_info["bin_boundaries"] = boundaries self.binning_info["refl_per_bin"] = flex.double() n_cumul = 0 if Ih.size() > 100 * self.min_reflections_required: self.min_reflections_required = int(Ih.size() / 100.0) min_per_bin = min(self.min_reflections_required, int(n / (3.0 * self.n_bins))) for i in range(len(boundaries) - 1): maximum = boundaries[i] minimum = boundaries[i + 1] sel1 = Ih <= maximum sel2 = Ih > minimum sel = sel1 & sel2 isel = sel.iselection() n_in_bin = isel.size() if n_in_bin < min_per_bin: # need more in this bin m = n_cumul + min_per_bin if m < n: # still some refl left to use idx = size_order[m] intensity = Ih[idx] boundaries[i + 1] = intensity minimum = boundaries[i + 1] sel = sel1 & (Ih > minimum) isel = sel.iselection() n_in_bin = isel.size() self.binning_info["refl_per_bin"].append(n_in_bin) for j in isel: summation_matrix[j, i] = 1 n_cumul += n_in_bin cols_to_del = [] for i, col in enumerate(summation_matrix.cols()): if col.non_zeroes < min_per_bin - 5: cols_to_del.append(i) n_new_cols = summation_matrix.n_cols - len(cols_to_del) if n_new_cols == self.n_bins: for i in range(len(boundaries) - 1): maximum = boundaries[i] minimum = boundaries[i + 1] sel1 = Ih <= maximum sel2 = Ih > minimum sel = sel1 & sel2 m = flex.mean(Ih.select(sel)) self.binning_info["mean_intensities"].append(m) return summation_matrix new_sum_matrix = sparse.matrix(summation_matrix.n_rows, n_new_cols) next_col = 0 refl_per_bin = flex.double() new_bounds = [] for i, col in enumerate(summation_matrix.cols()): if i not in cols_to_del: new_sum_matrix[:, next_col] = col next_col += 1 new_bounds.append(boundaries[i]) refl_per_bin.append(self.binning_info["refl_per_bin"][i]) self.binning_info["refl_per_bin"] = refl_per_bin new_bounds.append(boundaries[-1]) self.binning_info["bin_boundaries"] = new_bounds for i in range(len(new_bounds) - 1): maximum = new_bounds[i] minimum = new_bounds[i + 1] sel1 = Ih <= maximum sel2 = Ih > minimum sel = sel1 & sel2 m = flex.mean(Ih.select(sel)) self.binning_info["mean_intensities"].append(m) return new_sum_matrix
def run(self, method): from xia2.Handlers.Streams import Debug Debug.write("Running dials.index") self.clear_command_line() for f in self._sweep_filenames: self.add_command_line(f) for f in self._spot_filenames: self.add_command_line(f) if len(self._sweep_filenames) > 1: self.add_command_line("auto_reduction.action=fix") self.add_command_line("indexing.method=%s" % method) nproc = PhilIndex.params.xia2.settings.multiprocessing.nproc self.set_cpu_threads(nproc) self.add_command_line("indexing.nproc=%i" % nproc) if PhilIndex.params.xia2.settings.small_molecule: self.add_command_line("filter_ice=false") if self._reflections_per_degree is not None: self.add_command_line("reflections_per_degree=%i" % self._reflections_per_degree) if self._fft3d_n_points is not None: self.add_command_line( "fft3d.reciprocal_space_grid.n_points=%i" % self._fft3d_n_points) if self._close_to_spindle_cutoff is not None: self.add_command_line("close_to_spindle_cutoff=%f" % self._close_to_spindle_cutoff) if self._outlier_algorithm: self.add_command_line("outlier.algorithm=%s" % self._outlier_algorithm) if self._max_cell: self.add_command_line("max_cell=%g" % self._max_cell) if self._max_cell_max_height_fraction is not None: self.add_command_line("max_height_fraction=%g" % self._max_cell_max_height_fraction) if self._min_cell: self.add_command_line("min_cell=%d" % self._min_cell) if self._histogram_binning is not None: self.add_command_line( "max_cell_estimation.histogram_binning=%s" % self._histogram_binning) if self._nearest_neighbor_percentile is not None: self.add_command_line( "max_cell_estimation.nearest_neighbor_percentile=%s" % self._nearest_neighbor_percentile) if self._d_min_start: self.add_command_line("d_min_start=%f" % self._d_min_start) if self._indxr_input_lattice is not None: from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number self._symm = lattice_to_spacegroup_number( self._indxr_input_lattice) self.add_command_line("known_symmetry.space_group=%s" % self._symm) if self._indxr_input_cell is not None: self.add_command_line( 'known_symmetry.unit_cell="%s,%s,%s,%s,%s,%s"' % self._indxr_input_cell) if self._maximum_spot_error: self.add_command_line("maximum_spot_error=%.f" % self._maximum_spot_error) if self._detector_fix: self.add_command_line("detector.fix=%s" % self._detector_fix) if self._beam_fix: self.add_command_line("beam.fix=%s" % self._beam_fix) if self._phil_file is not None: self.add_command_line("%s" % self._phil_file) self._experiment_filename = os.path.join( self.get_working_directory(), "%d_indexed.expt" % self.get_xpid()) self._indexed_filename = os.path.join( self.get_working_directory(), "%d_indexed.refl" % self.get_xpid()) self.add_command_line("output.experiments=%s" % self._experiment_filename) self.add_command_line("output.reflections=%s" % self._indexed_filename) self.start() self.close_wait() if not os.path.isfile( self._experiment_filename) or not os.path.isfile( self._indexed_filename): # Indexing failed with open(self.get_log_file(), "r") as fh: if "No suitable lattice could be found" in fh.read(): raise libtbx.utils.Sorry( "No suitable indexing solution could be found.\n\n" "You can view the reciprocal space with:\n" "dials.reciprocal_lattice_viewer %s" % " ".join( os.path.normpath( os.path.join(self.get_working_directory(), p)) for p in self._sweep_filenames + self._spot_filenames)) else: raise RuntimeError( "dials.index failed, see log file for more details: %s" % self.get_log_file()) self.check_for_errors() for record in self.get_all_output(): if "Too few reflections to parameterise" in record: Debug.write(record.strip()) from dials.array_family import flex from dxtbx.serialize import load self._experiment_list = load.experiment_list( self._experiment_filename) self._reflections = flex.reflection_table.from_file( self._indexed_filename) crystal = self._experiment_list.crystals()[0] self._p1_cell = crystal.get_unit_cell().parameters() refined_sel = self._reflections.get_flags( self._reflections.flags.used_in_refinement) refl = self._reflections.select(refined_sel) xc, yc, zc = refl["xyzcal.px"].parts() xo, yo, zo = refl["xyzobs.px.value"].parts() import math self._nref = refl.size() self._rmsd_x = math.sqrt(flex.mean(flex.pow2(xc - xo))) self._rmsd_y = math.sqrt(flex.mean(flex.pow2(yc - yo))) self._rmsd_z = math.sqrt(flex.mean(flex.pow2(zc - zo)))
def __call__(self): """Determine optimal mosaicity and domain size model (monochromatic)""" RR = self.refinery.predict_for_reflection_table(self.reflections) excursion_rad = RR["delpsical.rad"] delta_psi_deg = excursion_rad * 180./math.pi print print flex.max(delta_psi_deg), flex.min(delta_psi_deg) mean_excursion = flex.mean(delta_psi_deg) print "The mean excursion is %7.3f degrees, r.m.s.d %7.3f"%(mean_excursion, math.sqrt(flex.mean(RR["delpsical2"]))) crystal = self.experiments[0].crystal beam = self.experiments[0].beam miller_indices = self.reflections["miller_index"] # FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection two_thetas = crystal.get_unit_cell().two_theta(miller_indices,beam.get_wavelength(),deg=True) dspacings = crystal.get_unit_cell().d(miller_indices) dspace_sq = dspacings * dspacings # First -- try to get a reasonable envelope for the observed excursions. ## minimum of three regions; maximum of 50 measurements in each bin print "fitting parameters on %d spots"%len(excursion_rad) n_bins = min(max(3, len(excursion_rad)//25),50) bin_sz = len(excursion_rad)//n_bins print "nbins",n_bins,"bin_sz",bin_sz order = flex.sort_permutation(two_thetas) two_thetas_env = flex.double() dspacings_env = flex.double() excursion_rads_env = flex.double() for x in xrange(0,n_bins): subset = order[x*bin_sz:(x+1)*bin_sz] two_thetas_env.append(flex.mean(two_thetas.select(subset))) dspacings_env.append(flex.mean(dspacings.select(subset))) excursion_rads_env.append(flex.max(flex.abs(excursion_rad.select(subset)))) # Second -- parameter fit ## solve the normal equations sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env) sum_inv_u = flex.sum(dspacings_env) sum_te_u = flex.sum(dspacings_env * excursion_rads_env) sum_te = flex.sum(excursion_rads_env) Normal_Mat = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env))) Vector = col((sum_te_u, sum_te)) solution = Normal_Mat.inverse() * Vector s_ang = 1./(2*solution[0]) print "Best LSQ fit Scheerer domain size is %9.2f ang"%( s_ang) tan_phi_rad = dspacings / (2. * s_ang) tan_phi_deg = tan_phi_rad * 180./math.pi k_degrees = solution[1]* 180./math.pi print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees) tan_outer_deg = tan_phi_deg + k_degrees from xfel.mono_simulation.max_like import minimizer # coerce the estimates to be positive for max-likelihood lower_limit_domain_size = math.pow(crystal.get_unit_cell().volume(), 1./3.)*3 # params.refinement.domain_size_lower_limit d_estimate = max(s_ang, lower_limit_domain_size) M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]), Deff = d_estimate) print "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas)) tan_phi_rad_ML = dspacings / (2. / M.x[0]) tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi self.nv_acceptance_flags = flex.abs(delta_psi_deg) < tan_outer_deg_ML if self.graph_verbose: #params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit AD1TF7B_MAX2T = 30. AD1TF7B_MAXDP = 1. from matplotlib import pyplot as plt plt.plot(two_thetas, delta_psi_deg, "bo") minplot = flex.min(two_thetas) plt.plot([0,minplot],[mean_excursion,mean_excursion],"k-") LR = flex.linear_regression(two_thetas, delta_psi_deg) model_y = LR.slope()*two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas))) plt.plot(two_thetas, tan_phi_deg_ML, "r.") plt.plot(two_thetas, -tan_phi_deg_ML, "r.") plt.plot(two_thetas, tan_outer_deg_ML, "g.") plt.plot(two_thetas, -tan_outer_deg_ML, "g.") plt.xlim([0,AD1TF7B_MAX2T]) plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP]) plt.show() plt.close() from xfel.mono_simulation.util import green_curve_area self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML) print "The green curve area is ", self.green_curve_area crystal._ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi) crystal._ML_domain_size_ang = 2./M.x[0] self._ML_full_mosaicity_rad = M.x[1] self._ML_domain_size_ang = 2./M.x[0] #params.refinement.mosaic.model_expansion_factor """The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes as close as possible to # of observed reflections input, in the last integration call. Determine this by inspecting the output log file interactively. Do not exceed the bare minimum threshold needed. The intention is to find an optimal value, global for a given dataset.""" model_expansion_factor = 1.4 crystal._ML_half_mosaicity_deg *= model_expansion_factor crystal._ML_domain_size_ang /= model_expansion_factor return crystal