def __call__(self): """Determine optimal mosaicity and domain size model (monochromatic)""" if self.refinery is None: RR = self.reflections else: RR = self.refinery.predict_for_reflection_table(self.reflections) all_crystals = [] self.nv_acceptance_flags = flex.bool(len(self.reflections["id"])) from dxtbx.model import MosaicCrystalSauter2014 for iid, experiment in enumerate(self.experiments): excursion_rad = RR["delpsical.rad"].select(RR["id"] == iid) delta_psi_deg = excursion_rad * 180.0 / math.pi logger.info("") logger.info("%s %s", flex.max(delta_psi_deg), flex.min(delta_psi_deg)) mean_excursion = flex.mean(delta_psi_deg) logger.info( "The mean excursion is %7.3f degrees, r.m.s.d %7.3f", mean_excursion, math.sqrt(flex.mean(RR["delpsical2"].select(RR["id"] == iid))), ) crystal = MosaicCrystalSauter2014(self.experiments[iid].crystal) self.experiments[iid].crystal = crystal beam = self.experiments[iid].beam miller_indices = self.reflections["miller_index"].select( self.reflections["id"] == iid) # FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection two_thetas = crystal.get_unit_cell().two_theta( miller_indices, beam.get_wavelength(), deg=True) dspacings = crystal.get_unit_cell().d(miller_indices) # First -- try to get a reasonable envelope for the observed excursions. # minimum of three regions; maximum of 50 measurements in each bin logger.info("fitting parameters on %d spots", len(excursion_rad)) n_bins = min(max(3, len(excursion_rad) // 25), 50) bin_sz = len(excursion_rad) // n_bins logger.info("nbins %s bin_sz %s", n_bins, bin_sz) order = flex.sort_permutation(two_thetas) two_thetas_env = flex.double() dspacings_env = flex.double() excursion_rads_env = flex.double() for x in range(0, n_bins): subset = order[x * bin_sz:(x + 1) * bin_sz] two_thetas_env.append(flex.mean(two_thetas.select(subset))) dspacings_env.append(flex.mean(dspacings.select(subset))) excursion_rads_env.append( flex.max(flex.abs(excursion_rad.select(subset)))) # Second -- parameter fit # solve the normal equations sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env) sum_inv_u = flex.sum(dspacings_env) sum_te_u = flex.sum(dspacings_env * excursion_rads_env) sum_te = flex.sum(excursion_rads_env) Normal_Mat = sqr( (sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env))) Vector = col((sum_te_u, sum_te)) solution = Normal_Mat.inverse() * Vector s_ang = 1.0 / (2 * solution[0]) logger.info("Best LSQ fit Scheerer domain size is %9.2f ang", s_ang) k_degrees = solution[1] * 180.0 / math.pi logger.info( "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f", 2 * k_degrees, k_degrees, ) from xfel.mono_simulation.max_like import minimizer # coerce the estimates to be positive for max-likelihood lower_limit_domain_size = ( math.pow(crystal.get_unit_cell().volume(), 1.0 / 3.0) * 3 ) # params.refinement.domain_size_lower_limit d_estimate = max(s_ang, lower_limit_domain_size) M = minimizer( d_i=dspacings, psi_i=excursion_rad, eta_rad=abs(2.0 * solution[1]), Deff=d_estimate, ) logger.info( "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots", M.x[1] * 180.0 / math.pi, 2.0 / M.x[0], len(two_thetas), ) tan_phi_rad_ML = dspacings / (2.0 / M.x[0]) tan_phi_deg_ML = tan_phi_rad_ML * 180.0 / math.pi tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180.0 / math.pi # Only set the flags for those reflections that were indexed for this lattice self.nv_acceptance_flags.set_selected( self.reflections["id"] == iid, flex.abs(delta_psi_deg) < tan_outer_deg_ML, ) if ( self.graph_verbose ): # params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit AD1TF7B_MAX2T = 30.0 AD1TF7B_MAXDP = 1.0 from matplotlib import pyplot as plt plt.plot(two_thetas, delta_psi_deg, "bo") minplot = flex.min(two_thetas) plt.plot([0, minplot], [mean_excursion, mean_excursion], "k-") LR = flex.linear_regression(two_thetas, delta_psi_deg) model_y = LR.slope() * two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") plt.title( "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" % (M.x[1] * 180.0 / math.pi, 2.0 / M.x[0], len(two_thetas))) plt.plot(two_thetas, tan_phi_deg_ML, "r.") plt.plot(two_thetas, -tan_phi_deg_ML, "r.") plt.plot(two_thetas, tan_outer_deg_ML, "g.") plt.plot(two_thetas, -tan_outer_deg_ML, "g.") plt.xlim([0, AD1TF7B_MAX2T]) plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP]) plt.show() plt.close() from xfel.mono_simulation.util import green_curve_area self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML) logger.info("The green curve area is %s", self.green_curve_area) crystal.set_half_mosaicity_deg(M.x[1] * 180.0 / (2.0 * math.pi)) crystal.set_domain_size_ang(2.0 / M.x[0]) self._ML_full_mosaicity_rad = M.x[1] self._ML_domain_size_ang = 2.0 / M.x[0] # params.refinement.mosaic.model_expansion_factor """The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes as close as possible to # of observed reflections input, in the last integration call. Determine this by inspecting the output log file interactively. Do not exceed the bare minimum threshold needed. The intention is to find an optimal value, global for a given dataset.""" model_expansion_factor = 1.4 crystal.set_half_mosaicity_deg(crystal.get_half_mosaicity_deg() * model_expansion_factor) crystal.set_domain_size_ang(crystal.get_domain_size_ang() / model_expansion_factor) if (self.ewald_proximal_volume(iid) > self.params.indexing.stills.ewald_proximal_volume_max): raise DialsIndexError("Ewald proximity volume too high, %f" % self.ewald_proximal_volume(iid)) all_crystals.append(crystal) return all_crystals
def __call__(self): """Determine optimal mosaicity and domain size model (monochromatic)""" RR = self.refinery.predict_for_reflection_table(self.reflections) excursion_rad = RR["delpsical.rad"] delta_psi_deg = excursion_rad * 180. / math.pi print print flex.max(delta_psi_deg), flex.min(delta_psi_deg) mean_excursion = flex.mean(delta_psi_deg) print "The mean excursion is %7.3f degrees, r.m.s.d %7.3f" % ( mean_excursion, math.sqrt(flex.mean(RR["delpsical2"]))) crystal = self.experiments[0].crystal beam = self.experiments[0].beam miller_indices = self.reflections["miller_index"] # FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection two_thetas = crystal.get_unit_cell().two_theta(miller_indices, beam.get_wavelength(), deg=True) dspacings = crystal.get_unit_cell().d(miller_indices) dspace_sq = dspacings * dspacings # First -- try to get a reasonable envelope for the observed excursions. ## minimum of three regions; maximum of 50 measurements in each bin print "fitting parameters on %d spots" % len(excursion_rad) n_bins = min(max(3, len(excursion_rad) // 25), 50) bin_sz = len(excursion_rad) // n_bins print "nbins", n_bins, "bin_sz", bin_sz order = flex.sort_permutation(two_thetas) two_thetas_env = flex.double() dspacings_env = flex.double() excursion_rads_env = flex.double() for x in xrange(0, n_bins): subset = order[x * bin_sz:(x + 1) * bin_sz] two_thetas_env.append(flex.mean(two_thetas.select(subset))) dspacings_env.append(flex.mean(dspacings.select(subset))) excursion_rads_env.append( flex.max(flex.abs(excursion_rad.select(subset)))) # Second -- parameter fit ## solve the normal equations sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env) sum_inv_u = flex.sum(dspacings_env) sum_te_u = flex.sum(dspacings_env * excursion_rads_env) sum_te = flex.sum(excursion_rads_env) Normal_Mat = sqr( (sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env))) Vector = col((sum_te_u, sum_te)) solution = Normal_Mat.inverse() * Vector s_ang = 1. / (2 * solution[0]) print "Best LSQ fit Scheerer domain size is %9.2f ang" % (s_ang) tan_phi_rad = dspacings / (2. * s_ang) tan_phi_deg = tan_phi_rad * 180. / math.pi k_degrees = solution[1] * 180. / math.pi print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f" % ( 2 * k_degrees, k_degrees) tan_outer_deg = tan_phi_deg + k_degrees from xfel.mono_simulation.max_like import minimizer # coerce the estimates to be positive for max-likelihood lower_limit_domain_size = math.pow( crystal.get_unit_cell().volume(), 1. / 3.) * 3 # params.refinement.domain_size_lower_limit d_estimate = max(s_ang, lower_limit_domain_size) M = minimizer(d_i=dspacings, psi_i=excursion_rad, eta_rad=abs(2. * solution[1]), Deff=d_estimate) print "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" % ( M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas)) tan_phi_rad_ML = dspacings / (2. / M.x[0]) tan_phi_deg_ML = tan_phi_rad_ML * 180. / math.pi tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180. / math.pi self.nv_acceptance_flags = flex.abs(delta_psi_deg) < tan_outer_deg_ML if self.graph_verbose: #params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit AD1TF7B_MAX2T = 30. AD1TF7B_MAXDP = 1. from matplotlib import pyplot as plt plt.plot(two_thetas, delta_psi_deg, "bo") minplot = flex.min(two_thetas) plt.plot([0, minplot], [mean_excursion, mean_excursion], "k-") LR = flex.linear_regression(two_thetas, delta_psi_deg) model_y = LR.slope() * two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" % (M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas))) plt.plot(two_thetas, tan_phi_deg_ML, "r.") plt.plot(two_thetas, -tan_phi_deg_ML, "r.") plt.plot(two_thetas, tan_outer_deg_ML, "g.") plt.plot(two_thetas, -tan_outer_deg_ML, "g.") plt.xlim([0, AD1TF7B_MAX2T]) plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP]) plt.show() plt.close() from xfel.mono_simulation.util import green_curve_area self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML) print "The green curve area is ", self.green_curve_area crystal._ML_half_mosaicity_deg = M.x[1] * 180. / (2. * math.pi) crystal._ML_domain_size_ang = 2. / M.x[0] self._ML_full_mosaicity_rad = M.x[1] self._ML_domain_size_ang = 2. / M.x[0] #params.refinement.mosaic.model_expansion_factor """The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes as close as possible to # of observed reflections input, in the last integration call. Determine this by inspecting the output log file interactively. Do not exceed the bare minimum threshold needed. The intention is to find an optimal value, global for a given dataset.""" model_expansion_factor = 1.4 crystal._ML_half_mosaicity_deg *= model_expansion_factor crystal._ML_domain_size_ang /= model_expansion_factor return crystal
def refine_rotx_roty2(OO,enable_rotational_target=True): helper = OO.per_frame_helper_factory() helper.restart() if enable_rotational_target: print "Trying least squares minimization of excursions", from scitbx.lstbx import normal_eqns_solving iterations = normal_eqns_solving.naive_iterations( non_linear_ls = helper, gradient_threshold = 1.E-10) results = helper.x print "with %d reflections"%len(OO.parent.indexed_pairs), print "result %6.2f degrees"%(results[1]*180./math.pi), print "result %6.2f degrees"%(results[0]*180./math.pi) if False: # Excursion histogram print "The input mosaicity is %7.3f deg full width"%OO.parent.inputai.getMosaicity() # final histogram if OO.pvr_fix: final = 360.* helper.fvec_callable_pvr(results) else: final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results) rmsdexc = math.sqrt(flex.mean(final*final)) from matplotlib import pyplot as plt nbins = len(final)//20 n,bins,patches = plt.hist(final, nbins, normed=0, facecolor="orange", alpha=0.75) plt.xlabel("Rotation on e1 axis, rmsd %7.3f deg"%rmsdexc) plt.title("Histogram of cctbx.xfel misorientation") plt.axis([-0.5,0.5,0,100]) plt.plot([rmsdexc],[18],"b|") plt.show() # Determine optimal mosaicity and domain size model (monochromatic) if OO.pvr_fix: final = 360.* helper.fvec_callable_pvr(results) else: final = 360.* helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results) #Guard against misindexing -- seen in simulated data, with zone nearly perfectly aligned guard_stats = flex.max(final), flex.min(final) if False and REMOVETEST_KILLING_LEGITIMATE_EXCURSIONS (guard_stats[0] > 2.0 or guard_stats[1] < -2.0): raise Exception("Misindexing diagnosed by meaningless excursion angle (bandpass_gaussian model)"); print "The mean excursion is %7.3f degrees"%(flex.mean(final)) two_thetas = helper.last_set_orientation.unit_cell().two_theta(OO.reserve_indices,OO.central_wavelength_ang,deg=True) dspacings = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) dspace_sq = dspacings * dspacings excursion_rad = final * math.pi/ 180. # First -- try to get a reasonable envelope for the observed excursions. ## minimum of three regions; maximum of 50 measurements in each bin print "fitting parameters on %d spots"%len(excursion_rad) n_bins = min(max(3, len(excursion_rad)//25),50) bin_sz = len(excursion_rad)//n_bins print "nbins",n_bins,"bin_sz",bin_sz order = flex.sort_permutation(two_thetas) two_thetas_env = flex.double() dspacings_env = flex.double() excursion_rads_env = flex.double() for x in xrange(0,n_bins): subset = order[x*bin_sz:(x+1)*bin_sz] two_thetas_env.append( flex.mean(two_thetas.select(subset)) ) dspacings_env.append( flex.mean(dspacings.select(subset))) excursion_rads_env.append( flex.max( flex.abs( excursion_rad.select(subset)))) # Second -- parameter fit ## solve the normal equations sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env) sum_inv_u = flex.sum(dspacings_env) sum_te_u = flex.sum(dspacings_env * excursion_rads_env) sum_te = flex.sum(excursion_rads_env) Normal_Mat = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env))) Vector = col((sum_te_u, sum_te)) solution = Normal_Mat.inverse() * Vector s_ang = 1./(2*solution[0]) print "Best LSQ fit Scheerer domain size is %9.2f ang"%( s_ang) tan_phi_rad = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. * s_ang) tan_phi_deg = tan_phi_rad * 180./math.pi k_degrees = solution[1]* 180./math.pi print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees) tan_outer_deg = tan_phi_deg + k_degrees if OO.mosaic_refinement_target=="ML": from xfel.mono_simulation.max_like import minimizer print "input", s_ang,2. * solution[1]*180/math.pi # coerce the estimates to be positive for max-likelihood lower_limit_domain_size = math.pow( helper.last_set_orientation.unit_cell().volume(), 1./3.)*20 # 10-unit cell block size minimum reasonable domain d_estimate = max(s_ang, lower_limit_domain_size) M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]), Deff = d_estimate) print "output",1./M.x[0], M.x[1]*180./math.pi tan_phi_rad_ML = helper.last_set_orientation.unit_cell().d(OO.reserve_indices) / (2. / M.x[0]) tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi # bugfix: Need factor of 0.5 because the plot shows half mosaicity (displacement from the center point defined as zero) tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi if OO.parent.horizons_phil.integration.mosaic.enable_polychromatic: # add code here to perform polychromatic modeling. """ get miller indices DONE get model-predicted mono-wavelength centroid S1 vectors back-convert S1vec, with mono-wavelength, to detector-plane position, factoring in subpixel correction compare with spot centroid measured position compare with locus of bodypixels """ print list(OO.reserve_indices) print len(OO.reserve_indices), len(two_thetas) positions = [ OO.ucbp3.simple_forward_calculation_spot_position( wavelength = OO.central_wavelength_ang, observation_no = obsno).position for obsno in xrange(len(OO.parent.indexed_pairs))] print len(positions) print positions # model-predicted positions print len(OO.parent.spots) print OO.parent.indexed_pairs print OO.parent.spots print len(OO.parent.spots) meas_spots = [OO.parent.spots[pair["spot"]] for pair in OO.parent.indexed_pairs] # for xspot in meas_spots: # xspot.ctr_mass_x(),xspot.ctr_mass_y() # xspot.max_pxl_x() # xspot.bodypixels # xspot.ctr_mass_x() # Do some work to calculate an rmsd diff_vecs = flex.vec3_double() for p,xspot in zip(positions, meas_spots): diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0)) # could use diff_vecs.rms_length() diff_vecs_sq = diff_vecs.dot(diff_vecs) mean_diff_vec_sq = flex.mean(diff_vecs_sq) rmsd = math.sqrt(mean_diff_vec_sq) print "mean obs-pred diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd) positions_to_fictitious = [ OO.ucbp3.simple_forward_calculation_spot_position( wavelength = OO.central_wavelength_ang, observation_no = obsno).position_to_fictitious for obsno in xrange(len(OO.parent.indexed_pairs))] # Do some work to calculate an rmsd diff_vecs = flex.vec3_double() for p,xspot in zip(positions_to_fictitious, meas_spots): diff_vecs.append((p[0]-xspot.ctr_mass_y(), p[1]-xspot.ctr_mass_x(), 0.0)) rmsd = diff_vecs.rms_length() print "mean obs-pred_to_fictitious diff vec on %d spots is %6.2f pixels"%(len(positions),rmsd) """ actually, it might be better if the entire set of experimental observations is transformed into the ideal detector plane, for the purposes of poly_treatment. start here. Now it would be good to actually implement probability of observing a body pixel given the model. We have everything needed right here. """ if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Image plot: obs and predicted positions + bodypixels from matplotlib import pyplot as plt plt.plot( [p[0] for p in positions_to_fictitious], [p[1] for p in positions_to_fictitious], "r.") plt.plot( [xspot.ctr_mass_y() for xspot in meas_spots], [xspot.ctr_mass_x() for xspot in meas_spots], "g.") bodypx = [] for xspot in meas_spots: for body in xspot.bodypixels: bodypx.append(body) plt.plot( [b.y for b in bodypx], [b.x for b in bodypx], "b.") plt.axes().set_aspect("equal") plt.show() print "MEAN excursion",flex.mean(final), if OO.mosaic_refinement_target=="ML": print "mosaicity deg FW=",M.x[1]*180./math.pi else: print if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Excursion vs resolution fit AD1TF7B_MAX2T = 30. AD1TF7B_MAXDP = 1. from matplotlib import pyplot as plt fig = plt.figure() plt.plot(two_thetas, final, "bo") mean = flex.mean(final) minplot = flex.min(two_thetas) plt.plot([0,minplot],[mean,mean],"k-") LR = flex.linear_regression(two_thetas, final) #LR.show_summary() model_y = LR.slope()*two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") print helper.last_set_orientation.unit_cell() #for sdp,tw in zip (dspacings,two_thetas): #print sdp,tw if OO.mosaic_refinement_target=="ML": plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas))) plt.plot(two_thetas, tan_phi_deg_ML, "r.") plt.plot(two_thetas, -tan_phi_deg_ML, "r.") plt.plot(two_thetas, tan_outer_deg_ML, "g.") plt.plot(two_thetas, -tan_outer_deg_ML, "g.") else: plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r|") plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r|") plt.plot(two_thetas_env, excursion_rads_env *180./math.pi, "r-") plt.plot(two_thetas_env, -excursion_rads_env *180./math.pi, "r-") plt.plot(two_thetas, tan_phi_deg, "r.") plt.plot(two_thetas, -tan_phi_deg, "r.") plt.plot(two_thetas, tan_outer_deg, "g.") plt.plot(two_thetas, -tan_outer_deg, "g.") plt.xlim([0,AD1TF7B_MAX2T]) plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP]) OO.parent.show_figure(plt,fig,"psi") plt.close() from xfel.mono_simulation.util import green_curve_area,ewald_proximal_volume if OO.mosaic_refinement_target=="ML": OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML) OO.parent.inputai.setMosaicity(M.x[1]*180./math.pi) # full width, degrees OO.parent.ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi) OO.parent.ML_domain_size_ang = 1./M.x[0] OO.parent.ewald_proximal_volume = ewald_proximal_volume( wavelength_ang = OO.central_wavelength_ang, resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff, domain_size_ang = 1./M.x[0], full_mosaicity_rad = M.x[1]) return results, helper.last_set_orientation,1./M.x[0] # full width domain size, angstroms else: assert OO.mosaic_refinement_target=="LSQ" OO.parent.green_curve_area = green_curve_area(two_thetas, tan_outer_deg) OO.parent.inputai.setMosaicity(2*k_degrees) # full width OO.parent.ML_half_mosaicity_deg = k_degrees OO.parent.ML_domain_size_ang = s_ang OO.parent.ewald_proximal_volume = ewald_proximal_volume( wavelength_ang = OO.central_wavelength_ang, resolution_cutoff_ang = OO.parent.horizons_phil.integration.mosaic.ewald_proximal_volume_resolution_cutoff, domain_size_ang = s_ang, full_mosaicity_rad = 2*k_degrees*math.pi/180.) return results, helper.last_set_orientation,s_ang # full width domain size, angstroms
def refine_rotx_roty2(OO, enable_rotational_target=True): helper = OO.per_frame_helper_factory() helper.restart() if enable_rotational_target: print "Trying least squares minimization of excursions", from scitbx.lstbx import normal_eqns_solving iterations = normal_eqns_solving.naive_iterations( non_linear_ls=helper, gradient_threshold=1.E-10) results = helper.x print "with %d reflections" % len(OO.parent.indexed_pairs), print "result %6.2f degrees" % (results[1] * 180. / math.pi), print "result %6.2f degrees" % (results[0] * 180. / math.pi) if False: # Excursion histogram print "The input mosaicity is %7.3f deg full width" % OO.parent.inputai.getMosaicity( ) # final histogram if OO.pvr_fix: final = 360. * helper.fvec_callable_pvr(results) else: final = 360. * helper.fvec_callable_NOT_USED_AFTER_BUGFIX( results) rmsdexc = math.sqrt(flex.mean(final * final)) from matplotlib import pyplot as plt nbins = len(final) // 20 n, bins, patches = plt.hist(final, nbins, normed=0, facecolor="orange", alpha=0.75) plt.xlabel("Rotation on e1 axis, rmsd %7.3f deg" % rmsdexc) plt.title("Histogram of cctbx.xfel misorientation") plt.axis([-0.5, 0.5, 0, 100]) plt.plot([rmsdexc], [18], "b|") plt.show() # Determine optimal mosaicity and domain size model (monochromatic) if OO.pvr_fix: final = 360. * helper.fvec_callable_pvr(results) else: final = 360. * helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results) #Guard against misindexing -- seen in simulated data, with zone nearly perfectly aligned guard_stats = flex.max(final), flex.min(final) if False and REMOVETEST_KILLING_LEGITIMATE_EXCURSIONS( guard_stats[0] > 2.0 or guard_stats[1] < -2.0): raise Exception( "Misindexing diagnosed by meaningless excursion angle (bandpass_gaussian model)" ) print "The mean excursion is %7.3f degrees" % (flex.mean(final)) two_thetas = helper.last_set_orientation.unit_cell().two_theta( OO.reserve_indices, OO.central_wavelength_ang, deg=True) dspacings = helper.last_set_orientation.unit_cell().d( OO.reserve_indices) dspace_sq = dspacings * dspacings excursion_rad = final * math.pi / 180. # First -- try to get a reasonable envelope for the observed excursions. ## minimum of three regions; maximum of 50 measurements in each bin print "fitting parameters on %d spots" % len(excursion_rad) n_bins = min(max(3, len(excursion_rad) // 25), 50) bin_sz = len(excursion_rad) // n_bins print "nbins", n_bins, "bin_sz", bin_sz order = flex.sort_permutation(two_thetas) two_thetas_env = flex.double() dspacings_env = flex.double() excursion_rads_env = flex.double() for x in range(0, n_bins): subset = order[x * bin_sz:(x + 1) * bin_sz] two_thetas_env.append(flex.mean(two_thetas.select(subset))) dspacings_env.append(flex.mean(dspacings.select(subset))) excursion_rads_env.append( flex.max(flex.abs(excursion_rad.select(subset)))) # Second -- parameter fit ## solve the normal equations sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env) sum_inv_u = flex.sum(dspacings_env) sum_te_u = flex.sum(dspacings_env * excursion_rads_env) sum_te = flex.sum(excursion_rads_env) Normal_Mat = sqr( (sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env))) Vector = col((sum_te_u, sum_te)) solution = Normal_Mat.inverse() * Vector s_ang = 1. / (2 * solution[0]) print "Best LSQ fit Scheerer domain size is %9.2f ang" % (s_ang) tan_phi_rad = helper.last_set_orientation.unit_cell().d( OO.reserve_indices) / (2. * s_ang) tan_phi_deg = tan_phi_rad * 180. / math.pi k_degrees = solution[1] * 180. / math.pi print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f" % ( 2 * k_degrees, k_degrees) tan_outer_deg = tan_phi_deg + k_degrees if OO.mosaic_refinement_target == "ML": from xfel.mono_simulation.max_like import minimizer print "input", s_ang, 2. * solution[1] * 180 / math.pi # coerce the estimates to be positive for max-likelihood lower_limit_domain_size = math.pow( helper.last_set_orientation.unit_cell().volume(), 1. / 3.) * 20 # 10-unit cell block size minimum reasonable domain d_estimate = max(s_ang, lower_limit_domain_size) M = minimizer(d_i=dspacings, psi_i=excursion_rad, eta_rad=abs(2. * solution[1]), Deff=d_estimate) print "output", 1. / M.x[0], M.x[1] * 180. / math.pi tan_phi_rad_ML = helper.last_set_orientation.unit_cell().d( OO.reserve_indices) / (2. / M.x[0]) tan_phi_deg_ML = tan_phi_rad_ML * 180. / math.pi # bugfix: Need factor of 0.5 because the plot shows half mosaicity (displacement from the center point defined as zero) tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180. / math.pi if OO.parent.horizons_phil.integration.mosaic.enable_polychromatic: # add code here to perform polychromatic modeling. """ get miller indices DONE get model-predicted mono-wavelength centroid S1 vectors back-convert S1vec, with mono-wavelength, to detector-plane position, factoring in subpixel correction compare with spot centroid measured position compare with locus of bodypixels """ print list(OO.reserve_indices) print len(OO.reserve_indices), len(two_thetas) positions = [ OO.ucbp3.simple_forward_calculation_spot_position( wavelength=OO.central_wavelength_ang, observation_no=obsno).position for obsno in range(len(OO.parent.indexed_pairs)) ] print len(positions) print positions # model-predicted positions print len(OO.parent.spots) print OO.parent.indexed_pairs print OO.parent.spots print len(OO.parent.spots) meas_spots = [ OO.parent.spots[pair["spot"]] for pair in OO.parent.indexed_pairs ] # for xspot in meas_spots: # xspot.ctr_mass_x(),xspot.ctr_mass_y() # xspot.max_pxl_x() # xspot.bodypixels # xspot.ctr_mass_x() # Do some work to calculate an rmsd diff_vecs = flex.vec3_double() for p, xspot in zip(positions, meas_spots): diff_vecs.append((p[0] - xspot.ctr_mass_y(), p[1] - xspot.ctr_mass_x(), 0.0)) # could use diff_vecs.rms_length() diff_vecs_sq = diff_vecs.dot(diff_vecs) mean_diff_vec_sq = flex.mean(diff_vecs_sq) rmsd = math.sqrt(mean_diff_vec_sq) print "mean obs-pred diff vec on %d spots is %6.2f pixels" % ( len(positions), rmsd) positions_to_fictitious = [ OO.ucbp3.simple_forward_calculation_spot_position( wavelength=OO.central_wavelength_ang, observation_no=obsno).position_to_fictitious for obsno in range(len(OO.parent.indexed_pairs)) ] # Do some work to calculate an rmsd diff_vecs = flex.vec3_double() for p, xspot in zip(positions_to_fictitious, meas_spots): diff_vecs.append((p[0] - xspot.ctr_mass_y(), p[1] - xspot.ctr_mass_x(), 0.0)) rmsd = diff_vecs.rms_length() print "mean obs-pred_to_fictitious diff vec on %d spots is %6.2f pixels" % ( len(positions), rmsd) """ actually, it might be better if the entire set of experimental observations is transformed into the ideal detector plane, for the purposes of poly_treatment. start here. Now it would be good to actually implement probability of observing a body pixel given the model. We have everything needed right here. """ if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Image plot: obs and predicted positions + bodypixels from matplotlib import pyplot as plt plt.plot([p[0] for p in positions_to_fictitious], [p[1] for p in positions_to_fictitious], "r.") plt.plot([xspot.ctr_mass_y() for xspot in meas_spots], [xspot.ctr_mass_x() for xspot in meas_spots], "g.") bodypx = [] for xspot in meas_spots: for body in xspot.bodypixels: bodypx.append(body) plt.plot([b.y for b in bodypx], [b.x for b in bodypx], "b.") plt.axes().set_aspect("equal") plt.show() print "MEAN excursion", flex.mean(final), if OO.mosaic_refinement_target == "ML": print "mosaicity deg FW=", M.x[1] * 180. / math.pi else: print if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Excursion vs resolution fit AD1TF7B_MAX2T = 30. AD1TF7B_MAXDP = 1. from matplotlib import pyplot as plt fig = plt.figure() plt.plot(two_thetas, final, "bo") mean = flex.mean(final) minplot = flex.min(two_thetas) plt.plot([0, minplot], [mean, mean], "k-") LR = flex.linear_regression(two_thetas, final) #LR.show_summary() model_y = LR.slope() * two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") print helper.last_set_orientation.unit_cell() #for sdp,tw in zip (dspacings,two_thetas): #print sdp,tw if OO.mosaic_refinement_target == "ML": plt.title( "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" % (M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas))) plt.plot(two_thetas, tan_phi_deg_ML, "r.") plt.plot(two_thetas, -tan_phi_deg_ML, "r.") plt.plot(two_thetas, tan_outer_deg_ML, "g.") plt.plot(two_thetas, -tan_outer_deg_ML, "g.") else: plt.plot(two_thetas_env, excursion_rads_env * 180. / math.pi, "r|") plt.plot(two_thetas_env, -excursion_rads_env * 180. / math.pi, "r|") plt.plot(two_thetas_env, excursion_rads_env * 180. / math.pi, "r-") plt.plot(two_thetas_env, -excursion_rads_env * 180. / math.pi, "r-") plt.plot(two_thetas, tan_phi_deg, "r.") plt.plot(two_thetas, -tan_phi_deg, "r.") plt.plot(two_thetas, tan_outer_deg, "g.") plt.plot(two_thetas, -tan_outer_deg, "g.") plt.xlim([0, AD1TF7B_MAX2T]) plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP]) OO.parent.show_figure(plt, fig, "psi") plt.close() from xfel.mono_simulation.util import green_curve_area, ewald_proximal_volume if OO.mosaic_refinement_target == "ML": OO.parent.green_curve_area = green_curve_area( two_thetas, tan_outer_deg_ML) OO.parent.inputai.setMosaicity(M.x[1] * 180. / math.pi) # full width, degrees OO.parent.ML_half_mosaicity_deg = M.x[1] * 180. / (2. * math.pi) OO.parent.ML_domain_size_ang = 1. / M.x[0] OO.parent.ewald_proximal_volume = ewald_proximal_volume( wavelength_ang=OO.central_wavelength_ang, resolution_cutoff_ang=OO.parent.horizons_phil.integration. mosaic.ewald_proximal_volume_resolution_cutoff, domain_size_ang=1. / M.x[0], full_mosaicity_rad=M.x[1]) return results, helper.last_set_orientation, 1. / M.x[ 0] # full width domain size, angstroms else: assert OO.mosaic_refinement_target == "LSQ" OO.parent.green_curve_area = green_curve_area( two_thetas, tan_outer_deg) OO.parent.inputai.setMosaicity(2 * k_degrees) # full width OO.parent.ML_half_mosaicity_deg = k_degrees OO.parent.ML_domain_size_ang = s_ang OO.parent.ewald_proximal_volume = ewald_proximal_volume( wavelength_ang=OO.central_wavelength_ang, resolution_cutoff_ang=OO.parent.horizons_phil.integration. mosaic.ewald_proximal_volume_resolution_cutoff, domain_size_ang=s_ang, full_mosaicity_rad=2 * k_degrees * math.pi / 180.) return results, helper.last_set_orientation, s_ang # full width domain size, angstroms
def extract_mosaic_parameters_using_lambda_spread(experiment, refls, verbose=True, as_eV=True, Deff_init=3000, eta_init=0.01): """ Estimates a wavelength spread from a set of indexed Bragg spots Recomputes delta_psi, Deffective (mosaic domain size), and eta (mosaic spread) using the estimated wavelength spread following Acta Cryst. (2014). D70, 3299–3309 :param experiment: dxtbx.model.Experiment object :param refls: dials.array_family.flex reflection_table object :param verbose: bool, verbosity flag :param as_eV: bool, if True, retrun all_spot_wavelen in electron volts, else leave as Angstroms :param Deff_init: float, initial guess value for Deffective (mosaic domain size in Angstroms) :param eta_init: float, initial guess value for eta (mosaic spread in degrees) :return: tuple all_spot_wavelen, delpsi_new, Deff_opt, eta_opt all_spot_wavelen: list of floats, wavelengths estimated from spot predictor delpsi_new: list of floats, new value for delta psi, computed using per-spot wavelength estimates Deff_opt: float, mosaic domain size (in Angstroms), optimized using protocol outlined in above reference eta_opt: float, mosaic spread (in degrees), optimized along with Deff_opt """ crystal = experiment.crystal beam = experiment.beam det = experiment.detector if crystal is None: print("Crystal required in expeirment") return if beam is None: print("beam required in expeirment") return if det is None: print("detector required in expeirment") return Amat = crystal.get_A() wave = beam.get_wavelength() s0_nominal = col(beam.get_s0()) required_cols = 'panel', 'xyzobs.mm.value', 'xyzcal.mm', 'rlp', 'miller_index' nmisses = 0 for column in required_cols: if column not in refls: print("Column %s missing from refl table" % column) nmisses += 1 if nmisses > 0: print( "Please format refl table to include above missing columns. Exiting." ) return all_spot_wavelen = [] all_new_delpsi = [] d_i = [] # resolution twotheta = [] try: delpsi_table_vals = refls["delpsical.rad"] except KeyError: delpsi_table_vals = [None] * len(refls) for i_ref in range(len(refls)): delpsi_table = delpsi_table_vals[i_ref] pid = refls[i_ref]['panel'] panel = det[pid] # is this necessary ? #panel.set_px_mm_strategy(SimplePxMmStrategy()) #panel.set_mu(0) #panel.set_thickness(0) # Extract an estimated wavelength for this spot res = 1 / np.linalg.norm(refls[i_ref]['rlp']) d_i.append(res) cent = panel.get_beam_centre_lab(beam.get_unit_s0()) x, y, _ = refls[i_ref]['xyzobs.mm.value'] xcal, ycal, _ = refls[i_ref]['xyzcal.mm'] xyz = panel.get_lab_coord((x, y)) rad_dist = np.sqrt(np.sum((np.array(xyz) - np.array(cent))**2)) det_dist = panel.get_distance() theta = 0.5 * np.arctan(rad_dist / det_dist) twotheta.append(2 * theta) spot_wavelen = 2 * res * np.sin(theta) # extracted per-spot wavelength all_spot_wavelen.append(spot_wavelen) # get the rlp hkl = col(refls[i_ref]['miller_index']) qhkl = sqr(Amat) * hkl qnorm2 = qhkl.length()**2 q0 = qhkl.normalize() # get the s1 vector with nominal shot wavelength xyzcal = col(panel.get_lab_coord((xcal, ycal))) s1_nominal = xyzcal.normalize() / wave # r-vector (Q) with shot wavelength qcal = s1_nominal - s0_nominal # s0 vector with new wavelength s0_recalc = s0_nominal.normalize() / spot_wavelen # this is just a shortcut to repredict the spot position with new wavelength, and then compute the new delta psi e1 = (q0.cross(s0_recalc)).normalize() c0 = (s0_recalc.cross(e1)).normalize() q1 = q0.cross(e1) a = qnorm2 * spot_wavelen / 2. b = np.sqrt(qnorm2 - a * a) qcal_recalc = -a * (s0_nominal.normalize()) + b * c0 delpsi_nominal = -np.arctan2(qcal.dot(q1), qcal.dot(q0)) delpsi_new = -np.arctan2(qcal_recalc.dot(q1), qcal_recalc.dot(q0)) all_new_delpsi.append(delpsi_new) if verbose: print("\n<><><><><><><><><><><><><><><><><><><>") print("Delta psi spot %d/%d" % (i_ref + 1, len(refls))) print("<><><><><><><><><><><><><><><><><><><>") print("nominal wavelen: %.6f" % wave) print("per-spot wavelen: %.6f" % spot_wavelen) if delpsi_table is not None: print("Value in table: %.6f " % delpsi_table) print("recompute table value (sanity test): %.6f" % delpsi_nominal) print("compute updated value with per-spot wavelen: %.6f" % delpsi_new) delpsi_vals = flex.double(all_new_delpsi) out = max_like.minimizer(flex.double(d_i), delpsi_vals, eta_init * np.pi / 180., Deff_init) Deff_opt = 2 / out.x[0] eta_opt = out.x[1] * 180 / np.pi all_spot_wavelen = flex.double(all_spot_wavelen) all_spot_energies = flex.double( [ENERGY_CONV / wave for wave in all_spot_wavelen]) mean_en = np.mean(all_spot_energies) sig_en = np.std(all_spot_energies) if verbose: print("\n<><><><>") print("Results:") print("<><><><>") print("Deff with per-spot wavelen: %.6f Angstrom." % Deff_opt) print("eta with per-spot wavelen: %.6f deg." % eta_opt) print("Per-spot wavelengths: %.4f +- %.4f eV. \n" % (mean_en, sig_en)) poly_data = all_spot_wavelen if as_eV: poly_data = all_spot_energies return poly_data, delpsi_vals, Deff_opt, eta_opt
def __call__(self): """Determine optimal mosaicity and domain size model (monochromatic)""" RR = self.refinery.predict_for_reflection_table(self.reflections) excursion_rad = RR["delpsical.rad"] delta_psi_deg = excursion_rad * 180./math.pi print print flex.max(delta_psi_deg), flex.min(delta_psi_deg) mean_excursion = flex.mean(delta_psi_deg) print "The mean excursion is %7.3f degrees, r.m.s.d %7.3f"%(mean_excursion, math.sqrt(flex.mean(RR["delpsical2"]))) crystal = self.experiments[0].crystal beam = self.experiments[0].beam miller_indices = self.reflections["miller_index"] # FIXME XXX revise this formula so as to use a different wavelength potentially for each reflection two_thetas = crystal.get_unit_cell().two_theta(miller_indices,beam.get_wavelength(),deg=True) dspacings = crystal.get_unit_cell().d(miller_indices) dspace_sq = dspacings * dspacings # First -- try to get a reasonable envelope for the observed excursions. ## minimum of three regions; maximum of 50 measurements in each bin print "fitting parameters on %d spots"%len(excursion_rad) n_bins = min(max(3, len(excursion_rad)//25),50) bin_sz = len(excursion_rad)//n_bins print "nbins",n_bins,"bin_sz",bin_sz order = flex.sort_permutation(two_thetas) two_thetas_env = flex.double() dspacings_env = flex.double() excursion_rads_env = flex.double() for x in xrange(0,n_bins): subset = order[x*bin_sz:(x+1)*bin_sz] two_thetas_env.append(flex.mean(two_thetas.select(subset))) dspacings_env.append(flex.mean(dspacings.select(subset))) excursion_rads_env.append(flex.max(flex.abs(excursion_rad.select(subset)))) # Second -- parameter fit ## solve the normal equations sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env) sum_inv_u = flex.sum(dspacings_env) sum_te_u = flex.sum(dspacings_env * excursion_rads_env) sum_te = flex.sum(excursion_rads_env) Normal_Mat = sqr((sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env))) Vector = col((sum_te_u, sum_te)) solution = Normal_Mat.inverse() * Vector s_ang = 1./(2*solution[0]) print "Best LSQ fit Scheerer domain size is %9.2f ang"%( s_ang) tan_phi_rad = dspacings / (2. * s_ang) tan_phi_deg = tan_phi_rad * 180./math.pi k_degrees = solution[1]* 180./math.pi print "The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f"%(2*k_degrees, k_degrees) tan_outer_deg = tan_phi_deg + k_degrees from xfel.mono_simulation.max_like import minimizer # coerce the estimates to be positive for max-likelihood lower_limit_domain_size = math.pow(crystal.get_unit_cell().volume(), 1./3.)*3 # params.refinement.domain_size_lower_limit d_estimate = max(s_ang, lower_limit_domain_size) M = minimizer(d_i = dspacings, psi_i = excursion_rad, eta_rad = abs(2. * solution[1]), Deff = d_estimate) print "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas)) tan_phi_rad_ML = dspacings / (2. / M.x[0]) tan_phi_deg_ML = tan_phi_rad_ML * 180./math.pi tan_outer_deg_ML = tan_phi_deg_ML + 0.5*M.x[1]*180./math.pi self.nv_acceptance_flags = flex.abs(delta_psi_deg) < tan_outer_deg_ML if self.graph_verbose: #params.refinement.mosaic.enable_AD14F7B: # Excursion vs resolution fit AD1TF7B_MAX2T = 30. AD1TF7B_MAXDP = 1. from matplotlib import pyplot as plt plt.plot(two_thetas, delta_psi_deg, "bo") minplot = flex.min(two_thetas) plt.plot([0,minplot],[mean_excursion,mean_excursion],"k-") LR = flex.linear_regression(two_thetas, delta_psi_deg) model_y = LR.slope()*two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") plt.title("ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots"%(M.x[1]*180./math.pi, 2./M.x[0], len(two_thetas))) plt.plot(two_thetas, tan_phi_deg_ML, "r.") plt.plot(two_thetas, -tan_phi_deg_ML, "r.") plt.plot(two_thetas, tan_outer_deg_ML, "g.") plt.plot(two_thetas, -tan_outer_deg_ML, "g.") plt.xlim([0,AD1TF7B_MAX2T]) plt.ylim([-AD1TF7B_MAXDP,AD1TF7B_MAXDP]) plt.show() plt.close() from xfel.mono_simulation.util import green_curve_area self.green_curve_area = green_curve_area(two_thetas, tan_outer_deg_ML) print "The green curve area is ", self.green_curve_area crystal._ML_half_mosaicity_deg = M.x[1]*180./(2.*math.pi) crystal._ML_domain_size_ang = 2./M.x[0] self._ML_full_mosaicity_rad = M.x[1] self._ML_domain_size_ang = 2./M.x[0] #params.refinement.mosaic.model_expansion_factor """The expansion factor should be initially set to 1, then expanded so that the # reflections matched becomes as close as possible to # of observed reflections input, in the last integration call. Determine this by inspecting the output log file interactively. Do not exceed the bare minimum threshold needed. The intention is to find an optimal value, global for a given dataset.""" model_expansion_factor = 1.4 crystal._ML_half_mosaicity_deg *= model_expansion_factor crystal._ML_domain_size_ang /= model_expansion_factor return crystal