def integration_concept(self, image_number=0, cb_op_to_primitive=None, verbose=False, **kwargs): self.image_number = image_number NEAR = 10 pxlsz = self.pixel_size self.get_predictions_accounting_for_centering(cb_op_to_primitive, **kwargs) FWMOSAICITY = self.inputai.getMosaicity() DOMAIN_SZ_ANG = kwargs.get("domain_size_ang", self.__dict__.get("actual", 0)) refineflag = {True: 0, False: 1}[kwargs.get("domain_size_ang", 0) == 0] self.inputpd["symmetry"].show_summary( prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f " % (refineflag, FWMOSAICITY, DOMAIN_SZ_ANG)) from annlib_ext import AnnAdaptor self.cell = self.inputai.getOrientation().unit_cell() query = flex.double() for pred in self.predicted: # predicted spot coord in pixels query.append(pred[0] / pxlsz) query.append(pred[1] / pxlsz) self.reserve_hkllist_for_signal_search = self.hkllist reference = flex.double() spots = self.get_observations_with_outlier_removal() assert len( spots) > NEAR # Can't do spot/pred matching with too few spots for spot in spots: reference.append(spot.ctr_mass_x()) reference.append(spot.ctr_mass_y()) IS_adapt = AnnAdaptor(data=reference, dim=2, k=NEAR) IS_adapt.query(query) print "Calculate correction vectors for %d observations & %d predictions" % ( len(spots), len(self.predicted)) indexed_pairs_provisional = [] correction_vectors_provisional = [] c_v_p_flex = flex.vec3_double() idx_cutoff = float(min(self.mask_focus[image_number])) if verbose: print "idx_cutoff distance in pixels", idx_cutoff if not self.horizons_phil.integration.enable_one_to_one_safeguard: # legacy code, no safeguard against many-to-one predicted-to-observation mapping for i in range(len(self.predicted)): # loop over predicteds #for n in range(NEAR): # loop over near spotfinder spots for n in range(1): # only consider the nearest spotfinder spot Match = dict(spot=IS_adapt.nn[i * NEAR + n], pred=i) if n == 0 and math.sqrt( IS_adapt.distances[i * NEAR + n]) < idx_cutoff: indexed_pairs_provisional.append(Match) vector = matrix.col([ spots[Match["spot"]].ctr_mass_x() - self.predicted[Match["pred"]][0] / pxlsz, spots[Match["spot"]].ctr_mass_y() - self.predicted[Match["pred"]][1] / pxlsz ]) correction_vectors_provisional.append(vector) c_v_p_flex.append((vector[0], vector[1], 0.)) else: one_to_one = {} for i in range(len(self.predicted)): # loop over predicteds annresultidx = i * NEAR obsidx = IS_adapt.nn[annresultidx] this_distancesq = IS_adapt.distances[annresultidx] if obsidx not in one_to_one or \ this_distancesq < one_to_one[obsidx]["distancesq"]: if math.sqrt(this_distancesq) < idx_cutoff: one_to_one[obsidx] = dict(spot=obsidx, pred=i, distancesq=this_distancesq) for key, value in one_to_one.items(): indexed_pairs_provisional.append(value) vector = matrix.col([ spots[value["spot"]].ctr_mass_x() - self.predicted[value["pred"]][0] / pxlsz, spots[value["spot"]].ctr_mass_y() - self.predicted[value["pred"]][1] / pxlsz ]) correction_vectors_provisional.append(vector) c_v_p_flex.append((vector[0], vector[1], 0.)) print "... %d provisional matches" % len( correction_vectors_provisional), print "r.m.s.d. in pixels: %5.2f" % (math.sqrt( flex.mean(c_v_p_flex.dot(c_v_p_flex)))) if self.horizons_phil.integration.enable_residual_scatter: from matplotlib import pyplot as plt fig = plt.figure() for cv in correction_vectors_provisional: plt.plot([cv[1]], [-cv[0]], "b.") plt.title(" %d matches, r.m.s.d. %5.2f pixels" % (len(correction_vectors_provisional), math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt, fig, "res") plt.close() if self.horizons_phil.integration.enable_residual_map: from matplotlib import pyplot as plt fig = plt.figure() for match, cv in zip(indexed_pairs_provisional, correction_vectors_provisional): plt.plot([spots[match["spot"]].ctr_mass_y()], [-spots[match["spot"]].ctr_mass_x()], "r.") plt.plot([self.predicted[match["pred"]][1] / pxlsz], [-self.predicted[match["pred"]][0] / pxlsz], "g.") plt.plot([ spots[match["spot"]].ctr_mass_y(), spots[match["spot"]].ctr_mass_y() + 10. * cv[1] ], [ -spots[match["spot"]].ctr_mass_x(), -spots[match["spot"]].ctr_mass_x() - 10. * cv[0] ], 'b-') plt.xlim([0, float(self.inputpd["size2"])]) plt.ylim([-float(self.inputpd["size1"]), 0]) plt.title(" %d matches, r.m.s.d. %5.2f pixels" % (len(correction_vectors_provisional), math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt, fig, "map") plt.close() # insert code here to remove correction length outliers... # they are causing terrible # problems for finding legitimate correction vectors (print out the list) # also remove outliers for the purpose of reporting RMS outlier_rejection = True cache_refinement_spots = getattr(slip_callbacks.slip_callback, "requires_refinement_spots", False) if outlier_rejection: correction_lengths = flex.double( [v.length() for v in correction_vectors_provisional]) clorder = flex.sort_permutation(correction_lengths) sorted_cl = correction_lengths.select(clorder) ACCEPTABLE_LIMIT = 2 limit = int( 0.33 * len(sorted_cl) ) # best 1/3 of data are assumed to be correctly modeled. if (limit <= ACCEPTABLE_LIMIT): raise Sorry( "Not enough indexed spots to reject outliers; have %d need >%d" % (limit, ACCEPTABLE_LIMIT)) y_data = flex.double(len(sorted_cl)) for i in range(len(y_data)): y_data[i] = float(i) / float(len(y_data)) # ideas are explained in Sauter & Poon (2010) J Appl Cryst 43, 611-616. from rstbx.outlier_spots.fit_distribution import fit_cdf, rayleigh fitted_rayleigh = fit_cdf(x_data=sorted_cl[0:limit], y_data=y_data[0:limit], distribution=rayleigh) inv_cdf = [ fitted_rayleigh.distribution.inv_cdf(cdf) for cdf in y_data ] #print "SORTED LIST OF ",len(sorted_cl), "with sigma",fitted_rayleigh.distribution.sigma indexed_pairs = [] correction_vectors = [] self.correction_vectors = [] for icand in range(len(sorted_cl)): # somewhat arbitrary sigma = 1.0 cutoff for outliers if (sorted_cl[icand] - inv_cdf[icand] ) / fitted_rayleigh.distribution.sigma > 1.0: break indexed_pairs.append(indexed_pairs_provisional[clorder[icand]]) correction_vectors.append( correction_vectors_provisional[clorder[icand]]) if cache_refinement_spots: self.spotfinder.images[self.frame_numbers[ self.image_number]]["refinement_spots"].append( spots[indexed_pairs[-1]["spot"]]) if kwargs.get("verbose_cv") == True: print "CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f" % ( float(self.inputpd["size1"]) / 2., float(self.inputpd["size2"]) / 2., self.inputai.xbeam() / pxlsz, self.inputai.ybeam() / pxlsz), print "OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f" % ( spots[indexed_pairs[-1]["spot"]].ctr_mass_x(), spots[indexed_pairs[-1]["spot"]].ctr_mass_y(), self.predicted[indexed_pairs[-1]["pred"]][0] / pxlsz, self.predicted[indexed_pairs[-1]["pred"]][1] / pxlsz), the_hkl = self.hkllist[indexed_pairs[-1]["pred"]] print "HKL %4d %4d %4d" % the_hkl, "%2d" % self.setting_id, radial, azimuthal = spots[indexed_pairs[-1][ "spot"]].get_radial_and_azimuthal_size( self.inputai.xbeam() / pxlsz, self.inputai.ybeam() / pxlsz) print "RADIALpx %5.3f AZIMUTpx %5.3f" % (radial, azimuthal) # Store a list of correction vectors in self. radial, azimuthal = spots[ indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size( self.inputai.xbeam() / pxlsz, self.inputai.ybeam() / pxlsz) self.correction_vectors.append( dict(obscenter=(float(self.inputpd['size1']) / 2, float(self.inputpd['size2']) / 2), refinedcenter=(self.inputai.xbeam() / pxlsz, self.inputai.ybeam() / pxlsz), obsspot=( spots[indexed_pairs[-1]['spot']].ctr_mass_x(), spots[indexed_pairs[-1]['spot']].ctr_mass_y()), predspot=( self.predicted[indexed_pairs[-1]['pred']][0] / pxlsz, self.predicted[indexed_pairs[-1]['pred']][1] / pxlsz), hkl=(self.hkllist[indexed_pairs[-1]['pred']][0], self.hkllist[indexed_pairs[-1]['pred']][1], self.hkllist[indexed_pairs[-1]['pred']][2]), setting_id=self.setting_id, radial=radial, azimuthal=azimuthal)) print "After outlier rejection %d indexed spotfinder spots remain." % len( indexed_pairs) if False: rayleigh_cdf = [ fitted_rayleigh.distribution.cdf(x=sorted_cl[c]) for c in range(len(sorted_cl)) ] from matplotlib import pyplot as plt plt.plot(sorted_cl, y_data, "r+") #plt.plot(sorted_cl,rayleigh_cdf,"g.") plt.plot(inv_cdf, y_data, "b.") plt.show() else: indexed_pairs = indexed_pairs_provisional correction_vectors = correction_vectors_provisional ########### finished with outlier rejection self.inputpd["symmetry"].show_summary(prefix="SETTING ") is_triclinic = (self.setting_id == 1) if is_triclinic: self.triclinic_pairs = [ dict(pred=self.hkllist[a["pred"]], spot=a["spot"]) for a in indexed_pairs ] if self.horizons_phil.integration.model == "user_supplied": if kwargs.get("user-reentrant", None) == None: from cxi_user import post_outlier_rejection self.indexed_pairs = indexed_pairs self.spots = spots post_outlier_rejection(self, image_number, cb_op_to_primitive, self.horizons_phil, kwargs) return ########### finished with user-supplied code if self.horizons_phil.integration.spot_shape_verbose: from rstbx.new_horizons.spot_shape import spot_shape_verbose spot_shape_verbose(rawdata=self.imagefiles.images[ self.image_number].linearintdata, beam_center_pix=matrix.col( (self.inputai.xbeam() / pxlsz, self.inputai.ybeam() / pxlsz)), indexed_pairs=indexed_pairs, spotfinder_observations=spots, distance_mm=self.inputai.distance(), mm_per_pixel=pxlsz, hkllist=self.hkllist, unit_cell=self.cell, wavelength_ang=self.inputai.wavelength) #Other checks to be implemented (future): # spot is within active area of detector on a circular detector such as the Mar IP # integration masks do not overlap; or deconvolute correction_lengths = flex.double( [v.length() for v in correction_vectors]) if verbose: print "average correction %5.2f over %d vectors" % ( flex.mean(correction_lengths), len(correction_lengths)), print "or %5.2f mm." % (pxlsz * flex.mean(correction_lengths)) self.r_residual = pxlsz * flex.mean(correction_lengths) #assert len(indexed_pairs)>NEAR # must have enough indexed spots if (len(indexed_pairs) <= NEAR): raise Sorry("Not enough indexed spots, only found %d, need %d" % (len(indexed_pairs), NEAR)) reference = flex.double() for item in indexed_pairs: reference.append(spots[item["spot"]].ctr_mass_x()) reference.append(spots[item["spot"]].ctr_mass_y()) PS_adapt = AnnAdaptor(data=reference, dim=2, k=NEAR) PS_adapt.query(query) self.BSmasks = [] #self.null_correction_mapping( predicted=self.predicted, # correction_vectors = correction_vectors, # IS_adapt = IS_adapt, # spots = spots) self.positional_correction_mapping( predicted=self.predicted, correction_vectors=correction_vectors, PS_adapt=PS_adapt, IS_adapt=IS_adapt, spots=spots) # which spots are close enough to interfere with background? MAXOVER = 6 OS_adapt = AnnAdaptor(data=query, dim=2, k=MAXOVER) #six near nbrs OS_adapt.query(query) if self.mask_focus[image_number] is None: raise Sorry( "No observed/predicted spot agreement; no Spotfinder masks; skip integration" ) nbr_cutoff = 2.0 * max(self.mask_focus[image_number]) FRAME = int(nbr_cutoff / 2) #print "The overlap cutoff is %d pixels"%nbr_cutoff nbr_cutoff_sq = nbr_cutoff * nbr_cutoff #print "Optimized C++ section...", self.set_frame(FRAME) self.set_background_factor(kwargs["background_factor"]) self.set_nbr_cutoff_sq(nbr_cutoff_sq) self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq) self.set_detector_gain(self.horizons_phil.integration.detector_gain) flex_sorted = flex.int() for item in self.sorted: flex_sorted.append(item[0]) flex_sorted.append(item[1]) if self.horizons_phil.integration.mask_pixel_value is not None: self.set_mask_pixel_val( self.horizons_phil.integration.mask_pixel_value) image_obj = self.imagefiles.imageindex( self.frame_numbers[self.image_number]) image_obj.read() rawdata = image_obj.linearintdata # assume image #1 if self.inputai.active_areas != None: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted, tiles=self.inputai.active_areas.IT, tile_id=self.inputai.active_areas.tile_id) else: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted) for i in range(len(self.predicted)): # loop over predicteds B_S_mask = {} keys = self.get_bsmask(i) for k in range(0, len(keys), 2): B_S_mask[(keys[k], keys[k + 1])] = True self.BSmasks.append(B_S_mask) #print "Done" return
def update_detail(self, horizon_phil, current_status, first_time_through, verbose): assert (len(self.observed_spots) == len(self.predicted_spots)) if horizon_phil.indexing.outlier_detection.verbose: classes = [ str(current_status[i]) for i in range(len(self.observed_spots)) ] class_types = set(classes) class_counts = dict([[item, classes.count(item)] for item in class_types]) flex_counts = flex.int(class_counts.values()) assert flex.sum(flex_counts) == len(self.observed_spots) #for pair in class_counts.items(): # print "%10s %6d"%pair #print "%10s %6d"%("TOTAL",len(self.observed_spots)) if status_with_marked_outliers == None: # status_with_marked_outliers==None is shorthand for identifying the first run through print """After indexing on a subset of %d spots (from all images), %d were reclassified as either lying on the spindle, or potential overlapped spots or ice rings.""" % ( len(self.observed_spots), len(self.observed_spots) - class_counts["GOOD"]) else: print """Rerefinement on just the well-fit spots followed by spot reclassification leaves %d good spots on which to calculate a triclinic rmsd.""" % ( class_counts["GOOD"]) # check good spots if (self.good is not None): match = 0 for i in range(len(self.observed_spots)): if ((current_status[i] == SpotClass.GOOD) and self.good[i]): match = match + 1 if self.verbose: print "Number of GOOD spots matched with previous model =", match # calculate differences for all spots self.sorted_observed_spots = {} self.dr = flex.double() self.not_good_dr = flex.double() self.dx = [0.0 for i in range(len(self.observed_spots))] self.dy = [0.0 for i in range(len(self.observed_spots))] for i in range(len(self.observed_spots)): o = self.observed_spots[i] p = self.predicted_spots[i] self.dx[i] = o[0] - p[0] self.dy[i] = o[1] - p[1] self.sorted_observed_spots[math.sqrt(self.dx[i] * self.dx[i] + self.dy[i] * self.dy[i])] = i # separate GOOD spots spotclasses = { SpotClass.GOOD: 0, SpotClass.SPINDLE: 0, SpotClass.OVERLAP: 0, SpotClass.ICE: 0, SpotClass.OUTLIER: 0, SpotClass.NONE: 0 } for key in sorted(self.sorted_observed_spots.keys()): spotclass = current_status[self.sorted_observed_spots[key]] spotclasses[spotclass] += 1 if (current_status[self.sorted_observed_spots[key]] == SpotClass.GOOD): self.dr.append(key) else: self.not_good_dr.append(key) if verbose: print ", ".join([ "=".join([str(i[0]), "%d" % i[1]]) for i in spotclasses.items() ]), totalsp = sum( [spotclasses.values()[iidx] for iidx in range(len(spotclasses))]) if verbose: print "Total=%d" % (totalsp), "# observed spots", len( self.observed_spots) assert totalsp == len( self.observed_spots ), "Some spot pairs have the same predicted-observed distances. Do you have duplicated images?" self.x = flex.double(len(self.dr)) for i in range(len(self.x)): self.x[i] = float(i) / float(len(self.x)) limit = int(self.fraction * len(self.dr)) if limit < 4: return # Basic sanity check, need at least a few good spots to fit the distribution fitted_rayleigh = fit_cdf(x_data=self.dr[0:limit], y_data=self.x[0:limit], distribution=rayleigh_cpp) if False: y_data = self.x[0:limit] inv_cdf = [ fitted_rayleigh.distribution.inv_cdf(cdf) for cdf in y_data ] from matplotlib import pyplot as plt plt.plot(self.dr[0:limit], self.x[0:limit], "r+") plt.plot(inv_cdf, y_data, "b.") plt.show() # store indices for spots used for fitting self.fraction_spot_indices = [] for dr in self.dr[0:limit]: self.fraction_spot_indices.append(self.sorted_observed_spots[dr]) # generate points for fitted distributions rayleigh_cdf_x = flex.double(range(500)) rayleigh_cdf_x /= float(len(rayleigh_cdf_x)) rayleigh_cdf = fitted_rayleigh.distribution.cdf(x=rayleigh_cdf_x) # generate points for pdf dr_bins, dr_histogram = make_histogram_data(data=self.dr, n_bins=100) rayleigh_pdf = flex.double(len(dr_bins)) for i in range(len(dr_bins)): rayleigh_pdf[i] = fitted_rayleigh.distribution.pdf(x=dr_bins[i]) rayleigh_pdf = rayleigh_pdf / flex.sum(rayleigh_pdf) dr_bins = flex.double(dr_bins) dr_histogram = flex.double(dr_histogram) # standard deviation for cdf sd = math.sqrt((4.0 - math.pi) / (2.0) * fitted_rayleigh.x[0] * fitted_rayleigh.x[0]) if self.verbose: print 'Standard deviation of Rayleigh fit = %4.3f' % sd sd_data = None radius_outlier_index = None limit_outlier = None # --- Quoted code superceeded by extension module call to find_green_bar """ for i in range(len(rayleigh_cdf_x)): mx = rayleigh_cdf_x[i] my = rayleigh_cdf[i] for j in range(1,len(self.dr)): upper_x = self.dr[j] upper_y = self.x[j] lower_x = self.dr[j-1] lower_y = self.x[j-1] if ((my >= lower_y) and (my < upper_y)): if ((sd <= (upper_x - mx)) and ((lower_x - mx) > 0.0)): sd_data = ((mx,my),(lower_x,lower_y)) radius_outlier_index = j-1 limit_outlier = lower_x if self.verbose:print "Width of green bar = %4.3f"%(lower_x - mx) break if (sd_data is not None): break """ from rstbx.indexing_api import find_green_bar green = find_green_bar(rayleigh_cdf_x=rayleigh_cdf_x, rayleigh_cdf=rayleigh_cdf, dr=self.dr, x=self.x, sd=sd) if green.is_set: #assert radius_outlier_index == green.radius_outlier_index #assert limit_outlier == green.limit_outlier #assert sd_data[0][0] == green.sd_mx #assert sd_data[0][1] == green.sd_my #assert sd_data[1][0] == green.sd_lower_x #assert sd_data[1][1] == green.sd_lower_y if self.verbose: print "Width of green bar = %4.3f" % (green.sd_lower_x - green.sd_mx) radius_outlier_index = green.radius_outlier_index limit_outlier = green.limit_outlier sd_data = ((green.sd_mx, green.sd_my), (green.sd_lower_x, green.sd_lower_y)) if (radius_outlier_index is None): radius_outlier_index = len(self.dr) if (limit_outlier is None): limit_outlier = self.dr[-1] radius_95 = None for i in range(len(rayleigh_cdf)): if (rayleigh_cdf[i] >= 0.95): radius_95 = rayleigh_cdf_x[i] break if (radius_95 is None): radius_95 = rayleigh_cdf_x[-1] upper_circle = [] lower_circle = [] d_radius = 2.0 * radius_95 / 100.0 x = -radius_95 r2 = radius_95 * radius_95 for i in range(100): y = math.sqrt(r2 - x * x) upper_circle.append((x, y)) lower_circle.append((x, -y)) x = x + d_radius y = 0.0 upper_circle.append((x, y)) lower_circle.append((x, -y)) self.sqrtr2 = math.sqrt(r2) # color code dx dy dxdy_fraction = [] dxdy_inliers = [] dxdy_outliers = [] limit = self.dr[int(self.fraction * len(self.dr))] trifold = dict(fraction=0, inlier=0, outlier=0, total=0) for key in self.dr: trifold["total"] += 1 i = self.sorted_observed_spots[key] if (key < limit): trifold["fraction"] += 1 if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or (self.dy[i] > 1.0) or (self.dy[i] < -1.0))): dxdy_fraction.append((self.dx[i], self.dy[i])) elif (key < limit_outlier): trifold["inlier"] += 1 if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or (self.dy[i] > 1.0) or (self.dy[i] < -1.0))): dxdy_inliers.append((self.dx[i], self.dy[i])) else: trifold["outlier"] += 1 if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or (self.dy[i] > 1.0) or (self.dy[i] < -1.0))): dxdy_outliers.append((self.dx[i], self.dy[i])) if verbose: print ", ".join( ["=".join([str(i[0]), "%d" % i[1]]) for i in trifold.items()]) # color code observed fractions o_fraction = [] o_inliers = [] o_outliers = [] mr = format_data(x_data=rayleigh_cdf_x, y_data=rayleigh_cdf) limit = int(self.fraction * len(self.dr)) for i in range(len(self.dr)): if (self.dr[i] <= 1.0): if (i < limit): o_fraction.append((self.dr[i], self.x[i])) elif (i < radius_outlier_index): o_inliers.append((self.dr[i], self.x[i])) else: o_outliers.append((self.dr[i], self.x[i])) if horizon_phil.indexing.outlier_detection.verbose: o_outliers_for_severity = [] for i in range(radius_outlier_index, len(self.dr)): o_outliers_for_severity.append((self.dr[i], self.x[i])) # limit data range for i in range(len(dr_bins)): if (dr_bins[i] > 1.0): dr_bins.resize(i) dr_histogram.resize(i) rayleigh_pdf.resize(i) break ho = format_data(x_data=dr_bins, y_data=dr_histogram) hr = format_data(x_data=dr_bins, y_data=rayleigh_pdf) # format data for graphing self.plot_dxdy_data = [ dxdy_fraction, dxdy_inliers, dxdy_outliers, [(0.0, 0.0)], [], [], [], [] ] self.framework = { 4: dict(status=SpotClass.SPINDLE), 5: dict(status=SpotClass.OVERLAP), 6: dict(status=SpotClass.OUTLIER), 7: dict(status=SpotClass.ICE), } for key in self.not_good_dr: i = self.sorted_observed_spots[key] status = current_status[i] if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or (self.dy[i] > 1.0) or (self.dy[i] < -1.0))): statuskey = [ k for k in self.framework.keys() if self.framework[k]["status"] == status ][0] self.plot_dxdy_data[statuskey].append((self.dx[i], self.dy[i])) self.plot_cdf_data = [mr, o_fraction, o_inliers, o_outliers] if (sd_data is not None): self.plot_cdf_data.append(sd_data) self.plot_pdf_data = [ho, hr] # mark outliers if (first_time_through): #i.e., first time through the update() method if (radius_outlier_index < len(self.dr)): for i in range(radius_outlier_index, len(self.dr)): current_status[self.sorted_observed_spots[ self.dr[i]]] = SpotClass.OUTLIER # reset good spots self.good = [False for i in range(len(self.observed_spots))] for i in range(len(self.observed_spots)): if (current_status[i] == SpotClass.GOOD): self.good[i] = True count_outlier = 0 count_good = 0 for i in range(len(self.observed_spots)): if (current_status[i] == SpotClass.OUTLIER): count_outlier = count_outlier + 1 elif (current_status[i] == SpotClass.GOOD): count_good = count_good + 1 if self.verbose: print 'Old GOOD =', len(self.dr),\ 'OUTLIER =', count_outlier,\ 'New GOOD =', count_good if horizon_phil.indexing.outlier_detection.verbose and status_with_marked_outliers is None: print "\nOf the remaining %d spots, %.1f%% were lattice outliers, leaving %d well-fit spots" % ( len(self.dr), 100. * count_outlier / len(self.dr), count_good) if count_outlier == 0: return #width of green bar is sd delta_spread = o_outliers_for_severity[1][ 1] - o_outliers_for_severity[0][1] severity = 0. for item in o_outliers_for_severity: delta_r = item[0] # obs - predicted deviation in mm spread = item[ 1] # order of observed deviation on a scale from 0 to 1 # now invert the cdf to find expected delta r: expected_delta_r = fitted_rayleigh.distribution.sigma * math.sqrt( -2. * math.log(1. - spread)) #print item, expected_delta_r, (delta_r - expected_delta_r) / sd severity += ((delta_r - expected_delta_r) / sd) severity *= delta_spread print "The outlier severity is %.2f sigma [defined in J Appl Cryst (2010) 43, p.611 sec. 4].\n" % severity return current_status
def update_detail(self,horizon_phil,current_status,first_time_through,verbose): assert(len(self.observed_spots) == len(self.predicted_spots)) if horizon_phil.indexing.outlier_detection.verbose: classes=[str(current_status[i]) for i in xrange(len(self.observed_spots))] class_types = set(classes) class_counts = dict([[item,classes.count(item)] for item in class_types]) flex_counts = flex.int(class_counts.values()) assert flex.sum(flex_counts) == len(self.observed_spots) #for pair in class_counts.items(): # print "%10s %6d"%pair #print "%10s %6d"%("TOTAL",len(self.observed_spots)) if status_with_marked_outliers == None: # status_with_marked_outliers==None is shorthand for identifying the first run through print """After indexing on a subset of %d spots (from all images), %d were reclassified as either lying on the spindle, or potential overlapped spots or ice rings."""%( len(self.observed_spots),len(self.observed_spots)-class_counts["GOOD"]) else: print """Rerefinement on just the well-fit spots followed by spot reclassification leaves %d good spots on which to calculate a triclinic rmsd."""%(class_counts["GOOD"]) # check good spots if (self.good is not None): match = 0 for i in xrange(len(self.observed_spots)): if ((current_status[i] == SpotClass.GOOD) and self.good[i]): match = match + 1 if self.verbose:print "Number of GOOD spots matched with previous model =",match # calculate differences for all spots self.sorted_observed_spots = {} self.dr = flex.double() self.not_good_dr = flex.double() self.dx = [0.0 for i in xrange(len(self.observed_spots))] self.dy = [0.0 for i in xrange(len(self.observed_spots))] for i in xrange(len(self.observed_spots)): o = self.observed_spots[i] p = self.predicted_spots[i] self.dx[i] = o[0] - p[0] self.dy[i] = o[1] - p[1] self.sorted_observed_spots[ math.sqrt(self.dx[i]*self.dx[i] + self.dy[i]*self.dy[i])] = i # separate GOOD spots spotclasses = {SpotClass.GOOD:0,SpotClass.SPINDLE:0,SpotClass.OVERLAP:0,SpotClass.ICE:0,SpotClass.OUTLIER:0,SpotClass.NONE:0} for key in sorted(self.sorted_observed_spots.keys()): spotclass = current_status[self.sorted_observed_spots[key]] spotclasses[spotclass]+=1 if (current_status[self.sorted_observed_spots[key]] == SpotClass.GOOD): self.dr.append(key) else: self.not_good_dr.append(key) if verbose: print ", ".join(["=".join([str(i[0]),"%d"%i[1]]) for i in spotclasses.items()]), totalsp = sum([spotclasses.values()[iidx] for iidx in xrange(len(spotclasses))]) if verbose: print "Total=%d"%(totalsp),"# observed spots",len(self.observed_spots) assert totalsp == len(self.observed_spots) self.x = flex.double(len(self.dr)) for i in xrange(len(self.x)): self.x[i] = float(i)/float(len(self.x)) limit = int(self.fraction*len(self.dr)) if limit < 4: return # Basic sanity check, need at least a few good spots to fit the distribution fitted_rayleigh = fit_cdf(x_data=self.dr[0:limit], y_data=self.x[0:limit],distribution=rayleigh) if False: y_data=self.x[0:limit] inv_cdf = [fitted_rayleigh.distribution.inv_cdf(cdf) for cdf in y_data] from matplotlib import pyplot as plt plt.plot(self.dr[0:limit],self.x[0:limit],"r+") plt.plot(inv_cdf,y_data,"b.") plt.show() # store indices for spots used for fitting self.fraction_spot_indices = [] for dr in self.dr[0:limit]: self.fraction_spot_indices.append(self.sorted_observed_spots[dr]) # generate points for fitted distributions rayleigh_cdf_x = flex.double(500) for i in xrange(len(rayleigh_cdf_x)): rayleigh_cdf_x[i] = float(i)/float(len(rayleigh_cdf_x)) rayleigh_cdf = flex.double(len(rayleigh_cdf_x)) for i in xrange(len(rayleigh_cdf_x)): rayleigh_cdf[i] = fitted_rayleigh.distribution.cdf(x=rayleigh_cdf_x[i]) # generate points for pdf dr_bins,dr_histogram = make_histogram_data(data=self.dr,n_bins=100) rayleigh_pdf = flex.double(len(dr_bins)) for i in xrange(len(dr_bins)): rayleigh_pdf[i] = fitted_rayleigh.distribution.pdf(x=dr_bins[i]) rayleigh_pdf = rayleigh_pdf/flex.sum(rayleigh_pdf) dr_bins = flex.double(dr_bins) dr_histogram = flex.double(dr_histogram) # standard deviation for cdf sd = math.sqrt((4.0-math.pi)/(2.0)* fitted_rayleigh.x[0]*fitted_rayleigh.x[0]) if self.verbose:print 'Standard deviation of Rayleigh fit = %4.3f'%sd sd_data = None radius_outlier_index = None limit_outlier = None for i in xrange(len(rayleigh_cdf_x)): mx = rayleigh_cdf_x[i] my = rayleigh_cdf[i] for j in xrange(1,len(self.dr)): upper_x = self.dr[j] upper_y = self.x[j] lower_x = self.dr[j-1] lower_y = self.x[j-1] if ((my >= lower_y) and (my < upper_y)): if ((sd <= (upper_x - mx)) and ((lower_x - mx) > 0.0)): sd_data = ((mx,my),(lower_x,lower_y)) radius_outlier_index = j-1 limit_outlier = lower_x if self.verbose:print "Width of green bar = %4.3f"%(lower_x - mx) break if (sd_data is not None): break if (radius_outlier_index is None): radius_outlier_index = len(self.dr) if (limit_outlier is None): limit_outlier = self.dr[-1] radius_95 = None for i in xrange(len(rayleigh_cdf)): if (rayleigh_cdf[i] >= 0.95): radius_95 = rayleigh_cdf_x[i] break if (radius_95 is None): radius_95 = rayleigh_cdf_x[-1] upper_circle = [] lower_circle = [] d_radius = 2.0*radius_95/100.0 x = -radius_95 r2 = radius_95*radius_95 for i in xrange(100): y = math.sqrt(r2 - x*x) upper_circle.append((x,y)) lower_circle.append((x,-y)) x = x + d_radius y = 0.0 upper_circle.append((x,y)) lower_circle.append((x,-y)) self.sqrtr2 = math.sqrt(r2) # color code dx dy dxdy_fraction = [] dxdy_inliers = [] dxdy_outliers = [] limit = self.dr[int(self.fraction*len(self.dr))] trifold = dict(fraction=0,inlier=0,outlier=0,total=0) for key in self.dr: trifold["total"]+=1 i = self.sorted_observed_spots[key] if (key < limit): trifold["fraction"]+=1 if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or (self.dy[i] > 1.0) or (self.dy[i] < -1.0))): dxdy_fraction.append((self.dx[i],self.dy[i])) elif (key < limit_outlier): trifold["inlier"]+=1 if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or (self.dy[i] > 1.0) or (self.dy[i] < -1.0))): dxdy_inliers.append((self.dx[i],self.dy[i])) else: trifold["outlier"]+=1 if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or (self.dy[i] > 1.0) or (self.dy[i] < -1.0))): dxdy_outliers.append((self.dx[i],self.dy[i])) if verbose: print ", ".join(["=".join([str(i[0]),"%d"%i[1]]) for i in trifold.items()]) # color code observed fractions o_fraction = [] o_inliers = [] o_outliers = [] mr = format_data(x_data=rayleigh_cdf_x,y_data=rayleigh_cdf) limit = int(self.fraction*len(self.dr)) for i in xrange(len(self.dr)): if (self.dr[i] <= 1.0): if (i < limit): o_fraction.append((self.dr[i],self.x[i])) elif (i < radius_outlier_index): o_inliers.append((self.dr[i],self.x[i])) else: o_outliers.append((self.dr[i],self.x[i])) if horizon_phil.indexing.outlier_detection.verbose: o_outliers_for_severity = [] for i in xrange(radius_outlier_index, len(self.dr)): o_outliers_for_severity.append((self.dr[i],self.x[i])) # limit data range for i in xrange(len(dr_bins)): if (dr_bins[i] > 1.0): dr_bins.resize(i) dr_histogram.resize(i) rayleigh_pdf.resize(i) break ho = format_data(x_data=dr_bins,y_data=dr_histogram) hr = format_data(x_data=dr_bins,y_data=rayleigh_pdf) # format data for graphing self.plot_dxdy_data = [dxdy_fraction,dxdy_inliers,dxdy_outliers, [(0.0,0.0)],[],[],[],[]] self.framework = {4:dict(status=SpotClass.SPINDLE), 5:dict(status=SpotClass.OVERLAP), 6:dict(status=SpotClass.OUTLIER), 7:dict(status=SpotClass.ICE), } for key in self.not_good_dr: i = self.sorted_observed_spots[key] status = current_status[i] if (not ((self.dx[i] > 1.0) or (self.dx[i] < -1.0) or (self.dy[i] > 1.0) or (self.dy[i] < -1.0))): statuskey = [k for k in self.framework.keys() if self.framework[k]["status"]==status][0] self.plot_dxdy_data[statuskey].append((self.dx[i],self.dy[i])) self.plot_cdf_data = [mr,o_fraction,o_inliers,o_outliers] if (sd_data is not None): self.plot_cdf_data.append(sd_data) self.plot_pdf_data = [ho,hr] # mark outliers if (first_time_through): #i.e., first time through the update() method if (radius_outlier_index < len(self.dr)): for i in xrange(radius_outlier_index,len(self.dr)): current_status[self.sorted_observed_spots[self.dr[i]]] = SpotClass.OUTLIER # reset good spots self.good = [False for i in xrange(len(self.observed_spots))] for i in xrange(len(self.observed_spots)): if (current_status[i] == SpotClass.GOOD): self.good[i] = True count_outlier = 0 count_good = 0 for i in xrange(len(self.observed_spots)): if (current_status[i] == SpotClass.OUTLIER): count_outlier = count_outlier + 1 elif (current_status[i] == SpotClass.GOOD): count_good = count_good + 1 if self.verbose:print 'Old GOOD =', len(self.dr),\ 'OUTLIER =', count_outlier,\ 'New GOOD =', count_good if horizon_phil.indexing.outlier_detection.verbose and status_with_marked_outliers is None: print "\nOf the remaining %d spots, %.1f%% were lattice outliers, leaving %d well-fit spots"%( len(self.dr),100.*count_outlier/len(self.dr), count_good ) if count_outlier==0:return #width of green bar is sd delta_spread = o_outliers_for_severity[1][1]-o_outliers_for_severity[0][1] severity = 0. for item in o_outliers_for_severity: delta_r = item[0] # obs - predicted deviation in mm spread = item[1] # order of observed deviation on a scale from 0 to 1 # now invert the cdf to find expected delta r: expected_delta_r = fitted_rayleigh.distribution.sigma * math.sqrt( -2.* math.log(1.-spread) ) #print item, expected_delta_r, (delta_r - expected_delta_r) / sd severity += ((delta_r - expected_delta_r) / sd) severity *= delta_spread print "The outlier severity is %.2f sigma [defined in J Appl Cryst (2010) 43, p.611 sec. 4].\n"%severity return current_status
def integration_concept(self,image_number=0,cb_op_to_primitive=None,verbose=False,**kwargs): self.image_number = image_number NEAR = 10 pxlsz = self.pixel_size self.get_predictions_accounting_for_centering(cb_op_to_primitive,**kwargs) FWMOSAICITY = self.inputai.getMosaicity() DOMAIN_SZ_ANG = kwargs.get("domain_size_ang", self.__dict__.get("actual",0) ) refineflag = {True:0,False:1}[kwargs.get("domain_size_ang",0)==0] self.inputpd["symmetry"].show_summary(prefix="EXCURSION%1d REPORT FWMOS= %6.4f DOMAIN= %6.1f "%(refineflag,FWMOSAICITY,DOMAIN_SZ_ANG)) from annlib_ext import AnnAdaptor self.cell = self.inputai.getOrientation().unit_cell() query = flex.double() for pred in self.predicted: # predicted spot coord in pixels query.append(pred[0]/pxlsz) query.append(pred[1]/pxlsz) self.reserve_hkllist_for_signal_search = self.hkllist reference = flex.double() spots = self.get_observations_with_outlier_removal() assert len(spots)>NEAR# Can't do spot/pred matching with too few spots for spot in spots: reference.append(spot.ctr_mass_x()) reference.append(spot.ctr_mass_y()) IS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR) IS_adapt.query(query) print "Calculate correction vectors for %d observations & %d predictions"%(len(spots),len(self.predicted)) indexed_pairs_provisional = [] correction_vectors_provisional = [] c_v_p_flex = flex.vec3_double() idx_cutoff = float(min(self.mask_focus[image_number])) if verbose: print "idx_cutoff distance in pixels",idx_cutoff if not self.horizons_phil.integration.enable_one_to_one_safeguard: # legacy code, no safeguard against many-to-one predicted-to-observation mapping for i in xrange(len(self.predicted)): # loop over predicteds #for n in xrange(NEAR): # loop over near spotfinder spots for n in xrange(1): # only consider the nearest spotfinder spot Match = dict(spot=IS_adapt.nn[i*NEAR+n],pred=i) if n==0 and math.sqrt(IS_adapt.distances[i*NEAR+n]) < idx_cutoff: indexed_pairs_provisional.append(Match) vector = matrix.col( [spots[Match["spot"]].ctr_mass_x() - self.predicted[Match["pred"]][0]/pxlsz, spots[Match["spot"]].ctr_mass_y() - self.predicted[Match["pred"]][1]/pxlsz]) correction_vectors_provisional.append(vector) c_v_p_flex.append((vector[0],vector[1],0.)) else: one_to_one = {} for i in xrange(len(self.predicted)): # loop over predicteds annresultidx = i*NEAR obsidx = IS_adapt.nn[annresultidx] this_distancesq = IS_adapt.distances[annresultidx] if not one_to_one.has_key(obsidx) or \ this_distancesq < one_to_one[obsidx]["distancesq"]: if math.sqrt(this_distancesq) < idx_cutoff: one_to_one[obsidx] = dict(spot=obsidx,pred=i,distancesq=this_distancesq) for key,value in one_to_one.items(): indexed_pairs_provisional.append(value) vector = matrix.col( [spots[value["spot"]].ctr_mass_x() - self.predicted[value["pred"]][0]/pxlsz, spots[value["spot"]].ctr_mass_y() - self.predicted[value["pred"]][1]/pxlsz]) correction_vectors_provisional.append(vector) c_v_p_flex.append((vector[0],vector[1],0.)) print "... %d provisional matches"%len(correction_vectors_provisional), print "r.m.s.d. in pixels: %5.2f"%(math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex)))) if self.horizons_phil.integration.enable_residual_scatter: from matplotlib import pyplot as plt fig = plt.figure() for cv in correction_vectors_provisional: plt.plot([cv[1]],[-cv[0]],"b.") plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt,fig,"res") plt.close() if self.horizons_phil.integration.enable_residual_map: from matplotlib import pyplot as plt fig = plt.figure() for match,cv in zip(indexed_pairs_provisional,correction_vectors_provisional): plt.plot([spots[match["spot"]].ctr_mass_y()],[-spots[match["spot"]].ctr_mass_x()],"r.") plt.plot([self.predicted[match["pred"]][1]/pxlsz],[-self.predicted[match["pred"]][0]/pxlsz],"g.") plt.plot([spots[match["spot"]].ctr_mass_y(), spots[match["spot"]].ctr_mass_y() + 10.*cv[1]], [-spots[match["spot"]].ctr_mass_x(), -spots[match["spot"]].ctr_mass_x() - 10.*cv[0]],'b-') plt.xlim([0,float(self.inputpd["size2"])]) plt.ylim([-float(self.inputpd["size1"]),0]) plt.title(" %d matches, r.m.s.d. %5.2f pixels"%(len(correction_vectors_provisional),math.sqrt(flex.mean(c_v_p_flex.dot(c_v_p_flex))))) plt.axes().set_aspect("equal") self.show_figure(plt,fig,"map") plt.close() # insert code here to remove correction length outliers... # they are causing terrible # problems for finding legitimate correction vectors (print out the list) # also remove outliers for the purpose of reporting RMS outlier_rejection = True cache_refinement_spots = getattr(slip_callbacks.slip_callback,"requires_refinement_spots",False) if outlier_rejection: correction_lengths = flex.double([v.length() for v in correction_vectors_provisional]) clorder = flex.sort_permutation(correction_lengths) sorted_cl = correction_lengths.select(clorder) ACCEPTABLE_LIMIT = 2 limit = int(0.33 * len(sorted_cl)) # best 1/3 of data are assumed to be correctly modeled. if (limit <= ACCEPTABLE_LIMIT): raise Sorry("Not enough indexed spots to reject outliers; have %d need >%d" % (limit, ACCEPTABLE_LIMIT)) y_data = flex.double(len(sorted_cl)) for i in xrange(len(y_data)): y_data[i] = float(i)/float(len(y_data)) # ideas are explained in Sauter & Poon (2010) J Appl Cryst 43, 611-616. from rstbx.outlier_spots.fit_distribution import fit_cdf,rayleigh fitted_rayleigh = fit_cdf(x_data = sorted_cl[0:limit], y_data = y_data[0:limit], distribution=rayleigh) inv_cdf = [fitted_rayleigh.distribution.inv_cdf(cdf) for cdf in y_data] #print "SORTED LIST OF ",len(sorted_cl), "with sigma",fitted_rayleigh.distribution.sigma indexed_pairs = [] correction_vectors = [] self.correction_vectors = [] for icand in xrange(len(sorted_cl)): # somewhat arbitrary sigma = 1.0 cutoff for outliers if (sorted_cl[icand]-inv_cdf[icand])/fitted_rayleigh.distribution.sigma > 1.0: break indexed_pairs.append(indexed_pairs_provisional[clorder[icand]]) correction_vectors.append(correction_vectors_provisional[clorder[icand]]) if cache_refinement_spots: self.spotfinder.images[self.frame_numbers[self.image_number]]["refinement_spots"].append( spots[indexed_pairs[-1]["spot"]]) if kwargs.get("verbose_cv")==True: print "CV OBSCENTER %7.2f %7.2f REFINEDCENTER %7.2f %7.2f"%( float(self.inputpd["size1"])/2.,float(self.inputpd["size2"])/2., self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz), print "OBSSPOT %7.2f %7.2f PREDSPOT %7.2f %7.2f"%( spots[indexed_pairs[-1]["spot"]].ctr_mass_x(), spots[indexed_pairs[-1]["spot"]].ctr_mass_y(), self.predicted[indexed_pairs[-1]["pred"]][0]/pxlsz, self.predicted[indexed_pairs[-1]["pred"]][1]/pxlsz), the_hkl = self.hkllist[indexed_pairs[-1]["pred"]] print "HKL %4d %4d %4d"%the_hkl,"%2d"%self.setting_id, radial, azimuthal = spots[indexed_pairs[-1]["spot"]].get_radial_and_azimuthal_size( self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz) print "RADIALpx %5.3f AZIMUTpx %5.3f"%(radial,azimuthal) # Store a list of correction vectors in self. radial, azimuthal = spots[indexed_pairs[-1]['spot']].get_radial_and_azimuthal_size( self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz) self.correction_vectors.append( dict(obscenter=(float(self.inputpd['size1']) / 2, float(self.inputpd['size2']) / 2), refinedcenter=(self.inputai.xbeam() / pxlsz, self.inputai.ybeam() / pxlsz), obsspot=(spots[indexed_pairs[-1]['spot']].ctr_mass_x(), spots[indexed_pairs[-1]['spot']].ctr_mass_y()), predspot=(self.predicted[indexed_pairs[-1]['pred']][0] / pxlsz, self.predicted[indexed_pairs[-1]['pred']][1] / pxlsz), hkl=(self.hkllist[indexed_pairs[-1]['pred']][0], self.hkllist[indexed_pairs[-1]['pred']][1], self.hkllist[indexed_pairs[-1]['pred']][2]), setting_id=self.setting_id, radial=radial, azimuthal=azimuthal)) print "After outlier rejection %d indexed spotfinder spots remain."%len(indexed_pairs) if False: rayleigh_cdf = [ fitted_rayleigh.distribution.cdf(x=sorted_cl[c]) for c in xrange(len(sorted_cl))] from matplotlib import pyplot as plt plt.plot(sorted_cl,y_data,"r+") #plt.plot(sorted_cl,rayleigh_cdf,"g.") plt.plot(inv_cdf,y_data,"b.") plt.show() else: indexed_pairs = indexed_pairs_provisional correction_vectors = correction_vectors_provisional ########### finished with outlier rejection self.inputpd["symmetry"].show_summary(prefix="SETTING ") is_triclinic = (self.setting_id==1) if is_triclinic: self.triclinic_pairs = [ dict(pred=self.hkllist[a["pred"]],spot=a["spot"]) for a in indexed_pairs ] if self.horizons_phil.integration.model == "user_supplied": if kwargs.get("user-reentrant",None)==None: from cxi_user import post_outlier_rejection self.indexed_pairs = indexed_pairs self.spots = spots post_outlier_rejection(self,image_number,cb_op_to_primitive,self.horizons_phil,kwargs) return ########### finished with user-supplied code if self.horizons_phil.integration.spot_shape_verbose: from rstbx.new_horizons.spot_shape import spot_shape_verbose spot_shape_verbose(rawdata = self.imagefiles.images[self.image_number].linearintdata, beam_center_pix = matrix.col((self.inputai.xbeam()/pxlsz, self.inputai.ybeam()/pxlsz)), indexed_pairs = indexed_pairs, spotfinder_observations = spots, distance_mm = self.inputai.distance(), mm_per_pixel = pxlsz, hkllist = self.hkllist, unit_cell = self.cell, wavelength_ang = self.inputai.wavelength ) #Other checks to be implemented (future): # spot is within active area of detector on a circular detector such as the Mar IP # integration masks do not overlap; or deconvolute correction_lengths=flex.double([v.length() for v in correction_vectors]) if verbose: print "average correction %5.2f over %d vectors"%(flex.mean(correction_lengths), len(correction_lengths)), print "or %5.2f mm."%(pxlsz*flex.mean(correction_lengths)) self.r_residual = pxlsz*flex.mean(correction_lengths) #assert len(indexed_pairs)>NEAR # must have enough indexed spots if (len(indexed_pairs) <= NEAR): raise Sorry("Not enough indexed spots, only found %d, need %d" % (len(indexed_pairs), NEAR)) reference = flex.double() for item in indexed_pairs: reference.append(spots[item["spot"]].ctr_mass_x()) reference.append(spots[item["spot"]].ctr_mass_y()) PS_adapt = AnnAdaptor(data=reference,dim=2,k=NEAR) PS_adapt.query(query) self.BSmasks = [] #self.null_correction_mapping( predicted=self.predicted, # correction_vectors = correction_vectors, # IS_adapt = IS_adapt, # spots = spots) self.positional_correction_mapping( predicted=self.predicted, correction_vectors = correction_vectors, PS_adapt = PS_adapt, IS_adapt = IS_adapt, spots = spots) # which spots are close enough to interfere with background? MAXOVER=6 OS_adapt = AnnAdaptor(data=query,dim=2,k=MAXOVER) #six near nbrs OS_adapt.query(query) if self.mask_focus[image_number] is None: raise Sorry("No observed/predicted spot agreement; no Spotfinder masks; skip integration") nbr_cutoff = 2.0* max(self.mask_focus[image_number]) FRAME = int(nbr_cutoff/2) #print "The overlap cutoff is %d pixels"%nbr_cutoff nbr_cutoff_sq = nbr_cutoff * nbr_cutoff #print "Optimized C++ section...", self.set_frame(FRAME) self.set_background_factor(kwargs["background_factor"]) self.set_nbr_cutoff_sq(nbr_cutoff_sq) self.set_guard_width_sq(self.horizons_phil.integration.guard_width_sq) self.set_detector_gain(self.horizons_phil.integration.detector_gain) flex_sorted = flex.int() for item in self.sorted: flex_sorted.append(item[0]);flex_sorted.append(item[1]); if self.horizons_phil.integration.mask_pixel_value is not None: self.set_mask_pixel_val(self.horizons_phil.integration.mask_pixel_value) image_obj = self.imagefiles.imageindex(self.frame_numbers[self.image_number]) image_obj.read() rawdata = image_obj.linearintdata # assume image #1 if self.inputai.active_areas != None: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted, tiles=self.inputai.active_areas.IT, tile_id=self.inputai.active_areas.tile_id); else: self.detector_xy_draft = self.safe_background( rawdata=rawdata, predicted=self.predicted, OS_adapt=OS_adapt, sorted=flex_sorted); for i in xrange(len(self.predicted)): # loop over predicteds B_S_mask = {} keys = self.get_bsmask(i) for k in xrange(0,len(keys),2): B_S_mask[(keys[k],keys[k+1])]=True self.BSmasks.append(B_S_mask) #print "Done" return