def __call__(self, sigma_m): """Calculate the fraction of observed intensity for each observation. Params: sigma_m The mosaicity Returns: A list of log intensity fractions """ # Tiny value TINY = 1e-10 assert sigma_m > TINY # Calculate the two components to the fraction a = scitbx.math.erf(self.e1 / sigma_m) b = scitbx.math.erf(self.e2 / sigma_m) # Calculate the fraction of observed reflection intensity R = (a - b) / 2.0 # Set any points <= 0 to 1e-10 (otherwise will get a floating # point error in log calculation below). assert R.all_ge(0) mask = R < TINY assert mask.count(True) < len(mask) R.set_selected(mask, TINY) # Return the logarithm of r return flex.log(R)
def target(self, log_sigma): """The target for minimization.""" sigma_m = math.exp(log_sigma[0]) # Tiny value TINY = 1e-10 assert sigma_m > TINY # Calculate the two components to the fraction a = scitbx.math.erf(self.e1 / sigma_m) b = scitbx.math.erf(self.e2 / sigma_m) n = self.n K = self.K # Calculate the fraction of observed reflection intensity zi = (a - b) / 2.0 # Set any points <= 0 to 1e-10 (otherwise will get a floating # point error in log calculation below). assert zi.all_ge(0) mask = zi < TINY assert mask.count(True) < len(mask) zi.set_selected(mask, TINY) # Compute the likelihood # # The likelihood here is a result of the sum of two log likelihood # functions: # # The first is the same as the one in Kabsch2010 as applied to the # reflection as a whole. This results in the term log(Z) # # The second is the likelihood for each reflection modelling as a Poisson # distribtution with shape given by sigma M. This gives sum(ci log(zi)) - # sum(ci)*log(sum(zi)) # # If the reflection is recorded on 1 frame, the second component is zero # and so the likelihood is dominated by the first term which can be seen # as a prior for sigma, which accounts for which reflections were actually # recorded. # L = 0 for j, (i0, i1) in enumerate(zip(self.indices[:-1], self.indices[1:])): selection = flex.size_t(range(i0, i1)) zj = zi.select(selection) nj = n.select(selection) kj = K[j] Z = flex.sum(zj) # L += flex.sum(nj * flex.log(zj)) - kj * Z # L += flex.sum(nj * flex.log(zj)) - kj * math.log(Z) L += flex.sum( nj * flex.log(zj)) - kj * math.log(Z) + math.log(Z) logger.debug("Sigma M: %f, log(L): %f", sigma_m * 180 / math.pi, L) # Return the logarithm of r return -L
def i_over_s_hist(self, rlist): """ Analyse the correlations. """ I = rlist["intensity.sum.value"] I_sig = flex.sqrt(rlist["intensity.sum.variance"]) I_over_S = I / I_sig fig = pyplot.figure() pyplot.title("Log I/Sigma histogram") pyplot.hist(flex.log(I_over_S), bins=20) pyplot.xlabel("Log I/Sigma") pyplot.ylabel("# reflections") fig.savefig(os.path.join(self.directory, "ioversigma_hist")) pyplot.close()
def i_over_s_hist(self, rlist): ''' Analyse the correlations. ''' from os.path import join I = rlist['intensity.sum.value'] I_sig = flex.sqrt(rlist['intensity.sum.variance']) I_over_S = I / I_sig fig = pyplot.figure() pyplot.title("Log I/Sigma histogram") pyplot.hist(flex.log(I_over_S), bins=20) pyplot.xlabel("Log I/Sigma") pyplot.ylabel("# reflections") fig.savefig(join(self.directory, "ioversigma_hist")) pyplot.close()
def plot_one_panel(self, ax, rlist): I_sig = flex.sqrt(rlist['intensity.%s.variance' %intensity_type]) sel = I_sig > 0 rlist = rlist.select(sel) I_sig = I_sig.select(sel) I = rlist['intensity.%s.value' %intensity_type] I_over_S = I / I_sig x, y, z = rlist['xyzcal.px'].parts() hex_ax = ax.hexbin( x.as_numpy_array(), y.as_numpy_array(), C=flex.log(I_over_S), gridsize=self.gridsize, ) return hex_ax
def i_over_s_vs_z(self, rlist): """ Plot I/Sigma vs Z. """ I = rlist["intensity.sum.value"] I_sig = flex.sqrt(rlist["intensity.sum.variance"]) I_over_S = I / I_sig x, y, z = rlist["xyzcal.px"].parts() fig = pyplot.figure() pyplot.title("Distribution of I/Sigma vs Z") cax = pyplot.hexbin(z, flex.log(I_over_S), gridsize=100) cax.axes.set_xlabel("z (images)") cax.axes.set_ylabel("Log I/Sigma") cbar = pyplot.colorbar(cax) cbar.ax.set_ylabel("# reflections") fig.savefig(os.path.join(self.directory, "ioversigma_vs_z.png")) pyplot.close()
def _i_over_s_vs_xy_plot(ax, rlist, gridsize): I_sig = flex.sqrt(rlist["intensity.%s.variance" % intensity_type]) sel = I_sig > 0 rlist = rlist.select(sel) I_sig = I_sig.select(sel) I = rlist["intensity.%s.value" % intensity_type] I_over_S = I / I_sig x, y, z = rlist["xyzcal.px"].parts() hex_ax = ax.hexbin( x.as_numpy_array(), y.as_numpy_array(), C=flex.log(I_over_S), gridsize=gridsize, ) return hex_ax
def i_over_s_vs_z(self, rlist): ''' Plot I/Sigma vs Z. ''' from os.path import join I = rlist['intensity.sum.value'] I_sig = flex.sqrt(rlist['intensity.sum.variance']) I_over_S = I / I_sig x, y, z = rlist['xyzcal.px'].parts() fig = pyplot.figure() pyplot.title("Distribution of I/Sigma vs Z") cax = pyplot.hexbin(z, flex.log(I_over_S), gridsize=100) cax.axes.set_xlabel("z") cax.axes.set_ylabel("Log I/Sigma") cbar = pyplot.colorbar(cax) cbar.ax.set_ylabel("# reflections") fig.savefig(join(self.directory, "ioversigma_vs_z.png")) pyplot.close()
def mean_vs_ios(self, rlist): ''' Analyse the correlations. ''' from os.path import join MEAN = rlist['background.mean'] I = rlist['intensity.sum.value'] I_sig = flex.sqrt(rlist['intensity.sum.variance']) I_over_S = I / I_sig mask = I_over_S > 0.1 I_over_S = I_over_S.select(mask) MEAN = MEAN.select(mask) fig = pyplot.figure() pyplot.title("Background Model mean vs Log I/Sigma") cax = pyplot.hexbin(flex.log(I_over_S), MEAN, gridsize=100) cbar = pyplot.colorbar(cax) cax.axes.set_xlabel("Log I/Sigma") cax.axes.set_ylabel("Background Model mean") cbar.ax.set_ylabel("# reflections") fig.savefig(join(self.directory, "background_model_mean_vs_ios.png")) pyplot.close()
def mean_vs_ios(self, rlist): """ Analyse the correlations. """ MEAN = rlist["background.mean"] I = rlist["intensity.sum.value"] I_sig = flex.sqrt(rlist["intensity.sum.variance"]) I_over_S = I / I_sig mask = I_over_S > 0.1 I_over_S = I_over_S.select(mask) MEAN = MEAN.select(mask) fig = pyplot.figure() pyplot.title("Background Model mean vs Log I/Sigma") cax = pyplot.hexbin(flex.log(I_over_S), MEAN, gridsize=100) cbar = pyplot.colorbar(cax) cax.axes.set_xlabel("Log I/Sigma") cax.axes.set_ylabel("Background Model mean") cbar.ax.set_ylabel("# reflections") fig.savefig( os.path.join(self.directory, "background_model_mean_vs_ios.png")) pyplot.close()
def ideal_reflection_corr_vs_ios(self, rlist, filename): ''' Analyse the correlations. ''' from os.path import join if 'correlation.ideal.profile' in rlist: corr = rlist['correlation.ideal.profile'] I = rlist['intensity.prf.value'] I_sig = flex.sqrt(rlist['intensity.prf.variance']) mask = I_sig > 0 I = I.select(mask) I_sig = I_sig.select(mask) corr = corr.select(mask) I_over_S = I / I_sig mask = I_over_S > 0.1 I_over_S = I_over_S.select(mask) corr = corr.select(mask) pyplot.title("Reflection correlations vs Log I/Sigma") cax = pyplot.hexbin(flex.log(I_over_S), corr, gridsize=100) cbar = pyplot.colorbar(cax) pyplot.xlabel("Log I/Sigma") pyplot.ylabel("Correlation with reference profile") cbar.ax.set_ylabel("# reflections") pyplot.savefig(join(self.directory, "ideal_%s_corr_vs_ios.png" % filename)) pyplot.close()
def reflection_corr_vs_ios(self, rlist, filename): """ Analyse the correlations. """ corr = rlist["profile.correlation"] I = rlist["intensity.prf.value"] I_sig = flex.sqrt(rlist["intensity.prf.variance"]) mask = I_sig > 0 I = I.select(mask) I_sig = I_sig.select(mask) corr = corr.select(mask) I_over_S = I / I_sig mask = I_over_S > 0.1 I_over_S = I_over_S.select(mask) corr = corr.select(mask) fig = pyplot.figure() pyplot.title("Reflection correlations vs Log I/Sigma") cax = pyplot.hexbin(flex.log(I_over_S), corr, gridsize=100) cbar = pyplot.colorbar(cax) cax.axes.set_xlabel("Log I/Sigma") cax.axes.set_ylabel("Correlation with reference profile") cbar.ax.set_ylabel("# reflections") fig.savefig( os.path.join(self.directory, "%s_corr_vs_ios.png" % filename)) pyplot.close()
def estimate_resolution_limit(reflections, imageset, ice_sel=None, plot_filename=None): if ice_sel is None: ice_sel = flex.bool(len(reflections), False) d_star_sq = flex.pow2(reflections['rlp'].norms()) d_spacings = uctbx.d_star_sq_as_d(d_star_sq) intensities = reflections['intensity.sum.value'] variances = reflections['intensity.sum.variance'] sel = variances > 0 intensities = intensities.select(sel) variances = variances.select(sel) ice_sel = ice_sel.select(sel) i_over_sigi = intensities / flex.sqrt(variances) log_i_over_sigi = flex.log(i_over_sigi) fit = flex.linear_regression(d_star_sq.select(~ice_sel), log_i_over_sigi.select(~ice_sel)) m = fit.slope() c = fit.y_intercept() log_i_sigi_lower = flex.double() d_star_sq_lower = flex.double() log_i_sigi_upper = flex.double() d_star_sq_upper = flex.double() binner = binner_equal_population(d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5) outliers_all = flex.bool(len(reflections), False) low_percentile_limit = 0.1 upper_percentile_limit = 1 - low_percentile_limit d_spacings = uctbx.d_star_sq_as_d(d_star_sq) for i_slot, slot in enumerate(binner.bins): sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min) sel = ~(ice_sel) & sel_all #sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min) #print "%.2f" %(sel.count(True)/sel_all.count(True)) if sel.count(True) == 0: #outliers_all.set_selected(sel_all & ice_sel, True) continue #if i_slot > i_slot_max: #break #else: #continue outliers = wilson_outliers(reflections.select(sel_all), ice_sel=ice_sel.select(sel_all)) #print "rejecting %d wilson outliers" %outliers.count(True) outliers_all.set_selected(sel_all, outliers) #if sel.count(True)/sel_all.count(True) < 0.25: #outliers_all.set_selected(sel_all & ice_sel, True) #from scitbx.math import median_statistics #intensities_sel = intensities.select(sel) #stats = median_statistics(intensities_sel) #z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation #outliers = z_score > 3.5 #perm = flex.sort_permutation(intensities_sel) ##print ' '.join('%.2f' %v for v in intensities_sel.select(perm)) ##print ' '.join('%.2f' %v for v in z_score.select(perm)) ##print isel = sel_all.iselection().select(~(outliers) & ~(ice_sel).select(sel_all)) log_i_over_sigi_sel = log_i_over_sigi.select(isel) d_star_sq_sel = d_star_sq.select(isel) perm = flex.sort_permutation(log_i_over_sigi_sel) i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))] i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))] log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower]) log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper]) d_star_sq_upper.append(d_star_sq_sel[i_lower]) d_star_sq_lower.append(d_star_sq_sel[i_upper]) fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper) m_upper = fit_upper.slope() c_upper = fit_upper.y_intercept() fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower) m_lower = fit_lower.slope() c_lower = fit_lower.y_intercept() #fit_upper.show_summary() #fit_lower.show_summary() if m_upper == m_lower: intersection = (-1, -1) resolution_estimate = -1 inside = flex.bool(len(d_star_sq), False) else: # http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines intersection = ((c_lower - c_upper) / (m_upper - m_lower), (m_upper * c_lower - m_lower * c_upper) / (m_upper - m_lower)) a = m_upper c_ = c_upper b = m_lower d = c_lower assert intersection == ((d - c_) / (a - b), (a * d - b * c_) / (a - b)) #inside = points_inside_envelope( #d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower) inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper, c_upper) inside = inside & ~outliers_all if inside.count(True) > 0: d_star_sq_estimate = flex.max(d_star_sq.select(inside)) #d_star_sq_estimate = intersection[0] resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate) else: resolution_estimate = -1 #resolution_estimate = max(resolution_estimate, flex.min(d_spacings)) if plot_filename is not None: if pyplot is None: raise Sorry("matplotlib must be installed to generate a plot.") fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.scatter(d_star_sq, log_i_over_sigi, marker='+') ax.scatter(d_star_sq.select(inside), log_i_over_sigi.select(inside), marker='+', color='green') ax.scatter(d_star_sq.select(ice_sel), log_i_over_sigi.select(ice_sel), marker='+', color='black') ax.scatter(d_star_sq.select(outliers_all), log_i_over_sigi.select(outliers_all), marker='+', color='grey') ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red') ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red') if (intersection[0] <= ax.get_xlim()[1] and intersection[1] <= ax.get_ylim()[1]): ax.scatter([intersection[0]], [intersection[1]], marker='x', s=50, color='b') #ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30) xlim = pyplot.xlim() ax.plot(xlim, [(m * x + c) for x in xlim]) ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red') ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red') ax.set_xlabel('d_star_sq') ax.set_ylabel('ln(I/sigI)') ax.set_xlim((max(-xlim[1], -0.05), xlim[1])) ax.set_ylim((0, ax.get_ylim()[1])) for i_slot, slot in enumerate(binner.bins): if i_slot == 0: ax.vlines(uctbx.d_as_d_star_sq(slot.d_max), 0, ax.get_ylim()[1], linestyle='dotted', color='grey') ax.vlines(uctbx.d_as_d_star_sq(slot.d_min), 0, ax.get_ylim()[1], linestyle='dotted', color='grey') ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis xticks = ax.get_xticks() xlim = ax.get_xlim() xticks_d = [ uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ] xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks] ax_.set_xticks(xticks) ax_.set_xlim(ax.get_xlim()) ax_.set_xlabel(r"Resolution ($\AA$)") ax_.set_xticklabels(["%.1f" % d for d in xticks_d]) #pyplot.show() pyplot.savefig(plot_filename) pyplot.close() return resolution_estimate
def paper_test(B, S): from numpy.random import poisson from math import exp background_shape = [1 for i in range(20)] signal_shape = [1 if i >= 6 and i < 15 else 0 for i in range(20)] background = [poisson(bb * B,1)[0] for bb in background_shape] signal = [poisson(ss * S, 1)[0] for ss in signal_shape] # background = [bb * B for bb in background_shape] # signal = [ss * S for ss in signal_shape] total = [b + s for b, s in zip(background, signal)] # from matplotlib import pylab # pylab.plot(total) # pylab.plot(signal) # pylab.plot(background) # pylab.show() total = [0, 1, 0, 0, 0, 0, 3, 1, 3, 3, 6, 6, 4, 1, 4, 0, 2, 0, 1, 1] total = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] # plot_prob_for_zero(total, background_shape, signal_shape) # signal_shape = [exp(-(x - 10.0)**2 / (2*3.0**2)) for x in range(20)] # signal_shape = [ss / sum(signal_shape) for ss in signal_shape] # print signal_shape #plot_valid(background_shape, signal_shape) B = 155.0 / 296.0 from dials.array_family import flex from math import log, factorial V = flex.double(flex.grid(100, 100)) L = flex.double(flex.grid(100, 100)) DB = flex.double(flex.grid(100, 100)) DS = flex.double(flex.grid(100, 100)) P = flex.double(flex.grid(100, 100)) Fb = sum(background_shape) Fs = sum(signal_shape) SV = [] MASK = flex.bool(flex.grid(100, 100), False) for BB in range(100): for SS in range(100): B = -5.0 + (BB) / 10.0 S = -5.0 + (SS) / 10.0 # SV.append(S) VV = 0 LL = 0 DDB = 0 DDS = 0 for i in range(20): s = signal_shape[i] b = background_shape[i] c = total[i] if B*b + S*s <= 0: # MASK[BB, SS] = True LL = 0 if b == 0: DDB += 0 else: DDB += 1e7 if s == 0: DDS += 0 else: DDS += 1e7 # break else: # VV += (b + s)*c / (B*b + S*s) # LL += c*log(B*b+S*s) - log(factorial(c)) - B*b - S*s DDB += c*b/(B*b+S*s) - b DDS += c*s/(B*b+S*s) - s VV -= (Fb + Fs) # print B, S, VV # V[BB, SS] = abs(VV) L[BB, SS] = LL DB[BB,SS] = DDB DS[BB,SS] = DDS max_ind = flex.max_index(L) j = max_ind // 100 i = max_ind % 100 print "Approx: ", (j+1) / 20.0, (i+1) / 20.0 print "Min/Max DB: ", flex.min(DB), flex.max(DB) print "Min/Max DS: ", flex.min(DS), flex.max(DS) from matplotlib import pylab # pylab.imshow(flex.log(V).as_numpy_array(), extent=[0.05, 5.05, 5.05, 0.05]) # pylab.plot(SV, V) # pylab.plot(SV, [0] * 100) # pylab.show() im = flex.exp(L).as_numpy_array() import numpy # im = numpy.ma.masked_array(im, mask=MASK.as_numpy_array()) # pylab.imshow(im)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower') pylab.imshow(DB.as_numpy_array(), vmin=-100, vmax=100)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower') pylab.contour(DB.as_numpy_array(), levels=[0], colors=['red']) pylab.contour(DS.as_numpy_array(), levels=[0], colors=['black']) pylab.show() # im = numpy.ma.masked_array(DB.as_numpy_array(), mask=MASK.as_numpy_array()) # pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100) # pylab.show() # im = numpy.ma.masked_array(DS.as_numpy_array(), mask=MASK.as_numpy_array()) # pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100) # pylab.show() # exit(0) S1, B1 = value(total, background_shape, signal_shape) exit(0) try: S1, B1 = value(total, background_shape, signal_shape) print "Result:" print S1, B1 exit(0) except Exception, e: raise e import sys import traceback traceback.print_exc() from dials.array_family import flex Fs = sum(signal_shape) Fb = sum(background_shape) Rs = flex.double(flex.grid(100, 100)) print "-----" print B, S # from matplotlib import pylab # pylab.plot(total) # pylab.show() print background_shape print signal_shape print total from math import exp, factorial, log minx = -1 miny = -1 minr = 9999 for BB in range(0, 100): for SS in range(0, 100): B = -10 + (BB) / 5.0 S = -10 + (SS) / 5.0 L = 0 Fb2 = 0 Fs2 = 0 for i in range(len(total)): c = total[i] b = background_shape[i] s = signal_shape[i] # P = exp(-(B*b + S*s)) * (B*b+S*s)**c / factorial(c) # print P # if P > 0: # L += log(P) den = B*b + S*s num1 = b*c num2 = s*c if den != 0: Fb2 += num1 / den Fs2 += num2 / den R = (Fb2 - Fb)**2 + (Fs2 - Fs)**2 if R > 1000: R = 0 # Rs[BB,SS] = L#R#Fs2 - Fs Rs[BB,SS] = R#Fs2 - Fs from matplotlib import pylab pylab.imshow(flex.log(Rs).as_numpy_array(), extent=[-5,5,5,-5]) pylab.show() exit(0)
def estimate(self, num_reflections): """ Estimate the model parameters """ from scitbx import simplex from copy import deepcopy from dials.array_family import flex from random import uniform # Select the reflections reflections = self._select_reflections(self.reflections, num_reflections) # The number of parameters num_parameters = len(self.history.names) # Setup the starting simplex if self.simplex is None: self.simplex = [] for i in range(num_parameters + 1): self.simplex.append( flex.log( flex.double([ uniform(0.0001, 0.01) for j in range(num_parameters) ]))) class Evaluator(object): """ Evaluator to simplex """ def __init__( self, history, experiment, reflections, num_integral, use_mosaic_block_angular_spread, use_wavelength_spread, ): """ Initialise """ from dials_scratch.jmp.profile_modelling import MLTarget3D self.func = MLTarget3D( experiment, reflections, num_integral=num_integral, use_mosaic_block_angular_spread= use_mosaic_block_angular_spread, use_wavelength_spread=use_wavelength_spread, ) self.count = 1 self.history = history self.logL = None def target(self, log_parameters): """ Compute the negative log likelihood """ from dials.array_family import flex parameters = flex.exp(log_parameters) self.logL = self.func.log_likelihood(parameters) logL = flex.sum(self.logL) self.count += 1 print(self.count, list(parameters), logL) self.history.append(parameters, logL) # Return negative log likelihood return -logL # Setup the simplex optimizer optimizer = simplex.simplex_opt( num_parameters, matrix=self.simplex, evaluator=Evaluator( self.history, self.experiments[0], reflections, num_integral=self.num_integral, use_mosaic_block_angular_spread=self. use_mosaic_block_angular_spread, use_wavelength_spread=self.use_wavelength_spread, ), tolerance=1e-7, ) # Get the solution self.parameters = flex.exp(optimizer.get_solution()) # Get the final simplex self.simplex = optimizer.matrix # Save the likelihood for each reflection self.log_likelihood = optimizer.evaluator.logL
def run(args): import libtbx.load_env usage = "%s experiments.json indexed.pickle [options]" % libtbx.env.dispatcher_name parser = OptionParser(usage=usage, phil=phil_scope, read_experiments=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if len(experiments) == 0: parser.print_help() return elif len(experiments) > 1: raise Sorry("More than one experiment present") experiment = experiments[0] assert (len(reflections) == 1) reflections = reflections[0] intensities = reflections['intensity.sum.value'] variances = reflections['intensity.sum.variance'] if 'intensity.prf.value' in reflections: intensities = reflections['intensity.prf.value'] variances = reflections['intensity.prf.variance'] sel = (variances > 0) intensities = intensities.select(sel) variances = variances.select(sel) sigmas = flex.sqrt(variances) indices = reflections['miller_index'].select(sel) from cctbx import crystal, miller crystal_symmetry = crystal.symmetry( space_group=experiment.crystal.get_space_group(), unit_cell=experiment.crystal.get_unit_cell()) miller_set = miller.set(crystal_symmetry=crystal_symmetry, anomalous_flag=True, indices=indices) miller_array = miller.array( miller_set=miller_set, data=intensities, sigmas=sigmas).set_observation_type_xray_intensity() #miller_array.setup_binner(n_bins=50, reflections_per_bin=100) miller_array.setup_binner(auto_binning=True, n_bins=20) result = miller_array.i_over_sig_i(use_binning=True) result.show() from cctbx import uctbx d_star_sq_centre = result.binner.bin_centers(2) i_over_sig_i = flex.double( [d if d is not None else 0 for d in result.data[1:-1]]) sel = (i_over_sig_i > 0) d_star_sq_centre = d_star_sq_centre.select(sel) i_over_sig_i = i_over_sig_i.select(sel) log_i_over_sig_i = flex.log(i_over_sig_i) weights = result.binner.counts()[1:-1].as_double().select(sel) fit = flex.linear_regression(d_star_sq_centre, log_i_over_sig_i, weights=weights) m = fit.slope() c = fit.y_intercept() import math y_cutoff = math.log(params.i_sigi_cutoff) x_cutoff = (y_cutoff - c) / m estimated_d_min = uctbx.d_star_sq_as_d(x_cutoff) print "estimated d_min: %.2f" % estimated_d_min if params.plot: from matplotlib import pyplot fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.plot(list(d_star_sq_centre), list(log_i_over_sig_i), label=r"ln(I/sigI)") ax.plot(pyplot.xlim(), [(m * x + c) for x in pyplot.xlim()], color='red') ax.plot([x_cutoff, x_cutoff], pyplot.ylim(), color='grey', linestyle='dashed') ax.plot(pyplot.xlim(), [y_cutoff, y_cutoff], color='grey', linestyle='dashed') ax.set_xlabel("d_star_sq") ax.set_ylabel("ln(I/sigI)") ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis xticks = ax.get_xticks() xlim = ax.get_xlim() xticks_d = [ uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ] xticks_ = [ds2 / (xlim[1] - xlim[0]) for ds2 in xticks] ax_.set_xticks(xticks) ax_.set_xlim(ax.get_xlim()) ax_.set_xlabel(r"Resolution ($\AA$)") ax_.set_xticklabels(["%.1f" % d for d in xticks_d]) pyplot.savefig("estimate_resolution_limit.png") pyplot.clf()
def plot_spotfinder_stats( stats, spot_length_stats, run_tags=[], run_statuses=[], minimalist=False, interactive=True, xsize=30, ysize=10, high_vis=False, title=None, ext='cbf', ): plot_ratio = max(min(xsize, ysize) / 2.5, 3) if high_vis: spot_ratio = plot_ratio * 4 text_ratio = plot_ratio * 4.5 else: spot_ratio = plot_ratio * 2 text_ratio = plot_ratio * 3 t, n_strong, enough_rate, n_min, window, lengths, boundaries, run_numbers = stats t_lengths, spot_lengths, spot_intensities = spot_length_stats # set up coloring of spot lengths by intensities from matplotlib.cm import ScalarMappable mappable = ScalarMappable(cmap="plasma") cmap = mappable.to_rgba(-flex.log(spot_intensities)) if len(t) == 0: return None n_runs = len(boundaries) // 2 if len(run_tags) != n_runs: run_tags = [[] for i in range(n_runs)] if len(run_statuses) != n_runs: run_statuses = [None for i in range(n_runs)] if minimalist: print "Minimalist mode activated." f, (ax1, ax2) = plt.subplots(2, sharex=True, sharey=False) axset = (ax1, ax2) else: f, (ax1, ax2, ax3) = plt.subplots(3, sharex=True, sharey=False) axset = (ax1, ax2, ax3) for a in axset: a.tick_params(axis='x', which='both', bottom='off', top='off') ax1.scatter(t, n_strong, edgecolors="none", color='deeppink', s=spot_ratio) ax1.set_ylim(ymin=0) ax1.axis('tight') ax1.set_ylabel("# strong spots", fontsize=text_ratio) ax1_twin = ax1.twinx() ax1_twin.plot(t, enough_rate * 100, color='orange') ax1_twin.set_ylim(ymin=0) ax1_twin.set_ylabel("%% images with\n>=%d spots" % n_min, fontsize=text_ratio) if spot_length_stats: ax2.scatter(t_lengths, spot_lengths, edgecolors="none", color=cmap, s=spot_ratio) ax2.set_ylim(ymin=0) ax2.axis('tight') ax2.set_ylabel("spot lengths\n(pixels)", fontsize=text_ratio) f.subplots_adjust(hspace=0) # add lines and text summaries at the timestamp boundaries if not minimalist: for boundary in boundaries: if boundary is not None: for a in (ax1, ax2): a.axvline(x=boundary, ymin=0, ymax=2, linewidth=1, color='k') run_starts = boundaries[0::2] run_ends = boundaries[1::2] start = 0 end = -1 for idx in range(len(run_numbers)): start_t = run_starts[idx] end_t = run_ends[idx] if start_t is None or end_t is None: continue end += lengths[idx] slice_t = t[start:end + 1] slice_enough = n_strong[start:end + 1] > n_min tags = run_tags[idx] status = run_statuses[idx] if status == "DONE": status_color = 'blue' elif status in ["RUN", "PEND", "SUBMITTED"]: status_color = 'green' elif status is None: status_color = 'black' else: status_color = 'red' if minimalist: ax2.set_xlabel( "timestamp (s)\n# images shown as all (%3.1f Angstroms)" % d_min, fontsize=text_ratio) ax2.set_yticks([]) else: ax3.text(start_t, 2.85, " " + ", ".join(tags) + " [%s]" % status, fontsize=text_ratio, color=status_color) ax3.text(start_t, .85, "run %d" % run_numbers[idx], fontsize=text_ratio) ax3.text(start_t, .65, "%d images" % lengths[idx], fontsize=text_ratio) ax3.text(start_t, .45, "%d n>=%d" % (slice_enough.count(True), n_min), fontsize=text_ratio) if spot_length_stats and len(spot_lengths) > 0: ax3.text(start_t, .25, "avg %d px" % int(flex.sum(spot_lengths) / len(spot_lengths)), fontsize=text_ratio) ax3.set_xlabel("timestamp (s)", fontsize=text_ratio) ax3.set_yticks([]) for item in axset: item.tick_params(labelsize=text_ratio) start += lengths[idx] if title is not None: plt.title(title) f.set_size_inches(xsize, ysize) f.savefig("spotfinder_tmp.png", bbox_inches='tight', dpi=100) plt.close(f) return "spotfinder_tmp.png"
def estimate_resolution_limit(reflections, ice_sel=None, plot_filename=None): if ice_sel is None: ice_sel = flex.bool(len(reflections), False) d_star_sq = flex.pow2(reflections["rlp"].norms()) d_spacings = uctbx.d_star_sq_as_d(d_star_sq) intensities = reflections["intensity.sum.value"] variances = reflections["intensity.sum.variance"] sel = variances > 0 intensities = intensities.select(sel) variances = variances.select(sel) ice_sel = ice_sel.select(sel) i_over_sigi = intensities / flex.sqrt(variances) log_i_over_sigi = flex.log(i_over_sigi) fit = flex.linear_regression(d_star_sq.select(~ice_sel), log_i_over_sigi.select(~ice_sel)) m = fit.slope() c = fit.y_intercept() log_i_sigi_lower = flex.double() d_star_sq_lower = flex.double() log_i_sigi_upper = flex.double() d_star_sq_upper = flex.double() binner = binner_equal_population(d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5) outliers_all = flex.bool(len(reflections), False) low_percentile_limit = 0.1 upper_percentile_limit = 1 - low_percentile_limit for i_slot, slot in enumerate(binner.bins): sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min) sel = ~(ice_sel) & sel_all if sel.count(True) == 0: continue outliers = wilson_outliers(reflections.select(sel_all), ice_sel=ice_sel.select(sel_all)) outliers_all.set_selected(sel_all, outliers) isel = sel_all.iselection().select(~(outliers) & ~(ice_sel).select(sel_all)) log_i_over_sigi_sel = log_i_over_sigi.select(isel) d_star_sq_sel = d_star_sq.select(isel) perm = flex.sort_permutation(log_i_over_sigi_sel) i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))] i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))] log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower]) log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper]) d_star_sq_upper.append(d_star_sq_sel[i_lower]) d_star_sq_lower.append(d_star_sq_sel[i_upper]) fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper) m_upper = fit_upper.slope() c_upper = fit_upper.y_intercept() fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower) m_lower = fit_lower.slope() c_lower = fit_lower.y_intercept() if m_upper == m_lower: intersection = (-1, -1) resolution_estimate = -1 inside = flex.bool(len(d_star_sq), False) else: # http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines # with: # a_ = m_upper # b_ = m_lower # c_ = c_upper # d_ = c_lower # intersection == ((d_ - c_) / (a_ - b_), (a_ * d_ - b_ * c_) / (a_ - b_)) intersection = ( (c_lower - c_upper) / (m_upper - m_lower), (m_upper * c_lower - m_lower * c_upper) / (m_upper - m_lower), ) inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper, c_upper) inside = inside & ~outliers_all & ~ice_sel if inside.count(True) > 0: d_star_sq_estimate = flex.max(d_star_sq.select(inside)) resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate) else: resolution_estimate = -1 if plot_filename is not None: from matplotlib import pyplot fig = pyplot.figure() ax = fig.add_subplot(1, 1, 1) ax.scatter(d_star_sq, log_i_over_sigi, marker="+") ax.scatter( d_star_sq.select(inside), log_i_over_sigi.select(inside), marker="+", color="green", ) ax.scatter( d_star_sq.select(ice_sel), log_i_over_sigi.select(ice_sel), marker="+", color="black", ) ax.scatter( d_star_sq.select(outliers_all), log_i_over_sigi.select(outliers_all), marker="+", color="grey", ) ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker="+", color="red") ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker="+", color="red") if intersection[0] <= ax.get_xlim( )[1] and intersection[1] <= ax.get_ylim()[1]: ax.scatter([intersection[0]], [intersection[1]], marker="x", s=50, color="b") xlim = pyplot.xlim() ax.plot(xlim, [(m * x + c) for x in xlim]) ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color="red") ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color="red") ax.set_xlabel("d_star_sq") ax.set_ylabel("ln(I/sigI)") ax.set_xlim((max(-xlim[1], -0.05), xlim[1])) ax.set_ylim((0, ax.get_ylim()[1])) for i_slot, slot in enumerate(binner.bins): if i_slot == 0: ax.vlines( uctbx.d_as_d_star_sq(slot.d_max), 0, ax.get_ylim()[1], linestyle="dotted", color="grey", ) ax.vlines( uctbx.d_as_d_star_sq(slot.d_min), 0, ax.get_ylim()[1], linestyle="dotted", color="grey", ) ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis xticks = ax.get_xticks() xticks_d = [ uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ] ax_.set_xticks(xticks) ax_.set_xlim(ax.get_xlim()) ax_.set_xlabel(r"Resolution ($\AA$)") ax_.set_xticklabels(["%.1f" % d for d in xticks_d]) pyplot.savefig(plot_filename) pyplot.close() return resolution_estimate
def estimate_resolution_limit(reflections, imageset, ice_sel=None, plot_filename=None): if ice_sel is None: ice_sel = flex.bool(len(reflections), False) d_star_sq = flex.pow2(reflections['rlp'].norms()) d_spacings = uctbx.d_star_sq_as_d(d_star_sq) intensities = reflections['intensity.sum.value'] variances = reflections['intensity.sum.variance'] sel = variances > 0 intensities = intensities.select(sel) variances = variances.select(sel) ice_sel = ice_sel.select(sel) i_over_sigi = intensities/flex.sqrt(variances) log_i_over_sigi = flex.log(i_over_sigi) fit = flex.linear_regression( d_star_sq.select(~ice_sel), log_i_over_sigi.select(~ice_sel)) m = fit.slope() c = fit.y_intercept() log_i_sigi_lower = flex.double() d_star_sq_lower = flex.double() log_i_sigi_upper = flex.double() d_star_sq_upper = flex.double() binner = binner_equal_population( d_star_sq, target_n_per_bin=20, max_slots=20, min_slots=5) outliers_all = flex.bool(len(reflections), False) low_percentile_limit = 0.1 upper_percentile_limit = 1-low_percentile_limit d_spacings = uctbx.d_star_sq_as_d(d_star_sq) for i_slot, slot in enumerate(binner.bins): sel_all = (d_spacings < slot.d_max) & (d_spacings >= slot.d_min) sel = ~(ice_sel) & sel_all #sel = ~(ice_sel) & (d_spacings < slot.d_max) & (d_spacings >= slot.d_min) #print "%.2f" %(sel.count(True)/sel_all.count(True)) if sel.count(True) == 0: #outliers_all.set_selected(sel_all & ice_sel, True) continue #if i_slot > i_slot_max: #break #else: #continue outliers = wilson_outliers( reflections.select(sel_all), ice_sel=ice_sel.select(sel_all)) #print "rejecting %d wilson outliers" %outliers.count(True) outliers_all.set_selected(sel_all, outliers) #if sel.count(True)/sel_all.count(True) < 0.25: #outliers_all.set_selected(sel_all & ice_sel, True) #from scitbx.math import median_statistics #intensities_sel = intensities.select(sel) #stats = median_statistics(intensities_sel) #z_score = 0.6745 * (intensities_sel - stats.median)/stats.median_absolute_deviation #outliers = z_score > 3.5 #perm = flex.sort_permutation(intensities_sel) ##print ' '.join('%.2f' %v for v in intensities_sel.select(perm)) ##print ' '.join('%.2f' %v for v in z_score.select(perm)) ##print isel = sel_all.iselection().select(~(outliers) & ~(ice_sel).select(sel_all)) log_i_over_sigi_sel = log_i_over_sigi.select(isel) d_star_sq_sel = d_star_sq.select(isel) perm = flex.sort_permutation(log_i_over_sigi_sel) i_lower = perm[int(math.floor(low_percentile_limit * len(perm)))] i_upper = perm[int(math.floor(upper_percentile_limit * len(perm)))] log_i_sigi_lower.append(log_i_over_sigi_sel[i_lower]) log_i_sigi_upper.append(log_i_over_sigi_sel[i_upper]) d_star_sq_upper.append(d_star_sq_sel[i_lower]) d_star_sq_lower.append(d_star_sq_sel[i_upper]) fit_upper = flex.linear_regression(d_star_sq_upper, log_i_sigi_upper) m_upper = fit_upper.slope() c_upper = fit_upper.y_intercept() fit_lower = flex.linear_regression(d_star_sq_lower, log_i_sigi_lower) m_lower = fit_lower.slope() c_lower = fit_lower.y_intercept() #fit_upper.show_summary() #fit_lower.show_summary() if m_upper == m_lower: intersection = (-1,-1) resolution_estimate = -1 inside = flex.bool(len(d_star_sq), False) else: # http://en.wikipedia.org/wiki/Line%E2%80%93line_intersection#Given_the_equations_of_the_lines intersection = ( (c_lower-c_upper)/(m_upper-m_lower), (m_upper*c_lower-m_lower*c_upper)/(m_upper-m_lower)) a = m_upper c_ = c_upper b = m_lower d = c_lower assert intersection == ((d-c_)/(a-b), (a*d-b*c_)/(a-b)) #inside = points_inside_envelope( #d_star_sq, log_i_over_sigi, m_upper, c_upper, m_lower, c_lower) inside = points_below_line(d_star_sq, log_i_over_sigi, m_upper, c_upper) inside = inside & ~outliers_all if inside.count(True) > 0: d_star_sq_estimate = flex.max(d_star_sq.select(inside)) #d_star_sq_estimate = intersection[0] resolution_estimate = uctbx.d_star_sq_as_d(d_star_sq_estimate) else: resolution_estimate = -1 #resolution_estimate = max(resolution_estimate, flex.min(d_spacings)) if plot_filename is not None: if pyplot is None: raise Sorry("matplotlib must be installed to generate a plot.") fig = pyplot.figure() ax = fig.add_subplot(1,1,1) ax.scatter(d_star_sq, log_i_over_sigi, marker='+') ax.scatter(d_star_sq.select(inside), log_i_over_sigi.select(inside), marker='+', color='green') ax.scatter(d_star_sq.select(ice_sel), log_i_over_sigi.select(ice_sel), marker='+', color='black') ax.scatter(d_star_sq.select(outliers_all), log_i_over_sigi.select(outliers_all), marker='+', color='grey') ax.scatter(d_star_sq_upper, log_i_sigi_upper, marker='+', color='red') ax.scatter(d_star_sq_lower, log_i_sigi_lower, marker='+', color='red') if (intersection[0] <= ax.get_xlim()[1] and intersection[1] <= ax.get_ylim()[1]): ax.scatter([intersection[0]], [intersection[1]], marker='x', s=50, color='b') #ax.hexbin(d_star_sq, log_i_over_sigi, gridsize=30) xlim = pyplot.xlim() ax.plot(xlim, [(m * x + c) for x in xlim]) ax.plot(xlim, [(m_upper * x + c_upper) for x in xlim], color='red') ax.plot(xlim, [(m_lower * x + c_lower) for x in xlim], color='red') ax.set_xlabel('d_star_sq') ax.set_ylabel('ln(I/sigI)') ax.set_xlim((max(-xlim[1], -0.05), xlim[1])) ax.set_ylim((0, ax.get_ylim()[1])) for i_slot, slot in enumerate(binner.bins): if i_slot == 0: ax.vlines(uctbx.d_as_d_star_sq(slot.d_max), 0, ax.get_ylim()[1], linestyle='dotted', color='grey') ax.vlines(uctbx.d_as_d_star_sq(slot.d_min), 0, ax.get_ylim()[1], linestyle='dotted', color='grey') ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis xticks = ax.get_xticks() xlim = ax.get_xlim() xticks_d = [ uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ] xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks] ax_.set_xticks(xticks) ax_.set_xlim(ax.get_xlim()) ax_.set_xlabel(r"Resolution ($\AA$)") ax_.set_xticklabels(["%.1f" %d for d in xticks_d]) #pyplot.show() pyplot.savefig(plot_filename) pyplot.close() return resolution_estimate
def run(args): import libtbx.load_env usage = "%s experiments.json indexed.pickle [options]" %libtbx.env.dispatcher_name parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if len(experiments) == 0: parser.print_help() return elif len(experiments) > 1: raise Sorry("More than one experiment present") experiment = experiments[0] assert(len(reflections) == 1) reflections = reflections[0] intensities = reflections['intensity.sum.value'] variances = reflections['intensity.sum.variance'] if 'intensity.prf.value' in reflections: intensities = reflections['intensity.prf.value'] variances = reflections['intensity.prf.variance'] sel = (variances > 0) intensities = intensities.select(sel) variances = variances.select(sel) sigmas = flex.sqrt(variances) indices = reflections['miller_index'].select(sel) from cctbx import crystal, miller crystal_symmetry = crystal.symmetry( space_group=experiment.crystal.get_space_group(), unit_cell=experiment.crystal.get_unit_cell()) miller_set = miller.set( crystal_symmetry=crystal_symmetry, anomalous_flag=True, indices=indices) miller_array = miller.array( miller_set=miller_set, data=intensities, sigmas=sigmas).set_observation_type_xray_intensity() #miller_array.setup_binner(n_bins=50, reflections_per_bin=100) miller_array.setup_binner(auto_binning=True, n_bins=20) result = miller_array.i_over_sig_i(use_binning=True) result.show() from cctbx import uctbx d_star_sq_centre = result.binner.bin_centers(2) i_over_sig_i = flex.double( [d if d is not None else 0 for d in result.data[1:-1]]) sel = (i_over_sig_i > 0) d_star_sq_centre = d_star_sq_centre.select(sel) i_over_sig_i = i_over_sig_i.select(sel) log_i_over_sig_i = flex.log(i_over_sig_i) weights = result.binner.counts()[1:-1].as_double().select(sel) fit = flex.linear_regression( d_star_sq_centre, log_i_over_sig_i, weights=weights) m = fit.slope() c = fit.y_intercept() import math y_cutoff = math.log(params.i_sigi_cutoff) x_cutoff = (y_cutoff - c)/m estimated_d_min = uctbx.d_star_sq_as_d(x_cutoff) print "estimated d_min: %.2f" %estimated_d_min if params.plot: from matplotlib import pyplot fig = pyplot.figure() ax = fig.add_subplot(1,1,1) ax.plot( list(d_star_sq_centre), list(log_i_over_sig_i), label=r"ln(I/sigI)") ax.plot(pyplot.xlim(), [(m * x + c) for x in pyplot.xlim()], color='red') ax.plot([x_cutoff, x_cutoff], pyplot.ylim(), color='grey', linestyle='dashed') ax.plot(pyplot.xlim(), [y_cutoff, y_cutoff], color='grey', linestyle='dashed') ax.set_xlabel("d_star_sq") ax.set_ylabel("ln(I/sigI)") ax_ = ax.twiny() # ax2 is responsible for "top" axis and "right" axis xticks = ax.get_xticks() xlim = ax.get_xlim() xticks_d = [ uctbx.d_star_sq_as_d(ds2) if ds2 > 0 else 0 for ds2 in xticks ] xticks_ = [ds2/(xlim[1]-xlim[0]) for ds2 in xticks] ax_.set_xticks(xticks) ax_.set_xlim(ax.get_xlim()) ax_.set_xlabel(r"Resolution ($\AA$)") ax_.set_xticklabels(["%.1f" %d for d in xticks_d]) pyplot.savefig("estimate_resolution_limit.png") pyplot.clf()
def paper_test(B, S): from numpy.random import poisson from math import exp background_shape = [1 for i in range(20)] signal_shape = [1 if i >= 6 and i < 15 else 0 for i in range(20)] background = [poisson(bb * B, 1)[0] for bb in background_shape] signal = [poisson(ss * S, 1)[0] for ss in signal_shape] # background = [bb * B for bb in background_shape] # signal = [ss * S for ss in signal_shape] total = [b + s for b, s in zip(background, signal)] # from matplotlib import pylab # pylab.plot(total) # pylab.plot(signal) # pylab.plot(background) # pylab.show() total = [0, 1, 0, 0, 0, 0, 3, 1, 3, 3, 6, 6, 4, 1, 4, 0, 2, 0, 1, 1] total = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] # plot_prob_for_zero(total, background_shape, signal_shape) # signal_shape = [exp(-(x - 10.0)**2 / (2*3.0**2)) for x in range(20)] # signal_shape = [ss / sum(signal_shape) for ss in signal_shape] # print signal_shape # plot_valid(background_shape, signal_shape) B = 155.0 / 296.0 from dials.array_family import flex from math import log, factorial V = flex.double(flex.grid(100, 100)) L = flex.double(flex.grid(100, 100)) DB = flex.double(flex.grid(100, 100)) DS = flex.double(flex.grid(100, 100)) P = flex.double(flex.grid(100, 100)) Fb = sum(background_shape) Fs = sum(signal_shape) SV = [] MASK = flex.bool(flex.grid(100, 100), False) for BB in range(100): for SS in range(100): B = -5.0 + (BB) / 10.0 S = -5.0 + (SS) / 10.0 # SV.append(S) VV = 0 LL = 0 DDB = 0 DDS = 0 for i in range(20): s = signal_shape[i] b = background_shape[i] c = total[i] if B * b + S * s <= 0: # MASK[BB, SS] = True LL = 0 if b == 0: DDB += 0 else: DDB += 1e7 if s == 0: DDS += 0 else: DDS += 1e7 # break else: # VV += (b + s)*c / (B*b + S*s) # LL += c*log(B*b+S*s) - log(factorial(c)) - B*b - S*s DDB += c * b / (B * b + S * s) - b DDS += c * s / (B * b + S * s) - s VV -= Fb + Fs # print B, S, VV # V[BB, SS] = abs(VV) L[BB, SS] = LL DB[BB, SS] = DDB DS[BB, SS] = DDS max_ind = flex.max_index(L) j = max_ind // 100 i = max_ind % 100 print("Approx: ", (j + 1) / 20.0, (i + 1) / 20.0) print("Min/Max DB: ", flex.min(DB), flex.max(DB)) print("Min/Max DS: ", flex.min(DS), flex.max(DS)) from matplotlib import pylab # pylab.imshow(flex.log(V).as_numpy_array(), extent=[0.05, 5.05, 5.05, 0.05]) # pylab.plot(SV, V) # pylab.plot(SV, [0] * 100) # pylab.show() im = flex.exp(L).as_numpy_array() import numpy # im = numpy.ma.masked_array(im, mask=MASK.as_numpy_array()) # pylab.imshow(im)#, extent=[-5.0, 5.0, 5.0, -5.0], origin='lower') pylab.imshow( DB.as_numpy_array(), vmin=-100, vmax=100 ) # , extent=[-5.0, 5.0, 5.0, -5.0], origin='lower') pylab.contour(DB.as_numpy_array(), levels=[0], colors=["red"]) pylab.contour(DS.as_numpy_array(), levels=[0], colors=["black"]) pylab.show() # im = numpy.ma.masked_array(DB.as_numpy_array(), mask=MASK.as_numpy_array()) # pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100) # pylab.show() # im = numpy.ma.masked_array(DS.as_numpy_array(), mask=MASK.as_numpy_array()) # pylab.imshow(im, extent=[-5.0, 5.0, 5.0, -5.0], vmin=-20, vmax=100) # pylab.show() # exit(0) S1, B1 = value(total, background_shape, signal_shape) exit(0) try: S1, B1 = value(total, background_shape, signal_shape) print("Result:") print(S1, B1) exit(0) except Exception as e: raise e import sys import traceback traceback.print_exc() from dials.array_family import flex Fs = sum(signal_shape) Fb = sum(background_shape) Rs = flex.double(flex.grid(100, 100)) print("-----") print(B, S) # from matplotlib import pylab # pylab.plot(total) # pylab.show() print(background_shape) print(signal_shape) print(total) from math import exp, factorial, log minx = -1 miny = -1 minr = 9999 for BB in range(0, 100): for SS in range(0, 100): B = -10 + (BB) / 5.0 S = -10 + (SS) / 5.0 L = 0 Fb2 = 0 Fs2 = 0 for i in range(len(total)): c = total[i] b = background_shape[i] s = signal_shape[i] # P = exp(-(B*b + S*s)) * (B*b+S*s)**c / factorial(c) # print P # if P > 0: # L += log(P) den = B * b + S * s num1 = b * c num2 = s * c if den != 0: Fb2 += num1 / den Fs2 += num2 / den R = (Fb2 - Fb) ** 2 + (Fs2 - Fs) ** 2 if R > 1000: R = 0 # Rs[BB,SS] = L#R#Fs2 - Fs Rs[BB, SS] = R # Fs2 - Fs from matplotlib import pylab pylab.imshow(flex.log(Rs).as_numpy_array(), extent=[-5, 5, 5, -5]) pylab.show() exit(0) exit(0) # print S, B, sum(signal), sum(background), S1, B1 return S, B, sum(signal), sum(background), S1, B1