def jacobian_callable(self, values): # Restore original sigmas refls = self.scaler.ISIGI refls['isigi'] = refls['scaled_intensity'] / refls['original_sigmas'] # Propagate errors from postrefinement self.propagator.error_terms = error_terms.from_x( values.propagate_terms) self.propagator.adjust_errors(dI_derrorterms=self.dI_derrorterms, compute_sums=False) all_sigmas_normalized, sigma_prime = self.get_normalized_sigmas(values) # et: error term df_derrorterms = [] for et, dI_det in zip( values.propagate_terms, self.dI_derrorterms[1:]): # don't need dI wrt iobs dsigmasq_det = 2 * et dsigmasq_detsq = dI_det**2 * dsigmasq_det dsigprimesq_detsq = values.SDFACSQ * dsigmasq_detsq df_detsq = self.df_dpsq(all_sigmas_normalized, sigma_prime, dsigprimesq_detsq) df_derrorterms.append(df_detsq) sdfac_derivatives = super(sdfac_propagate_refinery, self).jacobian_callable(values) return sdfac_derivatives + df_derrorterms
def fvec_callable(self, values): """ Compute the functional by first propagating errors from postrefinement and then applying the current values for the sd parameters to the input data, then computing the complete set of normalized deviations and finally using those normalized deviations to compute the functional.""" # Restore original sigmas refls = self.ISIGI refls['isigi'] = refls['scaled_intensity'] / refls['original_sigmas'] # Propagate errors from postrefinement self.propagator.error_terms = error_terms.from_x( values.propagate_terms) self.propagator.adjust_errors(dI_derrorterms=self.dI_derrorterms, compute_sums=False) # Apply SD terms all_sigmas_normalized, _ = self.get_normalized_sigmas(values) # Compute functional f = flex.double() for i, bin in enumerate(self.bins): binned_normalized_sigmas = all_sigmas_normalized.select(bin) n = len(binned_normalized_sigmas) if n == 0: f.append(0) continue # functional is weight * (1-rms(normalized_sigmas))^s summed over all intensitiy bins f.append(1 - math.sqrt( flex.mean(binned_normalized_sigmas * binned_normalized_sigmas))) return f
def fvec_callable(self, values): """ Compute the functional by first propagating errors from postrefinement and then applying the current values for the sd parameters to the input data, then computing the complete set of normalized deviations and finally using those normalized deviations to compute the functional.""" # Restore original sigmas refls = self.ISIGI refls['isigi'] = refls['scaled_intensity']/refls['original_sigmas'] # Propagate errors from postrefinement self.propagator.error_terms = error_terms.from_x(values.propagate_terms) self.propagator.adjust_errors(dI_derrorterms=self.dI_derrorterms, compute_sums=False) return super(sdfac_propagate_refinery, self).fvec_callable(values)
def adjust_errors(self): print >> self.log, "Starting adjust_errors" # Save original sigmas refls = self.scaler.ISIGI refls['original_sigmas'] = refls['scaled_intensity'] / refls['isigi'] print >> self.log, "Computing initial estimates of parameters" propagator = sdfac_propagate(self.scaler, verbose=False) propagator.initial_estimates() propagator.adjust_errors(compute_sums=False) init_params = flex.double(self.get_initial_sdparams_estimates()) init_params.extend(propagator.error_terms.to_x()) values = self.parameterization(init_params) print >> self.log, "Initial estimates:", values.show(self.log) print >> self.log, "Refining error correction parameters" sels, binned_intensities = self.get_binned_intensities() minimizer = self.run_minimzer(values, sels) values = minimizer.get_refined_params() print >> self.log, "Final", values.show(self.log) print >> self.log, "Applying sdfac/sdb/sdadd 1" # Restore original sigmas refls['isigi'] = refls['scaled_intensity'] / refls['original_sigmas'] # Propagate refined errors from postrefinement propagator.error_terms = error_terms.from_x(values.propagate_terms) propagator.adjust_errors() minimizer.apply_sd_error_params(self.scaler.ISIGI, values) self.scaler.summed_weight = flex.double(self.scaler.n_refl, 0.) self.scaler.summed_wt_I = flex.double(self.scaler.n_refl, 0.) print >> self.log, "Applying sdfac/sdb/sdadd 2" for i in xrange(len(self.scaler.ISIGI)): hkl_id = self.scaler.ISIGI['miller_id'][i] Intensity = self.scaler.ISIGI['scaled_intensity'][ i] # scaled intensity sigma = Intensity / self.scaler.ISIGI['isigi'][ i] # corrected sigma variance = sigma * sigma self.scaler.summed_wt_I[hkl_id] += Intensity / variance self.scaler.summed_weight[hkl_id] += 1 / variance if False: # validate using http://ccp4wiki.org/~ccp4wiki/wiki/index.php?title=Symmetry%2C_Scale%2C_Merge#Analysis_of_Standard_Deviations print >> self.log, "Validating" from matplotlib import pyplot as plt all_sigmas_normalized = compute_normalized_deviations( self.scaler.ISIGI, self.scaler.miller_set.indices()) plt.hist(all_sigmas_normalized, bins=100) plt.figure() binned_rms_normalized_sigmas = [] for i, sel in enumerate(sels): binned_rms_normalized_sigmas.append( math.sqrt( flex.mean( all_sigmas_normalized.select(sel) * all_sigmas_normalized.select(sel)))) plt.plot(binned_intensities, binned_rms_normalized_sigmas, 'o') plt.show() all_sigmas_normalized = all_sigmas_normalized.select( all_sigmas_normalized != 0) self.normal_probability_plot(all_sigmas_normalized, (-0.5, 0.5), plot=True)
def adjust_errors(self): print("Starting adjust_errors", file=self.log) # Save original sigmas refls = self.scaler.ISIGI refls['original_sigmas'] = refls['scaled_intensity'] / refls['isigi'] print("Computing initial estimates of parameters", file=self.log) propagator = sdfac_propagate(self.scaler, verbose=False) propagator.initial_estimates() propagator.adjust_errors(compute_sums=False) init_params = flex.double(self.get_initial_sdparams_estimates()) init_params.extend(propagator.error_terms.to_x()) values = self.parameterization(init_params) print("Initial estimates:", end=' ', file=self.log) values.show(self.log) print("Refining error correction parameters", file=self.log) sels, binned_intensities = self.get_binned_intensities() minimizer = self.run_minimzer(values, sels) values = minimizer.get_refined_params() print("Final", end=' ', file=self.log) values.show(self.log) print("Applying sdfac/sdb/sdadd 1", file=self.log) # Restore original sigmas refls['isigi'] = refls['scaled_intensity'] / refls['original_sigmas'] # Propagate refined errors from postrefinement propagator.error_terms = error_terms.from_x(values.propagate_terms) propagator.adjust_errors() minimizer.apply_sd_error_params(self.scaler.ISIGI, values) self.scaler.summed_weight = flex.double(self.scaler.n_refl, 0.) self.scaler.summed_wt_I = flex.double(self.scaler.n_refl, 0.) print("Applying sdfac/sdb/sdadd 2", file=self.log) for i in range(len(self.scaler.ISIGI)): hkl_id = self.scaler.ISIGI['miller_id'][i] Intensity = self.scaler.ISIGI['scaled_intensity'][ i] # scaled intensity sigma = Intensity / self.scaler.ISIGI['isigi'][ i] # corrected sigma variance = sigma * sigma self.scaler.summed_wt_I[hkl_id] += Intensity / variance self.scaler.summed_weight[hkl_id] += 1 / variance if self.scaler.params.raw_data.error_models.sdfac_refine.plot_refinement_steps: from matplotlib.pyplot import cm from matplotlib import pyplot as plt import numpy as np for i in range(2): f = plt.figure(i) lines = plt.gca().get_lines() color = cm.rainbow(np.linspace(0, 1, len(lines))) for line, c in zip(reversed(lines), color): line.set_color(c) plt.ioff() plt.show() if False: # validate using http://ccp4wiki.org/~ccp4wiki/wiki/index.php?title=Symmetry%2C_Scale%2C_Merge#Analysis_of_Standard_Deviations print("Validating", file=self.log) from matplotlib import pyplot as plt all_sigmas_normalized = self.compute_normalized_deviations( self.scaler.ISIGI, self.scaler.miller_set.indices()) plt.hist(all_sigmas_normalized, bins=100) plt.figure() binned_rms_normalized_sigmas = [] for i, sel in enumerate(sels): binned_rms_normalized_sigmas.append( math.sqrt( flex.mean( all_sigmas_normalized.select(sel) * all_sigmas_normalized.select(sel)))) plt.plot(binned_intensities, binned_rms_normalized_sigmas, 'o') plt.show() all_sigmas_normalized = all_sigmas_normalized.select( all_sigmas_normalized != 0) self.normal_probability_plot(all_sigmas_normalized, (-0.5, 0.5), plot=True)