def run(self): """Execute the script.""" # Parse the command line self.params, _ = self.parser.parse_args(show_diff_phil=True) if self.params.hklin is None: self.parser.print_help() sys.exit() self._extract_data_from_mtz() if self.params.fit_hyperbola: # fit by NLLS Levenberg Marquardt algorithm hyperbola_fit = HyperbolaFit(self.fc, self.fobs) hyperbola_fit.restart() normal_eqns_solving.levenberg_marquardt_iterations( hyperbola_fit, track_all=True, gradient_threshold=1e-8, step_threshold=1e-8, tau=1e-4, n_max_iterations=200, ) intercept = hyperbola_fit.param[0] print( "Model fit described by the formula: |Fo|^2 = sqrt(|Fc|^2 + |Fe|^2)" ) print("where |Fe| = {:.5f}\n".format(sqrt(intercept))) print("Goodness of fit:") gof = hyperbola_fit.goodness_of_fit() print("SSE: {:.5g}".format(gof["SSE"])) print("R-square: {:.5f}".format(gof["R-square"])) print("RMSE: {:.2f}".format(gof["RMSE"])) print() # Set the model_fit function using the determined intercept def hyperbola(x, c): return flex.sqrt(flex.pow2(x) + c) from functools import partial self.model_fit = partial(hyperbola, c=intercept) if self.params.plot_filename: self._plot() return
def __init__(self,x_obs,y_obs,w_obs,initial): self.counter = 0 self.x = initial.deep_copy() self.helper = levenberg_helper(initial_estimates = self.x) self.helper.set_cpp_data(x_obs,y_obs,w_obs) self.helper.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls = self.helper, n_max_iterations = 5000, track_all=True, step_threshold = 0.0001 ) ###### get esd's self.helper.build_up() upper = self.helper.step_equations().normal_matrix_packed_u() nm_elem = flex.double(25) self.c = flex.double(5) ctr = 0 for x in xrange(5): x_0 = ctr for y in xrange(4,x-1,-1): nm_elem[ 5*x+y ] = upper[x_0+(y-x)] ctr += 1 if x!= y: nm_elem[ 5*y+x ] = upper[x_0+(y-x)] else: self.c[x]=upper[x_0+(y-x)] NM = sqr(nm_elem) self.helper.solve() #print list(self.helper.step_equations().cholesky_factor_packed_u()) error_matrix = NM.inverse() self.error_diagonal = [error_matrix(a,a) for a in xrange(5)] print "End of minimization: Converged", self.helper.counter,"cycles"
def __init__(self,x_obs,y_obs,w_obs,initial): self.counter = 0 self.x = initial.deep_copy() self.helper = levenberg_helper(initial_estimates = self.x) self.helper.set_cpp_data(x_obs,y_obs,w_obs) self.helper.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls = self.helper, n_max_iterations = 5000, track_all=True, step_threshold = 0.0001 ) ###### get esd's self.helper.build_up() upper = self.helper.step_equations().normal_matrix_packed_u() nm_elem = flex.double(25) self.c = flex.double(5) ctr = 0 for x in xrange(5): x_0 = ctr for y in xrange(4,x-1,-1): nm_elem[ 5*x+y ] = upper[x_0+(y-x)] ctr += 1 if x!= y: nm_elem[ 5*y+x ] = upper[x_0+(y-x)] else: self.c[x]=upper[x_0+(y-x)] NM = sqr(nm_elem) self.helper.solve() #print list(self.helper.step_equations().cholesky_factor_packed_u()) error_matrix = NM.inverse() self.error_diagonal = [error_matrix(a,a) for a in xrange(5)] print "End of minimization: Converged", self.helper.counter,"cycles"
def exercise(self, fixed_twin_fraction): # Create a shaken structure xs ready for refinement xs0 = self.structure emma_ref = xs0.as_emma_model() xs = xs0.deep_copy_scatterers() xs.shake_sites_in_place(rms_difference=0.15) xs.shake_adp() for sc in xs.scatterers(): sc.flags.set_use_u_iso(False).set_use_u_aniso(True) sc.flags.set_grad_site(True).set_grad_u_aniso(True) # Setup L.S. problem connectivity_table = smtbx.utils.connectivity_table(xs) shaken_twin_fraction = (self.twin_fraction if fixed_twin_fraction else self.twin_fraction + 0.1 * flex.random_double()) # 2nd domain in __init__ twin_components = (xray.twin_component(twin_law=self.twin_law.r(), value=shaken_twin_fraction, grad=not fixed_twin_fraction), ) reparametrisation = constraints.reparametrisation( structure=xs, constraints=[], connectivity_table=connectivity_table, twin_fractions=twin_components) obs = self.fo_sq.as_xray_observations(twin_components=twin_components) ls = least_squares.crystallographic_ls( obs, reparametrisation, weighting_scheme=least_squares.unit_weighting(), origin_fixing_restraints_type=origin_fixing_restraints. atomic_number_weighting) # Refine till we get back the original structure (so we hope) cycles = normal_eqns_solving.levenberg_marquardt_iterations( ls, gradient_threshold=1e-12, step_threshold=1e-6, track_all=True) # Now let's start to check it all worked assert ls.n_parameters == 63 if fixed_twin_fraction else 64 match = emma.model_matches(emma_ref, xs.as_emma_model()).refined_matches[0] assert match.rt.r == matrix.identity(3) for pair in match.pairs: assert approx_equal(match.calculate_shortest_dist(pair), 0, eps=1e-4), pair if fixed_twin_fraction: assert ls.twin_fractions[0].value == self.twin_fraction else: assert approx_equal(ls.twin_fractions[0].value, self.twin_fraction, eps=1e-2) assert approx_equal(ls.scale_factor(), 1, eps=1e-5) assert approx_equal(ls.objective(), 0)
def exercise_levenberg_marquardt(non_linear_ls): non_linear_ls.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls, track_all=True, gradient_threshold=1e-12, step_threshold=1e-12, tau=1e-8, n_max_iterations=200) print "L-M: %i iterations" % iterations.n_iterations
def exercise(self, fixed_twin_fraction): # Create a shaken structure xs ready for refinement xs0 = self.structure emma_ref = xs0.as_emma_model() xs = xs0.deep_copy_scatterers() xs.shake_sites_in_place(rms_difference=0.15) xs.shake_adp() for sc in xs.scatterers(): sc.flags.set_use_u_iso(False).set_use_u_aniso(True) sc.flags.set_grad_site(True).set_grad_u_aniso(True) # Setup L.S. problem connectivity_table = smtbx.utils.connectivity_table(xs) shaken_twin_fraction = ( self.twin_fraction if fixed_twin_fraction else self.twin_fraction + 0.1*flex.random_double()) # 2nd domain in __init__ twin_components = (xray.twin_component( twin_law=self.twin_law.r(), value=shaken_twin_fraction, grad=not fixed_twin_fraction),) reparametrisation = constraints.reparametrisation( structure=xs, constraints=[], connectivity_table=connectivity_table, twin_fractions=twin_components) obs = self.fo_sq.as_xray_observations(twin_components=twin_components) ls = least_squares.crystallographic_ls( obs, reparametrisation, weighting_scheme=least_squares.unit_weighting(), origin_fixing_restraints_type= origin_fixing_restraints.atomic_number_weighting) # Refine till we get back the original structure (so we hope) cycles = normal_eqns_solving.levenberg_marquardt_iterations( ls, gradient_threshold=1e-12, step_threshold=1e-6, track_all=True) # Now let's start to check it all worked assert ls.n_parameters == 63 if fixed_twin_fraction else 64 match = emma.model_matches(emma_ref, xs.as_emma_model()).refined_matches[0] assert match.rt.r == matrix.identity(3) for pair in match.pairs: assert approx_equal(match.calculate_shortest_dist(pair), 0, eps=1e-4), pair if fixed_twin_fraction: assert ls.twin_fractions[0].value == self.twin_fraction else: assert approx_equal(ls.twin_fractions[0].value, self.twin_fraction, eps=1e-2) assert approx_equal(ls.scale_factor(), 1, eps=1e-5) assert approx_equal(ls.objective(), 0)
def exercise_levenberg_marquardt(non_linear_ls): non_linear_ls.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls, track_all=True, gradient_threshold=1e-08, step_threshold=1e-08, tau=1e-08, n_max_iterations=200) # from IPython import embed; embed(); exit() print "L-M: %i iterations" % iterations.n_iterations
def run_smtbx_ls(mode, cod_id, i_obs, f_obs, xray_structure, params): import smtbx.refinement fo_sq = i_obs assert fo_sq.sigmas() is not None sel = (fo_sq.data() == 0) & (fo_sq.sigmas() == 0) fo_sq = fo_sq.select(~sel) fo_sq.select(fo_sq.sigmas() <= 0).show_array() assert fo_sq.sigmas().all_gt(0) if (1): # work around bug currently in smtbx weighting scheme implementation fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.data().size(), 1)) xobs = fo_sq.as_xray_observations() tm = user_plus_sys_time() rm = smtbx.refinement.model( fo_sq=xobs, xray_structure=xray_structure, constraints=[], restraints_manager=smtbx.refinement.restraints.manager(), weighting_scheme=smtbx.refinement.least_squares.unit_weighting()) ls = rm.least_squares() if (mode == "simple"): for i_cycle in range(params.ls_simple_iterations): ls.build_up() try: ls.solve_and_step_forward() except RuntimeError as e: if (str(e).find("cholesky.failure") <= 0): raise print('Aborting run_smtbx_ls("simple"): cholesky.failure: %s' \ % cod_id) break for sc in xray_structure.scatterers(): if (sc.u_iso <= 0 or sc.u_iso > 1): sc.u_iso = 0.05 show_cc_r1(params, "ls%02d" % (i_cycle+1), f_obs, xray_structure) tm.show_elapsed(prefix="time smtbx_ls_simple_iterations: ") elif (mode == "lm"): from scitbx.lstbx import normal_eqns_solving thresh = 1e-6 try: cycles = normal_eqns_solving.levenberg_marquardt_iterations( ls, gradient_threshold=thresh, step_threshold=thresh, tau=1e-7) except RuntimeError as e: if (not str(e).startswith( "cctbx::adptbx::debye_waller_factor_exp: arg_limit exceeded")): raise print('Aborting run_smtbx_ls("lm"):' \ ' debye_waller_factor_exp failure: %s' % cod_id) show_cc_r1(params, "smtbx_lm", f_obs, xray_structure) tm.show_elapsed(prefix="time levenberg_marquardt_iterations: ") else: raise RuntimeError('Unknown run_smtbx_ls(mode="%s")' % mode)
def exercise_levenberg_marquardt(non_linear_ls): non_linear_ls.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls, track_all=True, gradient_threshold=1e-8, step_threshold=1e-8, tau=1e-4, n_max_iterations=200) assert non_linear_ls.n_equations == non_linear_ls.n_data assert approx_equal(non_linear_ls.x, non_linear_ls.arg_min, eps=5e-4) print "L-M: %i iterations" % iterations.n_iterations
def run_minimzer(self, values, sels, **kwargs): self.refinery = sdfac_refinery(self.scaler, self, self.scaler.miller_set.indices(), sels, self.log) self.helper = sdfac_helper(current_x=values.reference, parameterization=self.parameterization, refinery=self.refinery, out=self.log) self.iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls=self.helper, track_all=True, gradient_threshold=1e-08, step_threshold=1e-08, tau=1e-08, n_max_iterations=200) return self
def run_minimzer(self, values, sels, **kwargs): from xfel.merging.algorithms.error_model.sdfac_refine_levmar import sdfac_helper from scitbx.lstbx import normal_eqns_solving self.refinery = sdfac_propagate_refinery(self.scaler, self, self.scaler.miller_set.indices(), sels, self.log) self.helper = sdfac_helper(current_x = values.reference, parameterization = self.parameterization, refinery = self.refinery, out = self.log ) self.iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls = self.helper, track_all=True, gradient_threshold=1e-08, step_threshold=1e-08, tau=1e-08, n_max_iterations=200) return self
def fit_3_gaussians(self, histogram): fitted_gaussians = [] low_idx = self.work_params.fit_limits[0] high_idx = self.work_params.fit_limits[1] slot_centers = flex.double( range(self.work_params.first_slot_value, self.work_params.first_slot_value + len(histogram))) free_x = slot_centers[low_idx:high_idx] slots = flex.double(histogram.astype(np.float64)) free_y = slots[low_idx:high_idx] total_population = flex.sum(free_y) # zero_mean = 0. # originally intended mean=0 maxidx = flex.max_index( free_y) # but if dark subtraction (pedstal correction) is off zero_mean = free_x[maxidx] # use this non-zero maximum instead zero_amplitude = flex.max(free_y) assert 1. / zero_amplitude #guard against division by zero zero_sigma = self.work_params.gaussian_3.zero_sigma inelastic_amplitude = 0.001 elastic_amplitude = 0.001 elastic_sigma = self.work_params.gaussian_3.zero_sigma #elastic_mean = zero_mean+ zero_sigma*self.work_params.gaussian_3.elastic_gain_to_sigma #** helper = self.helper_3_gaussian_factory( initial_estimates=(zero_mean, 1.0, zero_sigma, inelastic_amplitude, elastic_amplitude, elastic_sigma), constants=flex.double(( self.work_params.gaussian_3.inelastic_gain_to_sigma, self.work_params.gaussian_3.elastic_gain_to_sigma, )), free_x=free_x, free_y=free_y / zero_amplitude ) # put y values on 0->1 scale for normal eqn solving helper.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls=helper, n_max_iterations=7, gradient_threshold=1.E-3) fitted_gaussians = helper.as_gaussians() for item in fitted_gaussians: item.params = (item.params[0] * zero_amplitude, item.params[1], item.params[2]) # convert back to full scale return fitted_gaussians
def exercise_levenberg_marquardt(non_linear_ls, plot=False): non_linear_ls.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls, track_all=True, gradient_threshold=1e-8, step_threshold=1e-8, tau=1e-4, n_max_iterations=200) assert non_linear_ls.n_equations == non_linear_ls.n_data assert approx_equal(non_linear_ls.x, non_linear_ls.arg_min, eps=5e-4) print "L-M: %i iterations" % iterations.n_iterations if plot: f = open('plot.nb', 'w') print >>f, "g=%s;" % iterations.gradient_norm_history.mathematica_form() print >>f, "\[Mu]=%s;" % iterations.mu_history.mathematica_form() print >>f, "ListLogPlot[{g,\[Mu]},Joined->True]" f.close()
def exercise_levenberg_marquardt(non_linear_ls, plot=False): non_linear_ls.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls, track_all=True, gradient_threshold=1e-8, step_threshold=1e-8, tau=1e-4, n_max_iterations=200) assert non_linear_ls.n_equations == non_linear_ls.n_data assert approx_equal(non_linear_ls.x, non_linear_ls.arg_min, eps=5e-4) print("L-M: %i iterations" % iterations.n_iterations) if plot: f = open('plot.nb', 'w') print("g=%s;" % iterations.gradient_norm_history.mathematica_form(), file=f) print("\[Mu]=%s;" % iterations.mu_history.mathematica_form(), file=f) print("ListLogPlot[{g,\[Mu]},Joined->True]", file=f) f.close()
def fit_3_gaussians(self,histogram): fitted_gaussians = [] low_idx = self.work_params.fit_limits[0] high_idx = self.work_params.fit_limits[1] slot_centers = flex.double(xrange(self.work_params.first_slot_value, self.work_params.first_slot_value + len(histogram))) free_x = slot_centers[low_idx:high_idx] slots = flex.double(histogram.astype(np.float64)) free_y = slots[low_idx:high_idx] total_population = flex.sum(free_y) # zero_mean = 0. # originally intended mean=0 maxidx = flex.max_index(free_y) # but if dark subtraction (pedstal correction) is off zero_mean = free_x[maxidx] # use this non-zero maximum instead zero_amplitude = flex.max(free_y) assert 1./zero_amplitude #guard against division by zero zero_sigma = self.work_params.gaussian_3.zero_sigma inelastic_amplitude = 0.001 elastic_amplitude = 0.001 elastic_sigma = self.work_params.gaussian_3.zero_sigma #elastic_mean = zero_mean+ zero_sigma*self.work_params.gaussian_3.elastic_gain_to_sigma #** helper = self.helper_3_gaussian_factory(initial_estimates = (zero_mean, 1.0, zero_sigma, inelastic_amplitude, elastic_amplitude, elastic_sigma), constants=flex.double((self.work_params.gaussian_3.inelastic_gain_to_sigma, self.work_params.gaussian_3.elastic_gain_to_sigma, )), free_x = free_x, free_y = free_y/zero_amplitude) # put y values on 0->1 scale for normal eqn solving helper.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls = helper, n_max_iterations = 7, gradient_threshold = 1.E-3) fitted_gaussians = helper.as_gaussians() for item in fitted_gaussians: item.params = (item.params[0] * zero_amplitude, item.params[1], item.params[2]) # convert back to full scale return fitted_gaussians
def fit_one_histogram_two_gaussians(self, histogram): fitted_gaussians = [] GAIN_TO_SIGMA = self.work_params.fudge_factor.gain_to_sigma low_idx = self.work_params.fit_limits[0] high_idx = self.work_params.fit_limits[1] slot_centers = flex.double( range(self.work_params.first_slot_value, self.work_params.first_slot_value + len(histogram))) free_x = slot_centers[low_idx:high_idx] #print list(free_x) slots = flex.double(histogram.astype(np.float64)) free_y = slots[low_idx:high_idx] # zero_mean = 0. # originally intended mean=0 maxidx = flex.max_index( free_y) # but if dark subtraction (pedstal correction) is off zero_mean = free_x[maxidx] # use this non-zero maximum instead zero_amplitude = flex.max(free_y) assert 1. / zero_amplitude #guard against division by zero total_population = flex.sum(free_y) zero_sigma = self.work_params.estimated_gain / GAIN_TO_SIGMA one_amplitude = 0.001 helper = self.per_pixel_helper_factory( initial_estimates=(zero_mean, 1.0, zero_sigma, one_amplitude), GAIN_TO_SIGMA=GAIN_TO_SIGMA, free_x=free_x, free_y=free_y / zero_amplitude ) # put y values on 0->1 scale for normal eqn solving helper.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls=helper, n_max_iterations=7, gradient_threshold=1.E-3) print("current values after iterations", list(helper.x), end=' ') fitted_gaussians = helper.as_gaussians() for item in fitted_gaussians: item.params = (item.params[0] * zero_amplitude, item.params[1], item.params[2]) # convert back to full scale return fitted_gaussians
def run(): import libtbx.utils libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser().option( None, "--normal_eqns_solving_method", default='naive').option(None, "--fix_random_seeds", action='store_true', default='naive')).process(args=sys.argv[1:]) opts = command_line.options if opts.fix_random_seeds: import random random.seed(1) flex.set_random_seed(1) gradient_threshold = 1e-8 step_threshold = 1e-8 if opts.normal_eqns_solving_method == 'naive': m = lambda eqns: normal_eqns_solving.naive_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=step_threshold) elif opts.normal_eqns_solving_method == 'levenberg-marquardt': m = lambda eqns: normal_eqns_solving.levenberg_marquardt_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=gradient_threshold, tau=1e-7) else: raise RuntimeError("Unknown method %s" % opts.normal_eqns_solving_method) for t in [ saturated_test_case(m), sucrose_test_case(m), symmetry_equivalent_test_case(m), fpfdp_test_case(m), constrained_fpfdp_test_case(m), scalar_scaled_adp_test_case(m), ]: t.run()
def fit_one_histogram_two_gaussians(self,histogram): fitted_gaussians = [] GAIN_TO_SIGMA = self.work_params.fudge_factor.gain_to_sigma low_idx = self.work_params.fit_limits[0] high_idx = self.work_params.fit_limits[1] slot_centers = flex.double(xrange(self.work_params.first_slot_value, self.work_params.first_slot_value + len(histogram))) free_x = slot_centers[low_idx:high_idx] #print list(free_x) slots = flex.double(histogram.astype(np.float64)) free_y = slots[low_idx:high_idx] # zero_mean = 0. # originally intended mean=0 maxidx = flex.max_index(free_y) # but if dark subtraction (pedstal correction) is off zero_mean = free_x[maxidx] # use this non-zero maximum instead zero_amplitude = flex.max(free_y) assert 1./zero_amplitude #guard against division by zero total_population = flex.sum(free_y) zero_sigma = self.work_params.estimated_gain / GAIN_TO_SIGMA one_amplitude = 0.001 helper = self.per_pixel_helper_factory(initial_estimates = (zero_mean, 1.0, zero_sigma, one_amplitude), GAIN_TO_SIGMA=GAIN_TO_SIGMA, free_x = free_x, free_y = free_y/zero_amplitude) # put y values on 0->1 scale for normal eqn solving helper.restart() iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls = helper, n_max_iterations = 7, gradient_threshold = 1.E-3) print "current values after iterations", list(helper.x), fitted_gaussians = helper.as_gaussians() for item in fitted_gaussians: item.params = (item.params[0] * zero_amplitude, item.params[1], item.params[2]) # convert back to full scale return fitted_gaussians
def run(): import libtbx.utils libtbx.utils.show_times_at_exit() import sys from libtbx.option_parser import option_parser command_line = (option_parser() .option(None, "--normal_eqns_solving_method", default='naive') .option(None, "--fix_random_seeds", action='store_true', default='naive') ).process(args=sys.argv[1:]) opts = command_line.options if opts.fix_random_seeds: import random random.seed(1) flex.set_random_seed(1) gradient_threshold=1e-8 step_threshold=1e-8 if opts.normal_eqns_solving_method == 'naive': m = lambda eqns: normal_eqns_solving.naive_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=step_threshold) elif opts.normal_eqns_solving_method == 'levenberg-marquardt': m = lambda eqns: normal_eqns_solving.levenberg_marquardt_iterations( eqns, gradient_threshold=gradient_threshold, step_threshold=gradient_threshold, tau=1e-7) else: raise RuntimeError("Unknown method %s" % opts.normal_eqns_solving_method) for t in [ saturated_test_case(m), sucrose_test_case(m), symmetry_equivalent_test_case(m), ]: t.run()
def run(args): processed = iotbx.phil.process_command_line( args=args, master_string=master_phil_str) args = processed.remaining_args work_params = processed.work.extract().xscale processed.work.show() assert len(args) == 1 observations=args[0] choose_method=work_params.method assert choose_method in ("G_only", "G_cycles", "G_wI", "G_wI_LM", "test_G_wI_LM_C++") if choose_method=="G_only": method=1 if choose_method=="G_cycles": method=2 if choose_method=="G_wI": method=3 if choose_method=="G_wI_LM": method=4 # if choose_method=="test_G_wI_LM_C++": method=5 eps = work_params.eps ref_G = work_params.ref_G ref_I = work_params.ref_I cycles = work_params.n_cycles mi=None f=open(observations) (I, W, H, Miller_list, F)=read_cxi_merge(f) t1=time.time() if method==1: from cxi_xdr_xes.XScale.minimizers.scaling_minimizer_G import xscale print "Loading objects......." n_frames = max(F)+1 (MI, Fr) = Load_object(I, W, H, Miller_list, F, n_frames) fit=xscale(MI, Fr, n_frames, eps) Gm=fit.x (Ih,mi)=read_objects(MI) if method==2: from cxi_xdr_xes.XScale.minimizers.scaling_minimizer_G_cycles import xscale N_miller=max(H)+1 Gm=flex.double(max(F)+1,1) t1=time.time() Ih=get_Ih(H, I, W, F, Gm, N_miller) for i in range(cycles): fit=xscale(I, W, H, F, flex.double(Gm), flex.double(Ih), eps) Gm=fit.x Ih=get_Ih(H, I, W, F, Gm, N_miller) if method==3: from cxi_xdr_xes.XScale.minimizers.scaling_minimizer_wI_G import xscale N_miller=max(H)+1 G=flex.double(max(F)+1,1) Ih=get_Ih(H, I, W, F, G, N_miller) fit=xscale(I, W, H, F, flex.double(G), flex.double(Ih), eps) Ih=fit.x[len(G):]*flex.double(Ih) Gm=fit.x[:len(G)] if method==4: from scitbx.lstbx import normal_eqns_solving from cxi_xdr_xes.XScale.minimizers.scaling_LM_minimizer_wI_G_cpp import xscale N_miller=max(H)+1 G=flex.double(max(F)+1,1) Ih=get_Ih(H, I, W, F, G, N_miller) fit=xscale(I, W, H, F, G, Ih) iterations = normal_eqns_solving.levenberg_marquardt_iterations( non_linear_ls = fit, gradient_threshold = eps) Ih=fit.x[len(G):]*flex.double(Ih) Gm=fit.x[:len(G)] print "Total time", time.time()-t1 if ref_G or ref_I: correlation(ref_G, ref_I, Gm, Ih, mi, False)
def exercise(self): xs0 = self.structure xs = xs0.deep_copy_scatterers() k1, s1, li1, o1, o2 = xs.scatterers() self.shake_point_group_3(k1) self.shake_point_group_3(s1) self.shake_point_group_3(li1) self.shake_point_group_3(o1) o2.site = tuple( [ x*(1 + random.uniform(-self.delta_site, self.delta_site)) for x in o2.site]) o2.u_star = tuple( [ u*(1 + random.uniform(-self.delta_u_star, self.delta_u_star)) for u in o2.u_star]) for sc in xs.scatterers(): sc.flags.set_use_u_iso(False).set_use_u_aniso(True) sc.flags.set_grad_site(True).set_grad_u_aniso(True) connectivity_table = smtbx.utils.connectivity_table(xs) reparametrisation = constraints.reparametrisation( structure=xs, constraints=[], connectivity_table=connectivity_table) ls = least_squares.crystallographic_ls( self.fo_sq.as_xray_observations(), reparametrisation, weighting_scheme=least_squares.unit_weighting(), origin_fixing_restraints_type= origin_fixing_restraints.atomic_number_weighting) cycles = normal_eqns_solving.levenberg_marquardt_iterations( ls, gradient_threshold=1e-12, step_threshold=1e-7, track_all=True) ## Test whether refinement brought back the shaked structure to its ## original state match = emma.model_matches(xs0.as_emma_model(), xs.as_emma_model()).refined_matches[0] assert match.rt.r == matrix.identity(3) assert not match.singles1 and not match.singles2 assert match.rms < 1e-6 delta_u_carts= ( xs.scatterers().extract_u_cart(xs.unit_cell()) - xs0.scatterers().extract_u_cart(xs.unit_cell())).norms() assert flex.abs(delta_u_carts) < 1e-6 assert approx_equal(ls.scale_factor(), 1, eps=1e-4) ## Test covariance matrix jac_tr = reparametrisation.jacobian_transpose_matching_grad_fc() cov = ls.covariance_matrix( jacobian_transpose=jac_tr, normalised_by_goof=False)\ .matrix_packed_u_as_symmetric() m, n = cov.accessor().focus() # x,y for point group 3 sites are fixed: no variance or correlation for i in (0, 9, 18, 27,): assert cov.matrix_copy_block(i, 0, 2, n) == 0 # u_star coefficients u13 and u23 for point group 3 sites are fixed # to 0: again no variance or correlation with any other param for i in (7, 16, 25, 34,): assert cov.matrix_copy_block(i, 0, 2, n).as_1d()\ .all_approx_equal(0., 1e-20) # u_star coefficients u11, u22 and u12 for point group 3 sites # are totally correlated, with variances in ratios 1:1:1/2 for i in (3, 12, 21, 30,): assert cov[i, i] != 0 assert approx_equal(cov[i, i], cov[i+1, i+1], eps=1e-15) assert approx_equal(cov[i, i+1]/cov[i, i], 1, eps=1e-12) assert approx_equal(cov[i, i+3]/cov[i, i], 0.5, eps=1e-12)
if (str(e).find("cholesky.failure") <= 0): raise print 'Aborting run_smtbx_ls("simple"): cholesky.failure: %s' \ % cod_id break for sc in xray_structure.scatterers(): if (sc.u_iso <= 0 or sc.u_iso > 1): sc.u_iso = 0.05 show_cc_r1(params, "ls%02d" % (i_cycle+1), f_obs, xray_structure) tm.show_elapsed(prefix="time smtbx_ls_simple_iterations: ") elif (mode == "lm"): from scitbx.lstbx import normal_eqns_solving thresh = 1e-6 try: cycles = normal_eqns_solving.levenberg_marquardt_iterations( ls, gradient_threshold=thresh, step_threshold=thresh, tau=1e-7) except RuntimeError, e: if (not str(e).startswith( "cctbx::adptbx::debye_waller_factor_exp: arg_limit exceeded")): raise print 'Aborting run_smtbx_ls("lm"):' \ ' debye_waller_factor_exp failure: %s' % cod_id show_cc_r1(params, "smtbx_lm", f_obs, xray_structure) tm.show_elapsed(prefix="time levenberg_marquardt_iterations: ") else: raise RuntimeError('Unknown run_smtbx_ls(mode="%s")' % mode) def remove_tmp_files(file_names): for fn in file_names:
yc_i = self.model(self.x, t[i], only_value=True) residuals[i] = yc_i - self.yo[i] self.add_residuals(residuals, weights=None) else: # each row = derivatives of one L.S. term derivatives = flex.double(flex.grid(m, n)) for i in xrange(m): yc_i, der_yc_i = self.model(self.x, t[i], only_value=False) residuals[i] = yc_i - self.yo[i] for j in xrange(n): derivatives[i, j] = der_yc_i[j] self.add_equations(residuals, derivatives, weights=None) test = my_fit(t, yo, model, flex.double(x_0)) runs = normal_eqns_solving.levenberg_marquardt_iterations( test, track_all=True, gradient_threshold=1e-8, step_threshold=1e-8, tau=1e-4, n_max_iterations=200) # other choices:` # - naive_iterations # - naive_iterations_with_damping # - naive_iterations_with_damping_and_shift_limit print "#iterations: {}".format(runs.n_iterations) print "Fitted parameters: {}".format(tuple(test.x)) print print 'OK'
def exercise_restrained_refinement(options): import random random.seed(1) flex.set_random_seed(1) xs0 = smtbx.development.random_xray_structure( sgtbx.space_group_info('P1'), n_scatterers=options.n_scatterers, elements="random") for sc in xs0.scatterers(): sc.flags.set_grad_site(True) sc0 = xs0.scatterers() uc = xs0.unit_cell() mi = xs0.build_miller_set(anomalous_flag=False, d_min=options.resolution) fo_sq = mi.structure_factors_from_scatterers( xs0, algorithm="direct").f_calc().norm() fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1)) i, j, k, l = random.sample(xrange(options.n_scatterers), 4) bond_proxies = geometry_restraints.shared_bond_simple_proxy() w = 1e9 d_ij = uc.distance(sc0[i].site, sc0[j].site) * 0.8 bond_proxies.append( geom.bond_simple_proxy(i_seqs=(i, j), distance_ideal=d_ij, weight=w)) d_jk = uc.distance(sc0[j].site, sc0[k].site) * 0.85 bond_proxies.append( geom.bond_simple_proxy(i_seqs=(j, k), distance_ideal=d_jk, weight=w)) d_ki = min( uc.distance(sc0[k].site, sc0[i].site) * 0.9, (d_ij + d_jk) * 0.8) bond_proxies.append( geom.bond_simple_proxy(i_seqs=(k, i), distance_ideal=d_ki, weight=w)) d_jl = uc.distance(sc0[j].site, sc0[l].site) * 0.9 bond_proxies.append( geom.bond_simple_proxy(i_seqs=(j, l), distance_ideal=d_jl, weight=w)) d_lk = min( uc.distance(sc0[l].site, sc0[k].site) * 0.8, 0.75 * (d_jk + d_jl)) bond_proxies.append( geom.bond_simple_proxy(i_seqs=(l, k), distance_ideal=d_jl, weight=w)) restraints_manager = restraints.manager(bond_proxies=bond_proxies) xs1 = xs0.deep_copy_scatterers() xs1.shake_sites_in_place(rms_difference=0.1) def ls_problem(): xs = xs1.deep_copy_scatterers() reparametrisation = constraints.reparametrisation( structure=xs, constraints=[], connectivity_table=smtbx.utils.connectivity_table(xs), temperature=20) return least_squares.crystallographic_ls( fo_sq.as_xray_observations(), reparametrisation=reparametrisation, restraints_manager=restraints_manager) gradient_threshold, step_threshold = 1e-6, 1e-6 eps = 5e-3 ls = ls_problem() t = wall_clock_time() cycles = normal_eqns_solving.naive_iterations( ls, gradient_threshold=gradient_threshold, step_threshold=step_threshold, track_all=True) if options.verbose: print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed()) sc = ls.xray_structure.scatterers() for p in bond_proxies: d = uc.distance(*[sc[i_pair].site for i_pair in p.i_seqs]) assert approx_equal(d, p.distance_ideal, eps) ls = ls_problem() t = wall_clock_time() cycles = normal_eqns_solving.levenberg_marquardt_iterations( ls, gradient_threshold=gradient_threshold, step_threshold=step_threshold, tau=1e-3, track_all=True) if options.verbose: print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed()) sc = ls.xray_structure.scatterers() sc = ls.xray_structure.scatterers() for p in bond_proxies: d = uc.distance(*[sc[i].site for i in p.i_seqs]) assert approx_equal(d, p.distance_ideal, eps)
ls.solve_and_step_forward() except RuntimeError, e: if (str(e).find("cholesky.failure") <= 0): raise print 'Aborting run_smtbx_ls("simple"): cholesky.failure: %s' \ % cod_id break for sc in xray_structure.scatterers(): if (sc.u_iso <= 0 or sc.u_iso > 1): sc.u_iso = 0.05 show_cc_r1(params, "ls%02d" % (i_cycle + 1), f_obs, xray_structure) tm.show_elapsed(prefix="time smtbx_ls_simple_iterations: ") elif (mode == "lm"): from scitbx.lstbx import normal_eqns_solving thresh = 1e-6 try: cycles = normal_eqns_solving.levenberg_marquardt_iterations( ls, gradient_threshold=thresh, step_threshold=thresh, tau=1e-7) except RuntimeError, e: if (not str(e).startswith( "cctbx::adptbx::debye_waller_factor_exp: arg_limit exceeded" )): raise print 'Aborting run_smtbx_ls("lm"):' \ ' debye_waller_factor_exp failure: %s' % cod_id show_cc_r1(params, "smtbx_lm", f_obs, xray_structure) tm.show_elapsed(prefix="time levenberg_marquardt_iterations: ") else: raise RuntimeError('Unknown run_smtbx_ls(mode="%s")' % mode) def remove_tmp_files(file_names): for fn in file_names:
def exercise_restrained_refinement(options): import random random.seed(1) flex.set_random_seed(1) xs0 = smtbx.development.random_xray_structure( sgtbx.space_group_info('P1'), n_scatterers=options.n_scatterers, elements="random") for sc in xs0.scatterers(): sc.flags.set_grad_site(True) sc0 = xs0.scatterers() uc = xs0.unit_cell() mi = xs0.build_miller_set(anomalous_flag=False, d_min=options.resolution) fo_sq = mi.structure_factors_from_scatterers( xs0, algorithm="direct").f_calc().norm() fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1)) i, j, k, l = random.sample(xrange(options.n_scatterers), 4) bond_proxies = geometry_restraints.shared_bond_simple_proxy() w = 1e9 d_ij = uc.distance(sc0[i].site, sc0[j].site)*0.8 bond_proxies.append(geom.bond_simple_proxy( i_seqs=(i, j), distance_ideal=d_ij, weight=w)) d_jk = uc.distance(sc0[j].site, sc0[k].site)*0.85 bond_proxies.append(geom.bond_simple_proxy( i_seqs=(j, k), distance_ideal=d_jk, weight=w)) d_ki = min(uc.distance(sc0[k].site, sc0[i].site)*0.9, (d_ij + d_jk)*0.8) bond_proxies.append(geom.bond_simple_proxy( i_seqs=(k, i), distance_ideal=d_ki, weight=w)) d_jl = uc.distance(sc0[j].site, sc0[l].site)*0.9 bond_proxies.append(geom.bond_simple_proxy( i_seqs=(j, l), distance_ideal=d_jl, weight=w)) d_lk = min(uc.distance(sc0[l].site, sc0[k].site)*0.8, 0.75*(d_jk + d_jl)) bond_proxies.append(geom.bond_simple_proxy( i_seqs=(l, k), distance_ideal=d_jl, weight=w)) restraints_manager = restraints.manager(bond_proxies=bond_proxies) xs1 = xs0.deep_copy_scatterers() xs1.shake_sites_in_place(rms_difference=0.1) def ls_problem(): xs = xs1.deep_copy_scatterers() reparametrisation = constraints.reparametrisation( structure=xs, constraints=[], connectivity_table=smtbx.utils.connectivity_table(xs), temperature=20) return least_squares.crystallographic_ls( fo_sq.as_xray_observations(), reparametrisation=reparametrisation, restraints_manager=restraints_manager) gradient_threshold, step_threshold = 1e-6, 1e-6 eps = 5e-3 ls = ls_problem() t = wall_clock_time() cycles = normal_eqns_solving.naive_iterations( ls, gradient_threshold=gradient_threshold, step_threshold=step_threshold, track_all=True) if options.verbose: print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed()) sc = ls.xray_structure.scatterers() for p in bond_proxies: d = uc.distance(*[ sc[i_pair].site for i_pair in p.i_seqs ]) assert approx_equal(d, p.distance_ideal, eps) ls = ls_problem() t = wall_clock_time() cycles = normal_eqns_solving.levenberg_marquardt_iterations( ls, gradient_threshold=gradient_threshold, step_threshold=step_threshold, tau=1e-3, track_all=True) if options.verbose: print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed()) sc = ls.xray_structure.scatterers() sc = ls.xray_structure.scatterers() for p in bond_proxies: d = uc.distance(*[ sc[i].site for i in p.i_seqs ]) assert approx_equal(d, p.distance_ideal, eps)
def exercise(self): xs0 = self.structure xs = xs0.deep_copy_scatterers() k1, s1, li1, o1, o2 = xs.scatterers() self.shake_point_group_3(k1) self.shake_point_group_3(s1) self.shake_point_group_3(li1) self.shake_point_group_3(o1) o2.site = tuple( [ x*(1 + random.uniform(-self.delta_site, self.delta_site)) for x in o2.site]) o2.u_star = tuple( [ u*(1 + random.uniform(-self.delta_u_star, self.delta_u_star)) for u in o2.u_star]) for sc in xs.scatterers(): sc.flags.set_use_u_iso(False).set_use_u_aniso(True) sc.flags.set_grad_site(True).set_grad_u_aniso(True) connectivity_table = smtbx.utils.connectivity_table(xs) reparametrisation = constraints.reparametrisation( structure=xs, constraints=[], connectivity_table=connectivity_table) ls = least_squares.crystallographic_ls( self.fo_sq.as_xray_observations(), reparametrisation, weighting_scheme=least_squares.unit_weighting(), origin_fixing_restraints_type= origin_fixing_restraints.atomic_number_weighting) cycles = normal_eqns_solving.levenberg_marquardt_iterations( ls, gradient_threshold=1e-12, step_threshold=1e-7, track_all=True) ## Test whether refinement brought back the shaked structure to its ## original state match = emma.model_matches(xs0.as_emma_model(), xs.as_emma_model()).refined_matches[0] assert match.rt.r == matrix.identity(3) assert not match.singles1 and not match.singles2 assert match.rms < 1e-6 delta_u_carts= ( xs.scatterers().extract_u_cart(xs.unit_cell()) - xs0.scatterers().extract_u_cart(xs.unit_cell())).norms() assert flex.abs(delta_u_carts) < 1e-6 assert approx_equal(ls.scale_factor(), 1, eps=1e-4) ## Test covariance matrix jac_tr = reparametrisation.jacobian_transpose_matching_grad_fc() cov = ls.covariance_matrix( jacobian_transpose=jac_tr, normalised_by_goof=False)\ .matrix_packed_u_as_symmetric() m, n = cov.accessor().focus() # x,y for point group 3 sites are fixed: no variance or correlation for i in (0, 9, 18, 27,): assert cov.matrix_copy_block(i, 0, 2, n) == 0 # u_star coefficients u13 and u23 for point group 3 sites are fixed # to 0: again no variance or correlation with any other param for i in (7, 16, 25, 34,): assert cov.matrix_copy_block(i, 0, 2, n).as_1d()\ .all_approx_equal(0., 1e-20) # u_star coefficients u11, u22 and u12 for point group 3 sites # are totally correlated, with variances in ratios 1:1:1/2 for i in (3, 12, 21, 30,): assert cov[i, i] != 0 assert approx_equal(cov[i, i], cov[i+1, i+1], eps=1e-15) assert approx_equal(cov[i, i+1]/cov[i, i], 1, eps=1e-12) assert approx_equal(cov[i, i+3]/cov[i, i], 0.5, eps=1e-12)