def d_target_d_site_cart(self): manager = self.manager xray.set_scatterer_grad_flags( scatterers=manager.xray_structure.scatterers(), site=True) return flex.vec3_double( self.gradients_wrt_atomic_parameters().packed())
def __init__(self, fmodel, max_iterations=100, sites=False, u_iso=False): self.fmodel = fmodel self.fmodel.xray_structure.scatterers().flags_set_grads(state=False) self.x_target_functor = self.fmodel.target_functor() self.sites = sites self.u_iso = u_iso if (self.sites): self.x = self.fmodel.xray_structure.sites_cart().as_double() if (self.u_iso): assert self.fmodel.xray_structure.scatterers().size() == \ self.fmodel.xray_structure.use_u_iso().count(True) self.x = self.fmodel.xray_structure.extract_u_iso_or_u_equiv() if (self.sites): xray.set_scatterer_grad_flags( scatterers=self.fmodel.xray_structure.scatterers(), site=True) if (self.u_iso): sel = flex.bool(self.fmodel.xray_structure.scatterers().size(), True).iselection() self.fmodel.xray_structure.scatterers().flags_set_grad_u_iso( iselection=sel) self.minimizer = scitbx.lbfgs.run( target_evaluator=self, termination_params=scitbx.lbfgs.termination_parameters( max_iterations=max_iterations), exception_handling_params=scitbx.lbfgs. exception_handling_parameters( ignore_line_search_failed_rounding_errors=True, ignore_line_search_failed_step_at_lower_bound=True, ignore_line_search_failed_maxfev=True)) self.fmodel.xray_structure.tidy_us() self.fmodel.xray_structure.apply_symmetry_sites() self.fmodel.update_xray_structure( xray_structure=self.fmodel.xray_structure, update_f_calc=True)
def initialize(self, fmodel=None): self.not_hd_selection = ~self.fmodel.xray_structure.hd_selection( ) # XXX UGLY assert fmodel is not None self.fmodel = fmodel self.fmodel.xray_structure.scatterers().flags_set_grads(state=False) xray.set_scatterer_grad_flags( scatterers=self.fmodel.xray_structure.scatterers(), site=True) self.x = self.fmodel.xray_structure.sites_cart().as_double() self.x_target_functor = self.fmodel.target_functor()
def __init__(self, fmodel, tlsos_initial, refine_T, refine_L, refine_S, selections, selections_1d, max_iterations, run_finite_differences_test=False, correct_adp=True): adopt_init_args(self, locals()) fmodel.xray_structure.scatterers().flags_set_grads(state=False) xray.set_scatterer_grad_flags( scatterers=fmodel.xray_structure.scatterers(), u_aniso=True) if (self.run_finite_differences_test): self.correct_adp = False self.fmodel_copy = self.fmodel.deep_copy() self.target_functor = self.fmodel_copy.target_functor() self.run_finite_differences_test_counter = 0 self.T_initial = [] self.L_initial = [] self.S_initial = [] self.origins = [] for tlso_ in tlsos_initial: self.T_initial.append(tlso_.t) self.L_initial.append(tlso_.l) self.S_initial.append(tlso_.s) self.origins.append(tlso_.origin) self.counter = 0 self.n_groups = len(self.T_initial) self.dim_T = len(self.T_initial[0]) self.dim_L = len(self.L_initial[0]) self.dim_S = len(self.S_initial[0]) self.T_min = self.T_initial self.L_min = self.L_initial self.S_min = self.S_initial self.x = self.pack(self.T_min, self.L_min, self.S_min) self.minimizer = lbfgs.run( target_evaluator=self, core_params=lbfgs.core_parameters(maxfev=10), termination_params=lbfgs.termination_parameters( min_iterations=max_iterations, max_calls=int(max_iterations * 1.5)), exception_handling_params=lbfgs.exception_handling_parameters( ignore_line_search_failed_step_at_lower_bound=True, ignore_line_search_failed_step_at_upper_bound=True, ignore_line_search_failed_maxfev=True)) self.compute_functional_and_gradients() del self.x self.tlsos_result = generate_tlsos( selections=self.selections, xray_structure=self.fmodel.xray_structure, T=self.T_min, L=self.L_min, S=self.S_min)
def __init__(self, fmodel, tlsos_initial, refine_T, refine_L, refine_S, selections, selections_1d, max_iterations, run_finite_differences_test = False, correct_adp = True): adopt_init_args(self, locals()) fmodel.xray_structure.scatterers().flags_set_grads(state=False) xray.set_scatterer_grad_flags(scatterers = fmodel.xray_structure.scatterers(), u_aniso = True) if(self.run_finite_differences_test): self.correct_adp = False self.fmodel_copy = self.fmodel.deep_copy() self.target_functor = self.fmodel_copy.target_functor() self.run_finite_differences_test_counter = 0 self.T_initial = [] self.L_initial = [] self.S_initial = [] self.origins = [] for tlso_ in tlsos_initial: self.T_initial.append(tlso_.t) self.L_initial.append(tlso_.l) self.S_initial.append(tlso_.s) self.origins.append(tlso_.origin) self.counter = 0 self.n_groups = len(self.T_initial) self.dim_T = len(self.T_initial[0]) self.dim_L = len(self.L_initial[0]) self.dim_S = len(self.S_initial[0]) self.T_min = self.T_initial self.L_min = self.L_initial self.S_min = self.S_initial self.x = self.pack(self.T_min, self.L_min, self.S_min) self.minimizer = lbfgs.run( target_evaluator = self, core_params = lbfgs.core_parameters(maxfev = 10), termination_params = lbfgs.termination_parameters( min_iterations = max_iterations, max_calls = int(max_iterations*1.5)), exception_handling_params = lbfgs.exception_handling_parameters( ignore_line_search_failed_step_at_lower_bound = True, ignore_line_search_failed_step_at_upper_bound = True, ignore_line_search_failed_maxfev = True)) self.compute_functional_and_gradients() del self.x self.tlsos_result = generate_tlsos( selections = self.selections, xray_structure = self.fmodel.xray_structure, T = self.T_min, L = self.L_min, S = self.S_min)
def __init__(self, fmodel, model, xs_initial, adp_nmas, fsub_adp, selections, selections_1d, max_iterations, n_modes, weight_nmre=1.0, run_finite_differences_test=False, correct_adp=True, zero_mode_flag=True): adopt_init_args(self, locals()) fmodel.xray_structure.scatterers().flags_set_grads(state=False) xray.set_scatterer_grad_flags( scatterers=fmodel.xray_structure.scatterers(), u_aniso=True) if (self.run_finite_differences_test): self.correct_adp = False self.fmodel_copy = self.fmodel.deep_copy() self.target_functor = self.fmodel_copy.target_functor() self.run_finite_differences_test_counter = 0 self.counter = 0 self.n_groups = len(self.xs_initial) self.dim_x = len(self.xs_initial[0]) self.xs_min = self.xs_initial self.x = self.pack(self.xs_min) # for adp_nma_selected, selection in zip(adp_nmas, selections): # weights_selected = self.fmodel.xray_structure.atomic_weights.select(selection) # modes1d_selected = selected_modes_to_1D(modes = self.modes, n_modes = self.n_modes, # selection = self.selection) # assert len(modes1d_selected)/n_modes == len(weights_selected) # adp_nma_selected = init_nm_adp(modes = modes1d_selected, # weights = weights_selected, # n_modes = n_modes, # zero_mode_flag = zero_mode_flag) # self.adp_nmas.append(adp_nma_selected) self.minimizer = lbfgs.run( target_evaluator=self, core_params=lbfgs.core_parameters(maxfev=10), termination_params=lbfgs.termination_parameters( min_iterations=max_iterations, max_calls=int(max_iterations * 1.5)), exception_handling_params=lbfgs.exception_handling_parameters( ignore_line_search_failed_step_at_lower_bound=True, ignore_line_search_failed_step_at_upper_bound=True, ignore_line_search_failed_maxfev=True)) self.compute_functional_and_gradients() del self.x self.xs_result = self.xs_min
def __init__(self, fmodel, states, restraints_manager, max_iterations=50, sites = False, u_iso = False, restraints_weight=1, data_weight=None, restraints_weight_scale=1): self.cntr = 0 self.rws = restraints_weight_scale self.states = states self.fmodel = fmodel self.restraints_weight = restraints_weight self.data_weight = data_weight self.restraints_manager = restraints_manager self.fmodel.xray_structure.scatterers().flags_set_grads(state=False) self.x_target_functor = self.fmodel.target_functor() self.sites = sites self.u_iso = u_iso if(self.sites): self.x = self.fmodel.xray_structure.sites_cart().as_double() if(self.u_iso): assert self.fmodel.xray_structure.scatterers().size() == \ self.fmodel.xray_structure.use_u_iso().count(True) self.x = self.fmodel.xray_structure.extract_u_iso_or_u_equiv() if(self.sites): xray.set_scatterer_grad_flags( scatterers = self.fmodel.xray_structure.scatterers(), site = True) if(self.u_iso): sel = flex.bool( self.fmodel.xray_structure.scatterers().size(), True).iselection() self.fmodel.xray_structure.scatterers().flags_set_grad_u_iso( iselection = sel) self.minimizer = scitbx.lbfgs.run( target_evaluator=self, termination_params=scitbx.lbfgs.termination_parameters( max_iterations=max_iterations), exception_handling_params=scitbx.lbfgs.exception_handling_parameters( ignore_line_search_failed_rounding_errors=True, ignore_line_search_failed_step_at_lower_bound=True, ignore_line_search_failed_maxfev=True)) self.fmodel.xray_structure.tidy_us() self.fmodel.xray_structure.apply_symmetry_sites() self.fmodel.update_xray_structure( xray_structure = self.fmodel.xray_structure, update_f_calc = True)
def __init__(self, fmodel, model, xs_initial, adp_nmas, selections, selections_1d, max_iterations, n_modes, weight_nmre = 1.0, run_finite_differences_test = False, correct_adp = True, zero_mode_flag = True): adopt_init_args(self, locals()) fmodel.xray_structure.scatterers().flags_set_grads(state=False) xray.set_scatterer_grad_flags(scatterers = fmodel.xray_structure.scatterers(), u_aniso = True) if(self.run_finite_differences_test): self.correct_adp = False self.fmodel_copy = self.fmodel.deep_copy() self.target_functor = self.fmodel_copy.target_functor() self.run_finite_differences_test_counter = 0 self.counter = 0 self.n_groups = len(self.xs_initial) self.dim_x = len(self.xs_initial[0]) self.xs_min = self.xs_initial self.x = self.pack(self.xs_min) # for adp_nma_selected, selection in zip(adp_nmas, selections): # weights_selected = self.fmodel.xray_structure.atomic_weights.select(selection) # modes1d_selected = selected_modes_to_1D(modes = self.modes, n_modes = self.n_modes, # selection = self.selection) # assert len(modes1d_selected)/n_modes == len(weights_selected) # adp_nma_selected = init_nm_adp(modes = modes1d_selected, # weights = weights_selected, # n_modes = n_modes, # zero_mode_flag = zero_mode_flag) # self.adp_nmas.append(adp_nma_selected) self.minimizer = lbfgs.run( target_evaluator = self, core_params = lbfgs.core_parameters(), termination_params = lbfgs.termination_parameters( min_iterations = max_iterations, max_calls = int(max_iterations*1.5)), exception_handling_params = lbfgs.exception_handling_parameters( ignore_line_search_failed_step_at_lower_bound = False, ignore_line_search_failed_step_at_upper_bound = True, ignore_line_search_failed_maxfev = False)) self.compute_functional_and_gradients() del self.x self.xs_result = self.xs_min
def site(structure_ideal, d_min, f_obs, verbose=0): sh = shifted_site(f_obs, structure_ideal, 0, 0, 0.01) if (0 or verbose): print("site") sh.structure_shifted.show_summary().show_scatterers() print() ls = xray.targets_least_squares_residual(f_obs.data(), sh.f_calc.data(), True, 1) gradient_flags = randomize_gradient_flags( xray.structure_factors.gradient_flags(site=True), f_obs.anomalous_flag()) xray.set_scatterer_grad_flags(scatterers=sh.structure_shifted.scatterers(), site=gradient_flags.site, u_iso=gradient_flags.u_iso, u_aniso=gradient_flags.u_aniso, occupancy=gradient_flags.occupancy, fp=gradient_flags.fp, fdp=gradient_flags.fdp) sfd = xray.structure_factors.gradients_direct( xray_structure=sh.structure_shifted, u_iso_refinable_params=None, miller_set=f_obs, d_target_d_f_calc=ls.derivatives(), n_parameters=0) re = resampling(miller_set=f_obs) map0 = re(xray_structure=sh.structure_shifted, u_iso_refinable_params=None, dp=miller.array(miller_set=f_obs, data=ls.derivatives()), n_parameters=0, verbose=verbose) sfd_d_target_d_site_cart = sfd.d_target_d_site_cart() top_gradient = None for i_scatterer, scatterer in enumerate(sh.structure_shifted.scatterers()): for i_xyz in (0, 1, 2): direct_summ = sfd_d_target_d_site_cart[i_scatterer][i_xyz] if (top_gradient is None): top_gradient = direct_summ fast_gradie = map0.d_target_d_site_cart()[i_scatterer][i_xyz] match = judge(scatterer, "site", direct_summ, fast_gradie, top_gradient) if (0 or verbose): print("direct summ[%d][%d]: " % (i_scatterer, i_xyz), direct_summ) print("fast gradie[%d][%d]: " % (i_scatterer, i_xyz), fast_gradie, match) print() assert not match.is_bad sys.stdout.flush()
def __init__(self, restraints_manager=None, fmodel=None, sites_cart=None, wx=None, wc=None, update_gradient_threshold=0): adopt_init_args(self, locals()) assert [self.fmodel, self.wx].count(None) in [0, 2] assert [self.restraints_manager, self.wc].count(None) in [0, 2] self.gx, self.gc = 0, 0 if (self.fmodel is not None): self.x_target_functor = self.fmodel.target_functor() xray.set_scatterer_grad_flags( scatterers=self.fmodel.xray_structure.scatterers(), site=True) self.gx = flex.vec3_double(self.x_target_functor(compute_gradients=True).\ gradients_wrt_atomic_parameters(site=True).packed())
def site(structure_ideal, d_min, f_obs, verbose=0): sh = shifted_site(f_obs, structure_ideal, 0, 0, 0.01) if (0 or verbose): print "site" sh.structure_shifted.show_summary().show_scatterers() print ls = xray.targets_least_squares_residual( f_obs.data(), sh.f_calc.data(), True, 1) gradient_flags = randomize_gradient_flags( xray.structure_factors.gradient_flags(site=True), f_obs.anomalous_flag()) xray.set_scatterer_grad_flags(scatterers = sh.structure_shifted.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp) sfd = xray.structure_factors.gradients_direct( xray_structure=sh.structure_shifted, u_iso_refinable_params=None, miller_set=f_obs, d_target_d_f_calc=ls.derivatives(), n_parameters=0) re = resampling(miller_set=f_obs) map0 = re( xray_structure=sh.structure_shifted, u_iso_refinable_params=None, dp=miller.array(miller_set=f_obs, data=ls.derivatives()), n_parameters=0, verbose=verbose) sfd_d_target_d_site_cart = sfd.d_target_d_site_cart() top_gradient = None for i_scatterer,scatterer in enumerate(sh.structure_shifted.scatterers()): for i_xyz in (0,1,2): direct_summ = sfd_d_target_d_site_cart[i_scatterer][i_xyz] if (top_gradient is None): top_gradient = direct_summ fast_gradie = map0.d_target_d_site_cart()[i_scatterer][i_xyz] match = judge(scatterer, "site", direct_summ, fast_gradie, top_gradient) if (0 or verbose): print "direct summ[%d][%d]: " % (i_scatterer,i_xyz), direct_summ print "fast gradie[%d][%d]: " % (i_scatterer,i_xyz), fast_gradie, match print assert not match.is_bad sys.stdout.flush()
def __init__(self, restraints_manager = None, fmodel = None, sites_cart = None, wx = None, wc = None, update_gradient_threshold = 0): adopt_init_args(self, locals()) assert [self.fmodel, self.wx].count(None) in [0,2] assert [self.restraints_manager, self.wc].count(None) in [0,2] self.gx, self.gc = 0, 0 if(self.fmodel is not None): self.x_target_functor = self.fmodel.target_functor() xray.set_scatterer_grad_flags( scatterers = self.fmodel.xray_structure.scatterers(), site = True) self.gx = flex.vec3_double(self.x_target_functor(compute_gradients=True).\ gradients_wrt_atomic_parameters(site=True).packed())
def __init__(self, fmodel, max_iterations=100, sites = False, u_iso = False): self.fmodel = fmodel self.fmodel.xray_structure.scatterers().flags_set_grads(state=False) self.x_target_functor = self.fmodel.target_functor() self.sites = sites self.u_iso = u_iso if(self.sites): # Coordiantes self.x = self.fmodel.xray_structure.sites_cart().as_double() if(self.u_iso): # anisotropic displacement factors assert self.fmodel.xray_structure.scatterers().size() == \ self.fmodel.xray_structure.use_u_iso().count(True) self.x = self.fmodel.xray_structure.extract_u_iso_or_u_equiv() if(self.sites): xray.set_scatterer_grad_flags( scatterers = self.fmodel.xray_structure.scatterers(), site = True) if(self.u_iso): sel = flex.bool( self.fmodel.xray_structure.scatterers().size(), True).iselection() self.fmodel.xray_structure.scatterers().flags_set_grad_u_iso( iselection = sel) self.minimizer = scitbx.lbfgs.run( target_evaluator=self, termination_params=scitbx.lbfgs.termination_parameters( max_iterations=max_iterations), exception_handling_params=scitbx.lbfgs.exception_handling_parameters( ignore_line_search_failed_rounding_errors=True, ignore_line_search_failed_step_at_lower_bound=True, ignore_line_search_failed_maxfev=True)) self.fmodel.xray_structure.tidy_us() self.fmodel.xray_structure.apply_symmetry_sites() self.fmodel.update_xray_structure( xray_structure = self.fmodel.xray_structure, update_f_calc = True)
def __init__(self, fmodel, ncs_restraints_group_list, refine_selection=None, use_ncs_constraints=True, restraints_manager=None, data_weight=None, refine_sites=False, refine_u_iso=False, refine_transformations=False, iso_restraints=None, use_hd=False): adopt_init_args(self, locals()) asu_size = self.fmodel.xray_structure.sites_cart().size() self.refine_selection = nu.get_refine_selection( refine_selection=self.refine_selection, number_of_atoms=asu_size) self.extended_ncs_selection = nu.get_extended_ncs_selection( ncs_restraints_group_list=ncs_restraints_group_list, refine_selection=self.refine_selection) self.fmodel.xray_structure.scatterers().flags_set_grads(state=False) self.x_target_functor = self.fmodel.target_functor() self.xray_structure = self.fmodel.xray_structure if self.refine_sites: xray.set_scatterer_grad_flags( scatterers=self.fmodel.xray_structure.scatterers(), site=True) elif self.refine_u_iso: xray.set_scatterer_grad_flags( scatterers=self.fmodel.xray_structure.scatterers(), u_iso=True) elif self.refine_transformations: xray.set_scatterer_grad_flags( scatterers=self.fmodel.xray_structure.scatterers(), site=True)
def finite_differences_test(): print "finite_differences_test: " fmodel = get_fmodel_from_pdb(pdb_file_name="enk_rbr.pdb", algorithm="direct", d_min=2.0, target="ls_wunit_k1") xray.set_scatterer_grad_flags( scatterers=fmodel.xray_structure.scatterers(), site=True) for convention in ["zyz", "xyz"]: rot_obj = scitbx.rigid_body.euler(phi=0, psi=0, the=0, convention=convention) size = fmodel.xray_structure.scatterers().size() selections = [flex.bool(size, True).iselection()] fmodel.xray_structure.apply_rigid_body_shift(rot=rot_obj.rot_mat(), trans=[1, 2, 3]) fmodel.update_xray_structure(update_f_calc=True) fmodel_copy = fmodel.deep_copy() centers_of_mass = [] for s in selections: xrs = fmodel_copy.xray_structure.select(s) centers_of_mass.append(xrs.center_of_mass()) tg_obj = mmtbx.refinement.rigid_body.target_and_grads( centers_of_mass=centers_of_mass, sites_cart=fmodel_copy.xray_structure.sites_cart(), target_functor=fmodel_copy.target_functor(), rot_objs=[rot_obj], selections=selections, suppress_gradients=False) assert approx_equal(tg_obj.target(), fmodel_copy.target_w()) g_rot, g_transl = tg_obj.gradients_wrt_r(), tg_obj.gradients_wrt_t() fd_transl = fd_translation(fmodel_copy, e=0.0001) assert approx_equal(list(g_transl[0]), fd_transl) fd_rot = fd_rotation(fmodel=fmodel_copy, e=0.0001, convention=convention) assert approx_equal(list(g_rot[0]), fd_rot)
def finite_differences_test(): print "finite_differences_test: " fmodel = get_fmodel_from_pdb(pdb_file_name = "enk_rbr.pdb", algorithm = "direct", d_min = 2.0, target = "ls_wunit_k1") xray.set_scatterer_grad_flags( scatterers = fmodel.xray_structure.scatterers(), site = True) for convention in ["zyz","xyz"]: rot_obj = scitbx.rigid_body.euler( phi = 0, psi = 0, the = 0, convention = convention) size = fmodel.xray_structure.scatterers().size() selections = [flex.bool(size, True).iselection()] fmodel.xray_structure.apply_rigid_body_shift( rot = rot_obj.rot_mat(), trans = [1,2,3]) fmodel.update_xray_structure(update_f_calc = True) fmodel_copy = fmodel.deep_copy() centers_of_mass = [] for s in selections: xrs = fmodel_copy.xray_structure.select(s) centers_of_mass.append(xrs.center_of_mass()) tg_obj = mmtbx.refinement.rigid_body.target_and_grads( centers_of_mass = centers_of_mass, sites_cart = fmodel_copy.xray_structure.sites_cart(), target_functor = fmodel_copy.target_functor(), rot_objs = [rot_obj], selections = selections, suppress_gradients = False) assert approx_equal(tg_obj.target(),fmodel_copy.target_w()) g_rot, g_transl = tg_obj.gradients_wrt_r(), tg_obj.gradients_wrt_t() fd_transl = fd_translation(fmodel_copy, e = 0.0001) assert approx_equal(list(g_transl[0]), fd_transl) fd_rot = fd_rotation(fmodel = fmodel_copy, e = 0.0001, convention = convention) assert approx_equal(list(g_rot[0]), fd_rot)
def __init__( self, fmodel, ncs_restraints_group_list, refine_selection=None, use_ncs_constraints=True, restraints_manager=None, data_weight=None, refine_sites=False, refine_u_iso=False, refine_transformations=False, iso_restraints = None, use_hd = False): adopt_init_args(self, locals()) asu_size = self.fmodel.xray_structure.sites_cart().size() self.refine_selection = nu.get_refine_selection( refine_selection=self.refine_selection, number_of_atoms=asu_size) self.extended_ncs_selection = nu.get_extended_ncs_selection( ncs_restraints_group_list=ncs_restraints_group_list, refine_selection=self.refine_selection) self.fmodel.xray_structure.scatterers().flags_set_grads(state=False) self.x_target_functor = self.fmodel.target_functor() self.xray_structure = self.fmodel.xray_structure if self.refine_sites: xray.set_scatterer_grad_flags( scatterers = self.fmodel.xray_structure.scatterers(), site = True) elif self.refine_u_iso: xray.set_scatterer_grad_flags( scatterers = self.fmodel.xray_structure.scatterers(), u_iso = True) elif self.refine_transformations: xray.set_scatterer_grad_flags( scatterers = self.fmodel.xray_structure.scatterers(), site = True)
def get_intensity_structure(base,FE1_model,FE2_model): """Now used stuff learned from test_fmodel_stuff to calculate new data structure for use in the program. """ # generate the list of all HKL to be used throughout. C2_structures = get_C2_structures() energy = 7070.0 GF_whole7070 = gen_fmodel_with_complex.from_structure(C2_structures[0],energy ).from_parameters(algorithm="fft") GF_whole7070 = GF_whole7070.as_P1_primitive() f_container7070 = GF_whole7070.get_fmodel() Fmodel_whole7070 = f_container7070.f_model Fmodel_indices = Fmodel_whole7070.indices() # common structure defines the indices MS = Fmodel_whole7070.set() # avoid having to repeatedly calculate indices CS = Fmodel_whole7070.crystal_symmetry() # same here result = flex.double(flex.grid((Fmodel_indices.size(),500))) GF_FE1 = gen_fmodel_with_complex.from_structure(C2_structures[2],energy ).from_parameters(algorithm="direct") GF_FE1 = GF_FE1.as_P1_primitive() GF_FE2 = gen_fmodel_with_complex.from_structure(C2_structures[3],energy ).from_parameters(algorithm="direct") GF_FE2 = GF_FE2.as_P1_primitive() for incr in range(100): print ("incr is",incr) energy = 7070.5 + incr GF_FE1.reset_specific_at_energy(label_has="FE1",tables=FE1_model,newvalue=energy) # not sure if I need this, takes a lot of time # f_container = GF_FE1.get_fmodel() # not sure if I need this, takes a lot of time # Fcalc_FE1 = f_container.fmodel.f_calc() GF_FE2.reset_specific_at_energy(label_has="FE2",tables=FE2_model,newvalue=energy) # not sure if I need this, takes a lot of time # f_container = GF_FE2.get_fmodel() # not sure if I need this, takes a lot of time # Fcalc_FE2 = f_container.fmodel.f_calc() ALGO = structure_factors.from_scatterers(crystal_symmetry=CS, d_min=GF_whole7070.params2.high_resolution) #hack cctbx/xray/structure_factors/structure_factors_direct.h """ /*#if !defined(CCTBX_XRAY_STRUCTURE_FACTORS_DIRECT_NO_PRAGMA_OMP) #if !defined(__DECCXX_VER) || (defined(_OPENMP) && _OPENMP > 199819) #pragma omp parallel for schedule(static) #endif #endif */ #pragma omp parallel for """ from_scatterers_direct_fe1 = ALGO(xray_structure=GF_FE1.xray_structure, miller_set=MS,algorithm="direct") Fcalc_FE1_dir = from_scatterers_direct_fe1.f_calc().data() from_scatterers_direct_fe2 = ALGO(xray_structure=GF_FE2.xray_structure, miller_set=MS,algorithm="direct") Fcalc_FE2_dir = from_scatterers_direct_fe2.f_calc().data() # Get total Fcalc at energy: F_bulk_non_Fe = base.matrix_copy_block(i_row=0,i_column=incr, n_rows=Fmodel_indices.size(),n_columns=1) Fcalc_FE1_dir.reshape(flex.grid((Fmodel_indices.size(),1))) Fcalc_FE2_dir.reshape(flex.grid((Fmodel_indices.size(),1))) Fcalc_total = F_bulk_non_Fe + Fcalc_FE1_dir + Fcalc_FE2_dir result.matrix_paste_block_in_place(flex.norm(Fcalc_total),0,incr) # gives I = F * F gradient_flags=xray.structure_factors.gradient_flags( site=False, u_iso=False, u_aniso=False, occupancy=False, fp=True, fdp=True) xray.set_scatterer_grad_flags(scatterers = GF_FE1.xray_structure.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp) xray.set_scatterer_grad_flags(scatterers = GF_FE2.xray_structure.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp) #hack cctbx/xray/structure_factors/each_hkl_gradients_direct.h #pragma omp parallel for (line 226) sf1 = xray.ext.each_hkl_gradients_direct( MS.unit_cell(), MS.space_group(), MS.indices(), GF_FE1.xray_structure.scatterers(), None, GF_FE1.xray_structure.scattering_type_registry(), GF_FE1.xray_structure.site_symmetry_table(), 0) sf2 = xray.ext.each_hkl_gradients_direct( MS.unit_cell(), MS.space_group(), MS.indices(), GF_FE2.xray_structure.scatterers(), None, GF_FE2.xray_structure.scattering_type_registry(), GF_FE2.xray_structure.site_symmetry_table(), 0) sf1.d_fcalc_d_fp().reshape(flex.grid((Fmodel_indices.size(),1))) sf1.d_fcalc_d_fdp().reshape(flex.grid((Fmodel_indices.size(),1))) sf2.d_fcalc_d_fp().reshape(flex.grid((Fmodel_indices.size(),1))) sf2.d_fcalc_d_fdp().reshape(flex.grid((Fmodel_indices.size(),1))) parts = Fcalc_total.parts() A = parts[0] B = parts[1] for offset,vec2 in zip([100,200,300,400], [sf1.d_fcalc_d_fp(),sf1.d_fcalc_d_fdp(),sf2.d_fcalc_d_fp(),sf2.d_fcalc_d_fdp()]): vparts = vec2.parts() vA = vparts[0] vB = vparts[1] partial_I_partial_q = 2. * (A * vA + B * vB) result.matrix_paste_block_in_place(partial_I_partial_q,0,incr + offset) # result holds a table of structure factor intensities. Rows are Miller indices H. # First 100 columns are I_H(energy, 100 channels). The next four groups of 100 columns # are the partial derivatives of I_H with respect to fp(FE1), fdp(FE1), fp(FE2), and fdp(FE2) # respectively all as a function of energy channel. Thus this is # the energy-dependent portion of the calculation that is dependent on the iron model. return result
def exercise(space_group_info, n_elements = 10, table = "wk1995", d_min = 2.0, k_sol = 0.35, b_sol = 45.0, b_cart = None, quick=False, verbose=0): xray_structure = random_structure.xray_structure( space_group_info = space_group_info, elements =(("O","N","C")*(n_elements//3+1))[:n_elements], volume_per_atom = 100, min_distance = 1.5, general_positions_only = True, random_u_iso = False, random_occupancy = False) xray_structure.scattering_type_registry(table = table) sg = xray_structure.space_group() uc = xray_structure.unit_cell() u_cart_1 = adptbx.random_u_cart(u_scale=5, u_min=5) u_star_1 = adptbx.u_cart_as_u_star(uc, u_cart_1) b_cart = adptbx.u_star_as_u_cart(uc, sg.average_u_star(u_star = u_star_1)) for anomalous_flag in [False, True]: scatterers = xray_structure.scatterers() if (anomalous_flag): assert scatterers.size() >= 7 for i in [1,7]: scatterers[i].fp = -0.2 scatterers[i].fdp = 5 have_non_zero_fdp = True else: for i in [1,7]: scatterers[i].fp = 0 scatterers[i].fdp = 0 have_non_zero_fdp = False f_obs = abs(xray_structure.structure_factors( d_min = d_min, anomalous_flag = anomalous_flag, cos_sin_table = sfg_params.cos_sin_table, algorithm = sfg_params.algorithm).f_calc()) f_obs_comp = f_obs.structure_factors_from_scatterers( xray_structure = xray_structure, algorithm = sfg_params.algorithm, cos_sin_table = sfg_params.cos_sin_table).f_calc() f_obs = abs(f_obs_comp) flags = f_obs.generate_r_free_flags(fraction = 0.1, max_free = 99999999) #flags = flags.array(data = flex.bool(f_obs.data().size(), False)) xrs = xray_structure.deep_copy_scatterers() xrs.shake_sites_in_place(rms_difference=0.3) for target in mmtbx.refinement.targets.target_names: if (quick): if (target not in ["ls_wunit_k1", "ml", "mlhl", "ml_sad"]): continue if (target == "mlhl"): if (have_non_zero_fdp): continue # XXX gradients not correct! experimental_phases = generate_random_hl(miller_set=f_obs) else: experimental_phases = None if (target == "ml_sad" and (not anomalous_flag or mmtbx.refinement.targets.phaser is None)): continue print " ",target xray.set_scatterer_grad_flags( scatterers = xrs.scatterers(), site = True) ss = 1./flex.pow2(f_obs.d_spacings().data()) / 4. u_star = adptbx.u_cart_as_u_star( f_obs.unit_cell(), adptbx.b_as_u(b_cart)) k_anisotropic = mmtbx.f_model.ext.k_anisotropic( f_obs.indices(), u_star) k_mask = mmtbx.f_model.ext.k_mask(ss, k_sol, b_sol) fmodel = mmtbx.f_model.manager( xray_structure = xrs, f_obs = f_obs, r_free_flags = flags, target_name = target, abcd = experimental_phases, sf_and_grads_accuracy_params = sfg_params, k_mask = k_mask, k_anisotropic = k_anisotropic, mask_params = masks.mask_master_params.extract()) fmodel.update_xray_structure( xray_structure=xrs, update_f_calc=True, update_f_mask=True) xray.set_scatterer_grad_flags( scatterers=fmodel.xray_structure.scatterers(), site=True) fmodel.update_xray_structure(update_f_calc=True) t_f = fmodel.target_functor() t_f.prepare_for_minimization() gs = t_f(compute_gradients=True).d_target_d_site_cart().as_double() gfd = finite_differences_site(target_functor=t_f) cc = flex.linear_correlation(gs, gfd).coefficient() if (0 or verbose): print "ana:", list(gs) print "fin:", list(gfd) print "rat:", [f/a for a,f in zip(gs,gfd)] print target, "corr:", cc, space_group_info print diff = gs - gfd diff /= max(1, flex.max(flex.abs(gfd))) tolerance = 1.2e-5 assert approx_equal(abs(flex.min(diff) ), 0.0, tolerance) assert approx_equal(abs(flex.mean(diff)), 0.0, tolerance) assert approx_equal(abs(flex.max(diff) ), 0.0, tolerance) assert approx_equal(cc, 1.0, tolerance) fmodel.model_error_ml()
def exercise_gradient_manager(structure_ideal, f_obs, anomalous_flag, verbose=0): sh, ls = shift_all(structure_ideal, f_obs, anomalous_flag) grad_manager = xray.structure_factors.gradients( miller_set=f_obs, quality_factor=100000, wing_cutoff=1.e-10) gradient_flags=xray.structure_factors.gradient_flags(default=True) xray.set_scatterer_grad_flags(scatterers = sh.structure_shifted.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp) if (0): print "exercise_gradient_manager" print "gradient_flags.site ", gradient_flags.site print "gradient_flags.u_iso ", gradient_flags.u_iso print "gradient_flags.u_aniso ", gradient_flags.u_aniso print "gradient_flags.occupancy ", gradient_flags.occupancy print "gradient_flags.fp ", gradient_flags.fp print "gradient_flags.fdp ", gradient_flags.fdp cntr_use_u_iso = 0 cntr_use_u_aniso = 0 cntr_grad_u_iso = 0 cntr_grad_u_aniso = 0 for scatterer in sh.structure_shifted.scatterers(): if (scatterer.flags.use_u_iso()): cntr_use_u_iso += 1 if (scatterer.flags.use_u_aniso()): cntr_use_u_aniso += 1 if (scatterer.flags.grad_u_iso()): cntr_grad_u_iso += 1 if (scatterer.flags.grad_u_aniso()): cntr_grad_u_aniso += 1 print "use_u_iso ", cntr_use_u_iso,cntr_grad_u_iso print "use_u_aniso ", cntr_use_u_aniso,cntr_grad_u_aniso if (random.random() > 0.5): n_parameters = 0 else: n_parameters = xray.scatterer_grad_flags_counts( sh.structure_shifted.scatterers()).n_parameters() assert n_parameters == sh.structure_shifted.n_parameters() gd = grad_manager( xray_structure=sh.structure_shifted, u_iso_refinable_params=None, miller_set=f_obs, d_target_d_f_calc=ls.derivatives(), n_parameters=n_parameters, algorithm="direct") gf = grad_manager( xray_structure=sh.structure_shifted, u_iso_refinable_params=None, miller_set=f_obs, d_target_d_f_calc=ls.derivatives(), n_parameters=n_parameters, algorithm="fft") grad_flags_counts = \ xray.scatterer_grad_flags_counts(sh.structure_shifted.scatterers()) if (n_parameters == 0): d = gd.d_target_d_site_frac() f = gf.d_target_d_site_frac() linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose) d = gd.d_target_d_site_cart() f = gf.d_target_d_site_cart() linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose) if(grad_flags_counts.u_iso > 0): d = gd.d_target_d_u_iso() f = gf.d_target_d_u_iso() linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose) if(grad_flags_counts.u_aniso > 0): linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose) d = gd.d_target_d_u_cart() f = gf.d_target_d_u_cart() linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose) d = gd.d_target_d_occupancy() f = gf.d_target_d_occupancy() linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose) d = gd.d_target_d_fp() f = gf.d_target_d_fp() linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose) if (anomalous_flag): d = gd.d_target_d_fdp() f = gf.d_target_d_fdp() linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose) else: correlation = flex.linear_correlation(gd.packed(), gf.packed()) assert correlation.is_well_defined() assert correlation.coefficient() > 0.995, correlation.coefficient()
def u_star(structure_ideal, d_min, f_obs, verbose=0): sh = shifted_u_star(f_obs, structure_ideal, 0, 0, 0.0001) if (0 or verbose): print "u_star" sh.structure_shifted.show_summary().show_scatterers() print ls = xray.targets_least_squares_residual( f_obs.data(), sh.f_calc.data(), True, 1) gradient_flags = randomize_gradient_flags( xray.structure_factors.gradient_flags(u_aniso=True), f_obs.anomalous_flag()) xray.set_scatterer_grad_flags(scatterers = sh.structure_shifted.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp) grad_flags_counts = xray.scatterer_grad_flags_counts(sh.structure_shifted.scatterers()) if(grad_flags_counts.n_parameters() > 0): if (0): print "u_aniso" print "gradient_flags.site ", gradient_flags.site print "gradient_flags.u_iso ", gradient_flags.u_iso print "gradient_flags.u_aniso ", gradient_flags.u_aniso print "gradient_flags.occupancy ", gradient_flags.occupancy print "gradient_flags.fp ", gradient_flags.fp print "gradient_flags.fdp ", gradient_flags.fdp cntr_use_u_iso = 0 cntr_use_u_aniso = 0 cntr_grad_u_iso = 0 cntr_grad_u_aniso = 0 for scatterer in sh.structure_shifted.scatterers(): if (scatterer.flags.use_u_iso()): cntr_use_u_iso += 1 if (scatterer.flags.use_u_aniso()): cntr_use_u_aniso += 1 if (scatterer.flags.grad_u_iso()): cntr_grad_u_iso += 1 if (scatterer.flags.grad_u_aniso()): cntr_grad_u_aniso += 1 print "use_u_iso ", cntr_use_u_iso,cntr_grad_u_iso print "use_u_aniso ", cntr_use_u_aniso,cntr_grad_u_aniso sfd = xray.structure_factors.gradients_direct( xray_structure=sh.structure_shifted, u_iso_refinable_params=None, miller_set=f_obs, d_target_d_f_calc=ls.derivatives(), n_parameters= 0 ) re = resampling(miller_set=f_obs) map0 = re( xray_structure=sh.structure_shifted, u_iso_refinable_params=None, dp=miller.array(miller_set=f_obs, data=ls.derivatives()), n_parameters= 0, verbose=verbose) grad_flags_counts = xray.scatterer_grad_flags_counts(sh.structure_shifted.scatterers()) if(grad_flags_counts.u_aniso): sfd_d_target_d_u_cart = sfd.d_target_d_u_cart() map0_d_target_d_u_cart = map0.d_target_d_u_cart() top_gradient = None gradients_1 = [] for i_scatterer,scatterer in enumerate(sh.structure_shifted.scatterers()): if(scatterer.flags.use_u_iso()): parameter_name = "u_iso" if(scatterer.flags.use_u_aniso()): parameter_name = "u_star" if(parameter_name == "u_iso" and scatterer.flags.grad_u_iso()): direct_summ = sfd.d_target_d_u_iso()[i_scatterer] if (top_gradient is None): top_gradient = direct_summ fast_gradie = map0.d_target_d_u_iso()[i_scatterer] sys.stdout.flush() gradients_1.append([direct_summ, fast_gradie]) match = judge(scatterer, parameter_name, direct_summ, fast_gradie, top_gradient) if (0 or verbose): print "direct summ[%d]: " % i_scatterer, direct_summ print "fast gradie[%d]: " % i_scatterer, fast_gradie, match print assert not match.is_bad if parameter_name == "u_star" and scatterer.flags.grad_u_aniso(): sfd_star = sfd.d_target_d_u_star()[i_scatterer] sfd_cart = adptbx.grad_u_star_as_u_cart( structure_ideal.unit_cell(), sfd_star) assert approx_equal( sfd_star, adptbx.grad_u_cart_as_u_star(structure_ideal.unit_cell(), sfd_cart)) for ij in xrange(6): direct_summ = sfd_d_target_d_u_cart[i_scatterer][ij] if (top_gradient is None): top_gradient = direct_summ fast_gradie = map0_d_target_d_u_cart[i_scatterer][ij] gradients_1.append([direct_summ, fast_gradie]) match =judge(scatterer,"u_star",direct_summ,fast_gradie,top_gradient) if (0 or verbose or match.is_bad): print "direct summ[%d][%d]: " % (i_scatterer, ij), direct_summ print "fast gradie[%d][%d]: " % (i_scatterer, ij),fast_gradie,match print assert not match.is_bad # Making sure that gradients_1 = gradients_2 for i_scatterer,scatterer in enumerate(sh.structure_shifted.scatterers()): if(not scatterer.flags.use_u_iso()): scatterer.u_iso = -12345.0 if(not scatterer.flags.use_u_aniso()): scatterer.u_star =(-999.,-999.,-999.,-999.,-999.,-999.) sfd = xray.structure_factors.gradients_direct( xray_structure=sh.structure_shifted, u_iso_refinable_params=None, miller_set=f_obs, d_target_d_f_calc=ls.derivatives(), n_parameters= 0 ) re = resampling(miller_set=f_obs) map0 = re( xray_structure=sh.structure_shifted, u_iso_refinable_params=None, dp=miller.array(miller_set=f_obs, data=ls.derivatives()), n_parameters= 0, verbose=verbose) grad_flags_counts = \ xray.scatterer_grad_flags_counts(sh.structure_shifted.scatterers()) if(grad_flags_counts.u_aniso): sfd_d_target_d_u_cart = sfd.d_target_d_u_cart() map0_d_target_d_u_cart = map0.d_target_d_u_cart() gradients_2 = [] for i_scatterer,scatterer in enumerate(sh.structure_shifted.scatterers()): if(scatterer.flags.use_u_iso()): parameter_name = "u_iso" if(scatterer.flags.use_u_aniso()): parameter_name = "u_star" if(parameter_name == "u_iso" and scatterer.flags.grad_u_iso()): direct_summ = sfd.d_target_d_u_iso()[i_scatterer] fast_gradie = map0.d_target_d_u_iso()[i_scatterer] gradients_2.append([direct_summ, fast_gradie]) if parameter_name == "u_star" and scatterer.flags.grad_u_aniso(): sfd_star = sfd.d_target_d_u_star()[i_scatterer] sfd_cart = adptbx.grad_u_star_as_u_cart( structure_ideal.unit_cell(), sfd_star) assert approx_equal( sfd_star, adptbx.grad_u_cart_as_u_star(structure_ideal.unit_cell(), sfd_cart)) for ij in xrange(6): direct_summ = sfd_d_target_d_u_cart[i_scatterer][ij] fast_gradie = map0_d_target_d_u_cart[i_scatterer][ij] gradients_2.append([direct_summ, fast_gradie]) for g1,g2 in zip(gradients_1, gradients_2): assert approx_equal(g1, g2) sys.stdout.flush()
def test_fmodel_stuff(energy,FE1_model,FE2_model): # section 1. C2 models. a) whole pdb, b) no Fe, c) FE1 only, d) FE2 only C2_structures = get_C2_structures() print("C2 models validate OK") # section 2. if False: for structure in C2_structures: GF = gen_fmodel_with_complex.from_structure(structure,energy).from_parameters() ID = GF.get_intensity_dictionary() # section 3. Validate that the old script work2_for_aca_lsq/remake_range_intensities.py # gives same intensities as the new gen_fmodel_with_complex class, for the # simple case of the reduced model at energy=7125 eV from LS49.work2_for_aca_lsq.remake_range_intensities import \ remake_intensities_at_energy as old_algorithm print("old") ID_ref = old_algorithm(energy,FE1_model,FE2_model) print("new") GF = gen_fmodel_with_complex.from_structure(C2_structures[0],energy).from_parameters() GF.reset_specific_at_energy(label_has="FE1",tables=FE1_model,newvalue=energy) GF.reset_specific_at_energy(label_has="FE2",tables=FE2_model,newvalue=energy) GF = GF.as_P1_primitive() ID_new = GF.get_intensity_dictionary() for key in ID_ref: assert ID_ref[key]==ID_new[key] assert len(ID_ref)==len(ID_new) print("whole structure validates OK") # section 4. Validate that Fmodel(whole pdb) = Fbulk + Fcalc(non-Fe) + Fcalc(FE1) + Fcalc(FE2) # For this validation it is necessary from cctbx.xray import structure_factors to carry the arithmetic using A+iB, not F*F #GF_whole = GF #from above, but compute it again here to use algo=direct GF_whole = gen_fmodel_with_complex.from_structure(C2_structures[0],energy).from_parameters(algorithm="fft") GF_whole.reset_specific_at_energy(label_has="FE1",tables=FE1_model,newvalue=energy) GF_whole.reset_specific_at_energy(label_has="FE2",tables=FE2_model,newvalue=energy) GF_whole = GF_whole.as_P1_primitive() f_container = GF_whole.get_fmodel() Fmodel_whole = f_container.f_model Fcalc_whole = f_container.fmodel.f_calc() Fbulk = Fcalc_whole.customized_copy(data=Fmodel_whole.data()-Fcalc_whole.data()) GF_non_Fe = gen_fmodel_with_complex.from_structure(C2_structures[1],energy).from_parameters(algorithm="fft") GF_non_Fe = GF_non_Fe.as_P1_primitive() f_container = GF_non_Fe.get_fmodel() Fcalc_non_Fe = f_container.fmodel.f_calc() GF_FE1 = gen_fmodel_with_complex.from_structure(C2_structures[2],energy).from_parameters(algorithm="direct") GF_FE1.reset_specific_at_energy(label_has="FE1",tables=FE1_model,newvalue=energy) GF_FE1 = GF_FE1.as_P1_primitive() f_container = GF_FE1.get_fmodel() Fcalc_FE1 = f_container.fmodel.f_calc() GF_FE2 = gen_fmodel_with_complex.from_structure(C2_structures[3],energy).from_parameters(algorithm="direct") GF_FE2.reset_specific_at_energy(label_has="FE2",tables=FE2_model,newvalue=energy) GF_FE2 = GF_FE2.as_P1_primitive() f_container = GF_FE2.get_fmodel() Fcalc_FE2 = f_container.fmodel.f_calc() # subpoint 4a. Fcalc(all) = Fcalc(non-FE)+Fcalc(FE1)+Fcalc(FE2) test4a = Fcalc_whole.customized_copy(data = Fcalc_whole.data() - Fcalc_non_Fe.data() - Fcalc_FE1.data() - Fcalc_FE2.data()) #print (list(test4a.data())) if True: n_outliers = 0 for ikey,key in enumerate(test4a.indices()): diff0 = Fcalc_whole.data()[ikey]-Fcalc_non_Fe.data()[ikey] diff1 = diff0 - Fcalc_FE1.data()[ikey] test4a_diff = test4a.data()[ikey] #It's not exact because the protein Fcalc's are done by FFT algorithm, but # the lack of closure is generally < 1%. if abs(test4a_diff) > 0.01 * abs(Fcalc_whole.data()[ikey]): n_outliers += 1 print ("%18s %8.2f /%8.2f %8.2f /%8.2f %8.2f /%8.2f %8.2f"%( key,abs(Fcalc_whole.data()[ikey]), abs(Fcalc_non_Fe.data()[ikey]), abs(diff0), abs(Fcalc_FE1.data()[ikey]), abs(diff1), abs(Fcalc_FE2.data()[ikey]), abs(test4a_diff) )) print("Test 4a N_outliers: %d out of %d"%(n_outliers, len(test4a.indices()))) assert (n_outliers / len(test4a.indices())) < 0.01 # subpoint 4b. Fmodel(whole pdb) = Fbulk + Fcalc(non-Fe) + Fcalc(FE1) + Fcalc(FE2) test4b = Fmodel_whole.customized_copy(data = Fmodel_whole.data() - Fbulk.data() - Fcalc_non_Fe.data() - Fcalc_FE1.data() - Fcalc_FE2.data()) n_outliers = 0 for ikey,key in enumerate(test4b.indices()): test4b_diff = test4b.data()[ikey] #It's not exact because the protein Fcalc's are done by FFT algorithm, but # the lack of closure is generally < 1%. if abs(test4b_diff) > 0.01 * abs(Fmodel_whole.data()[ikey]): n_outliers += 1 print ("%18s %8.2f / %8.2f"%(key,abs(Fmodel_whole.data()[ikey]), abs(test4b_diff) )) print("Test 4b N_outliers: %d out of %d"%(n_outliers, len(test4b.indices()))) assert (n_outliers / len(test4b.indices())) < 0.01 print("Complex arithmetic validates OK") # section 5. Repeat the section 4 assertions, but # section 5a. Assign different FE1(f',f") and FE2(f',f") & calculate Fmodel(whole) # section 5b. Assemble the same answer but with explicit Python-coded Fcalcs(FE1,FE2) table = FE1_model print(table.fp_fdp_at_wavelength(angstroms = 12398.425/energy)) new_FE1 = george_sherrell_proxy(-5,9) new_FE2 = george_sherrell_proxy(-4,8) GF_whole.reset_specific_at_energy(label_has="FE1",tables=new_FE1,newvalue=energy) GF_whole.reset_specific_at_energy(label_has="FE2",tables=new_FE2,newvalue=energy) f_container = GF_whole.get_fmodel() Fmodel_whole_new_fpfdp = f_container.f_model GF_FE1.reset_specific_at_energy(label_has="FE1",tables=new_FE1,newvalue=energy) f_container = GF_FE1.get_fmodel() Fcalc_FE1 = f_container.fmodel.f_calc() GF_FE2.reset_specific_at_energy(label_has="FE2",tables=new_FE2,newvalue=energy) f_container = GF_FE2.get_fmodel() Fcalc_FE2 = f_container.fmodel.f_calc() MS = Fmodel_whole_new_fpfdp.set() # avoid having to repeatedly calculate indices CS = Fmodel_whole_new_fpfdp.crystal_symmetry() # same here ALGO = structure_factors.from_scatterers(crystal_symmetry=CS, d_min=GF_whole.params2.high_resolution) from_scatterers_direct_fe1 = ALGO(xray_structure=GF_FE1.xray_structure, miller_set=MS,algorithm="direct") Fcalc_FE1_dir = from_scatterers_direct_fe1.f_calc() from_scatterers_direct_fe2 = ALGO(xray_structure=GF_FE2.xray_structure, miller_set=MS,algorithm="direct") Fcalc_FE2_dir = from_scatterers_direct_fe2.f_calc() for ikey,key in enumerate(MS.indices()): #print ("%18s %8.2f %8.2f"%(key,abs(Fcalc_FE1.data()[ikey]-Fcalc_FE1_dir.data()[ikey]), # abs(Fcalc_FE2.data()[ikey]-Fcalc_FE2_dir.data()[ikey]) #)) assert abs(Fcalc_FE1.data()[ikey]-Fcalc_FE1_dir.data()[ikey])==0. assert abs(Fcalc_FE2.data()[ikey]-Fcalc_FE2_dir.data()[ikey])==0. print("Test 5b, calculation with modified fp fdp (low-level interface) validates OK") """ Here we actually develop the Python code to give the A+iB for FE1 and FE2. looks like we should develop a new direct class to calculate F, dF/dfp, dF/dfdp all in one swoop, within a C++ extension module. Use this extension to calculate Fcalc for FE1,FE2, instead of the conventional formalism. actually, gradients direct already exists. So is it possible to just wrap existing functions at a low-enough level to get the answer? see tst_xray for usage. """ # section 6. Validate analytical derivatives' consistency with finite differences, # using correlation coefficient as a score. # section 6a. Get analytical derivatives from a low-level interface gradient_flags=xray.structure_factors.gradient_flags( site=False, u_iso=False, u_aniso=False, occupancy=False, fp=True, fdp=True) xray.set_scatterer_grad_flags(scatterers = GF_FE1.xray_structure.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp) xray.set_scatterer_grad_flags(scatterers = GF_FE2.xray_structure.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp) print ("gradients") sf1 = xray.ext.each_hkl_gradients_direct( MS.unit_cell(), MS.space_group(), MS.indices(), GF_FE1.xray_structure.scatterers(), None, GF_FE1.xray_structure.scattering_type_registry(), GF_FE1.xray_structure.site_symmetry_table(), 0) sf2 = xray.ext.each_hkl_gradients_direct( MS.unit_cell(), MS.space_group(), MS.indices(), GF_FE2.xray_structure.scatterers(), None, GF_FE2.xray_structure.scattering_type_registry(), GF_FE2.xray_structure.site_symmetry_table(), 0) print("Test 6a, Get analytical derivatives from a low-level interface") for excursion in [0.0001,0.001,0.01]: # finite differences incr_FE1_fp = george_sherrell_proxy(-5+excursion,9) incr_FE1_fdp = george_sherrell_proxy(-5,9+excursion) GF_FE1.reset_specific_at_energy(label_has="FE1",tables=incr_FE1_fp,newvalue=energy) f_container = GF_FE1.get_fmodel() incr_Fcalc_FE1_fp = f_container.fmodel.f_calc() GF_FE1.reset_specific_at_energy(label_has="FE1",tables=incr_FE1_fdp,newvalue=energy) f_container = GF_FE1.get_fmodel() incr_Fcalc_FE1_fdp = f_container.fmodel.f_calc() #analytical fp_analytical = Fcalc_FE1.data() + sf1.d_fcalc_d_fp()*excursion fdp_analytical = Fcalc_FE1.data() + sf1.d_fcalc_d_fdp()*excursion diffs_fp = fp_analytical - incr_Fcalc_FE1_fp.data() diffs_fdp = fdp_analytical - incr_Fcalc_FE1_fdp.data() #validation RMSDfp = math.sqrt( flex.sum( flex.pow( flex.abs(diffs_fp) ,2 ) ) ) RMSDfdp = math.sqrt(flex.sum(flex.pow( flex.abs(diffs_fdp) ,2))) print ("with excursion %16.14f the RMSD are fp %16.14f and fdp %16.14f"%(excursion,RMSDfp,RMSDfdp)) assert RMSDfp < 1.E-10 assert RMSDfdp < 1.E-10 print("Test 6b, Validate analytical vs. finite differences")
def __init__( self, structure, restraints_manager, temperature=300, protein_thermostat=False, n_steps=200, time_step=0.0005, initial_velocities_zero_fraction=0, vxyz=None, interleaved_minimization_params=None, n_print=20, fmodel=None, xray_target_weight=None, chem_target_weight=None, shift_update=0.0, xray_structure_last_updated=None, xray_gradient=None, reset_velocities=True, stop_cm_motion=False, update_f_calc=True, er_data=None, log=None, stop_at_diff=None, verbose=-1, ): adopt_init_args(self, locals()) assert self.n_print > 0 assert self.temperature >= 0.0 assert self.n_steps >= 0 assert self.time_step >= 0.0 assert self.log is not None or self.verbose < 1 xray.set_scatterer_grad_flags(scatterers=self.structure.scatterers(), site=True) self.structure_start = self.structure.deep_copy_scatterers() self.k_boltz = boltzmann_constant_akma self.ekcm = 0.0 self.timfac = akma_time_as_pico_seconds self.weights = self.structure.atomic_weights() if vxyz is None: self.vxyz = flex.vec3_double(self.weights.size(), (0, 0, 0)) else: self.vxyz = vxyz if self.er_data is not None and self.er_data.velocities is not None: self.vxyz = self.er_data.velocities if self.er_data is not None: self.er_data.geo_grad_rms = 0 self.er_data.xray_grad_rms = 0 if self.fmodel is not None: if self.er_data is None: self.fmodel_copy = self.fmodel.deep_copy() else: self.fmodel_copy = self.fmodel if self.er_data.fix_scale_factor is not None: self.fmodel_copy.set_scale_switch = self.er_data.fix_scale_factor # self.target_functor = self.fmodel_copy.target_functor() assert self.chem_target_weight is not None assert self.xray_target_weight is not None if self.xray_gradient is None: self.xray_gradient = self.xray_grads() # imp = self.interleaved_minimization_params self.interleaved_minimization_flag = imp is not None and imp.number_of_iterations > 0 if self.interleaved_minimization_flag: assert imp.time_step_factor > 0 self.time_step *= imp.time_step_factor if "bonds" not in imp.restraints: raise Sorry('Invalid choice: %s.restraints: "bonds" must always be included.' % imp.__phil_path__()) self.interleaved_minimization_angles = "angles" in imp.restraints else: self.interleaved_minimization_angles = False # self.tstep = self.time_step / self.timfac self.show_gradient_rms = False # XXX debug option # self()
def exercise(space_group_info, n_elements=10, table="wk1995", d_min=2.0, k_sol=0.35, b_sol=45.0, b_cart=None, quick=False, verbose=0): xray_structure = random_structure.xray_structure( space_group_info=space_group_info, elements=(("O", "N", "C") * (n_elements // 3 + 1))[:n_elements], volume_per_atom=100, min_distance=1.5, general_positions_only=True, random_u_iso=False, random_occupancy=False) xray_structure.scattering_type_registry(table=table) sg = xray_structure.space_group() uc = xray_structure.unit_cell() u_cart_1 = adptbx.random_u_cart(u_scale=5, u_min=5) u_star_1 = adptbx.u_cart_as_u_star(uc, u_cart_1) b_cart = adptbx.u_star_as_u_cart(uc, sg.average_u_star(u_star=u_star_1)) for anomalous_flag in [False, True]: scatterers = xray_structure.scatterers() if (anomalous_flag): assert scatterers.size() >= 7 for i in [1, 7]: scatterers[i].fp = -0.2 scatterers[i].fdp = 5 have_non_zero_fdp = True else: for i in [1, 7]: scatterers[i].fp = 0 scatterers[i].fdp = 0 have_non_zero_fdp = False f_obs = abs( xray_structure.structure_factors( d_min=d_min, anomalous_flag=anomalous_flag, cos_sin_table=sfg_params.cos_sin_table, algorithm=sfg_params.algorithm).f_calc()) f_obs_comp = f_obs.structure_factors_from_scatterers( xray_structure=xray_structure, algorithm=sfg_params.algorithm, cos_sin_table=sfg_params.cos_sin_table).f_calc() f_obs = abs(f_obs_comp) flags = f_obs.generate_r_free_flags(fraction=0.1, max_free=99999999) #flags = flags.array(data = flex.bool(f_obs.data().size(), False)) xrs = xray_structure.deep_copy_scatterers() xrs.shake_sites_in_place(rms_difference=0.3) for target in mmtbx.refinement.targets.target_names: if target == "mli": continue if (quick): if (target not in ["ls_wunit_k1", "ml", "mlhl", "ml_sad"]): continue if (target == "mlhl"): if (have_non_zero_fdp): continue # XXX gradients not correct! experimental_phases = generate_random_hl(miller_set=f_obs) else: experimental_phases = None if (target == "ml_sad" and (not anomalous_flag or mmtbx.refinement.targets.phaser is None)): continue print(" ", target) xray.set_scatterer_grad_flags(scatterers=xrs.scatterers(), site=True) ss = 1. / flex.pow2(f_obs.d_spacings().data()) / 4. u_star = adptbx.u_cart_as_u_star(f_obs.unit_cell(), adptbx.b_as_u(b_cart)) k_anisotropic = mmtbx.f_model.ext.k_anisotropic( f_obs.indices(), u_star) k_mask = mmtbx.f_model.ext.k_mask(ss, k_sol, b_sol) fmodel = mmtbx.f_model.manager( xray_structure=xrs, f_obs=f_obs, r_free_flags=flags, target_name=target, abcd=experimental_phases, sf_and_grads_accuracy_params=sfg_params, k_mask=k_mask, k_anisotropic=k_anisotropic, mask_params=masks.mask_master_params.extract()) fmodel.update_xray_structure(xray_structure=xrs, update_f_calc=True, update_f_mask=True) xray.set_scatterer_grad_flags( scatterers=fmodel.xray_structure.scatterers(), site=True) fmodel.update_xray_structure(update_f_calc=True) t_f = fmodel.target_functor() t_f.prepare_for_minimization() gs = t_f(compute_gradients=True).d_target_d_site_cart().as_double() gfd = finite_differences_site(target_functor=t_f) cc = flex.linear_correlation(gs, gfd).coefficient() if (0 or verbose): print("ana:", list(gs)) print("fin:", list(gfd)) print("rat:", [f / a for a, f in zip(gs, gfd)]) print(target, "corr:", cc, space_group_info) print() diff = gs - gfd diff /= max(1, flex.max(flex.abs(gfd))) tolerance = 1.2e-5 assert approx_equal(abs(flex.min(diff)), 0.0, tolerance) assert approx_equal(abs(flex.mean(diff)), 0.0, tolerance) assert approx_equal(abs(flex.max(diff)), 0.0, tolerance) assert approx_equal(cc, 1.0, tolerance) fmodel.model_error_ml()
def get_weight(fmodel=None, restraints_manager=None, sites=None, transformations=None, u_iso=None, ncs_restraints_group_list=None, refine_selection=None, minimized_obj=None): """ Calculates weights for refinements by slightly shaking the minimized parameters and taking the ratio: (restraint manager grad norm / parameters gradient norm) When calling this function during refinement macro cycle, the minimized object, "minimized_obj" , may contains fmodel, restraints_manager, refinement type info (sites, transformations, u_iso) and ncs_restraints_group_list. Args: fmodel : F-model object restraints_manager: Restraints manager object sites (bool): Refine by sites u_iso (bool): Refine using u_iso transformations (bool): Refine using transformations rotations, translations (matrix objects): ncs_restraints_group_list: list of ncs_restraint_group objects refine_selection (flex.size_t): selection of all ncs related copies and non ncs related parts to be included in selection (to be refined) minimized_obj:Minimization object containing all the other parameters above Returns: weight (int): Example: >>>get_weight(minimized_obj=minimized_obj) or >>>get_weight(fmodel=fmodel, restraints_manager=grm, sites=sites, transformations=transformations, u_iso=u_iso, ncs_restraints_group_list=ncs_restraints_group_list, refine_selection=refine_selection) """ grm = restraints_manager extended_ncs_selection = None # extract parameters from minimized_obj if minimized_obj: mo = minimized_obj fmodel = mo.fmodel grm = mo.grm sites = mo.sites transformations = mo.transformations u_iso = mo.u_iso ncs_restraints_group_list = mo.ncs_restraints_group_list # del: consider deleting the code below # if hasattr(mo,'extended_ncs_selection'): # extended_ncs_selection = mo.extended_ncs_selection # if not extended_ncs_selection: # extended_ncs_selection = get_extended_ncs_selection( # ncs_restraints_group_list=ncs_restraints_group_list, # refine_selection=refine_selection) # make sure sufficient input is provided assert bool(fmodel), 'F-model is not provided' assert bool(grm), 'F-restraints_manager is not provided' assert [sites,transformations,u_iso].count(True)==1, 'Refinement type Error' # have_transforms = ncs_restraints_group_list != [] fmdc = fmodel.deep_copy() if sites: fmdc.xray_structure.shake_sites_in_place(mean_distance=0.3) elif u_iso: fmdc.xray_structure.shake_adp() elif transformations and have_transforms: x = concatenate_rot_tran( ncs_restraints_group_list = ncs_restraints_group_list) x = shake_transformations( x = x, shake_angles_sigma=0.035, shake_translation_sigma=0.5) fmdc.update_xray_structure(xray_structure = fmdc.xray_structure, update_f_calc=True) fmdc.xray_structure.scatterers().flags_set_grads(state=False) if sites or transformations: xray.set_scatterer_grad_flags( scatterers = fmdc.xray_structure.scatterers(), site = True) # fmodel gradients gxc = flex.vec3_double(fmdc.one_time_gradients_wrt_atomic_parameters( site = True).packed()) # restraints manager, energy sites gradients gc = grm.energies_sites( sites_cart = fmdc.xray_structure.sites_cart(), compute_gradients = True).gradients elif u_iso: # Create energies_site gradient, to create geometry_restraints_manager # plain_pair_sym_table needed for the energies_adp_iso import mmtbx.refinement.adp_refinement temp = mmtbx.refinement.adp_refinement.adp_restraints_master_params iso_restraints = temp.extract().iso gc = grm.energies_sites( sites_cart = fmdc.xray_structure.sites_cart(), compute_gradients = True).gradients xray.set_scatterer_grad_flags( scatterers = fmdc.xray_structure.scatterers(), u_iso = True) # fmodel gradients gxc = fmdc.one_time_gradients_wrt_atomic_parameters( u_iso = True).as_double() # manager restraints, energy sites gradients gc = grm.energies_adp_iso( xray_structure = fmdc.xray_structure, parameters = iso_restraints, use_u_local_only = iso_restraints.use_u_local_only, use_hd = False, compute_gradients = True).gradients if transformations and have_transforms: # Apply NCS relations to gradients gxc = compute_transform_grad( grad_wrt_xyz = gxc.as_double(), ncs_restraints_group_list = ncs_restraints_group_list, xyz_asu = fmdc.xray_structure.sites_cart(), x = x) gc = compute_transform_grad( grad_wrt_xyz = gc.as_double(), ncs_restraints_group_list = ncs_restraints_group_list, xyz_asu = fmdc.xray_structure.sites_cart(), x = x) weight = 1. gc_norm = gc.norm() gxc_norm = gxc.norm() if(gxc_norm != 0.0): weight = gc_norm / gxc_norm weight =min(weight,1e6) # limit the weight max value return weight
def exercise(target_functor, data_type, parameter_name, space_group_info, anomalous_flag, cartesian_flag, n_elements=9, d_min=2.5, shake_sigma=0.25, test_hard=True, verbose=0): assert data_type == 'F' or data_type == 'F^2' if (data_type == 'F^2' and not target_functor == xray.unified_least_squares_residual): return if (parameter_name != "site" and cartesian_flag == True): return if (parameter_name == "fdp" and not anomalous_flag): return structure_ideal = random_structure.xray_structure( space_group_info, elements=(("O", "N", "C") * (n_elements))[:n_elements], volume_per_atom=100, random_f_prime_d_min=d_min, random_f_double_prime=anomalous_flag, use_u_aniso=True, use_u_iso=False, random_u_cart_scale=0.3, random_u_iso=False, random_occupancy=True) if (parameter_name in ["u_star", "u_iso"]): shake_sigma = shake_sigma / 2. random_structure.random_modify_adp_and_adp_flags( scatterers=structure_ideal.scatterers(), random_u_iso_scale=0.3, random_u_iso_min=0.0, parameter_name=parameter_name) rnd_f_calc = structure_ideal.structure_factors( anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct").f_calc() if data_type == 'F': y_obs = abs(rnd_f_calc) elif data_type == 'F^2': y_obs = rnd_f_calc.norm() y_obs.set_observation_type_xray_intensity() structure_shake = structure_ideal.random_modify_parameters( parameter_name, shake_sigma, vary_z_only=False) assert tuple(structure_ideal.special_position_indices()) \ == tuple(structure_shake.special_position_indices()) target_ftor = target_functor(y_obs) for structure in (structure_ideal, structure_shake)[:]: #SWITCH f_calc = y_obs.structure_factors_from_scatterers( xray_structure=structure, algorithm="direct").f_calc() target_result = target_ftor(f_calc, compute_derivatives=True) if (structure == structure_ideal): assert abs(target_result.target()) < 1.e-5 gradient_flags = xray.structure_factors.gradient_flags( site=(parameter_name == "site" or random.choice((False, True))), u_iso=(parameter_name == "u_iso" or random.choice((False, True))), u_aniso=(parameter_name == "u_star" or random.choice((False, True))), occupancy=(parameter_name == "occupancy" or random.choice( (False, True))), fp=(parameter_name == "fp" or random.choice((False, True))), fdp=(parameter_name == "fdp" or (anomalous_flag and random.choice( (False, True))))) xray.set_scatterer_grad_flags(scatterers=structure.scatterers(), site=gradient_flags.site, u_iso=gradient_flags.u_iso, u_aniso=gradient_flags.u_aniso, occupancy=gradient_flags.occupancy, fp=gradient_flags.fp, fdp=gradient_flags.fdp) grad_flags_counts = xray.scatterer_grad_flags_counts( structure.scatterers()) sf = xray.structure_factors.gradients_direct( xray_structure=structure, u_iso_refinable_params=None, miller_set=y_obs, d_target_d_f_calc=target_result.derivatives(), n_parameters=0) if (parameter_name == "site"): d_analytical = sf.d_target_d_site_frac() if (cartesian_flag ): # average d_analytical or d_numerical, but not both structure_ideal.apply_special_position_ops_d_target_d_site( d_analytical) if (cartesian_flag): d_analytical = sf.d_target_d_site_cart() d_numerical = finite_differences_site(cartesian_flag, target_ftor, structure) if (not cartesian_flag ): # aver. d_analytical or d_numerical, but not both structure_ideal.apply_special_position_ops_d_target_d_site( d_numerical) elif (parameter_name == "u_star" and grad_flags_counts.use_u_aniso > 0): d_analytical = sf.d_target_d_u_star() d_numerical = finite_differences_u_star(target_ftor, structure) else: if (parameter_name == "occupancy"): d_analytical = sf.d_target_d_occupancy() elif (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0): d_analytical = sf.d_target_d_u_iso() elif (parameter_name == "fp"): d_analytical = sf.d_target_d_fp() elif (parameter_name == "fdp"): d_analytical = sf.d_target_d_fdp() else: raise RuntimeError d_numerical = finite_differences_scalar(parameter_name, target_ftor, structure) linear_regression_test(d_analytical, d_numerical, test_hard, verbose=verbose) if (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0): u_iso_refinable_params = flex.double() for scatterer in structure.scatterers(): scatterer.flags.set_tan_u_iso(True) scatterer.flags.set_grad_u_iso(gradient_flags.u_iso) param = random.randint(90, 120) scatterer.flags.param = param value = math.tan(scatterer.u_iso * math.pi / adptbx.b_as_u(param) - math.pi / 2) u_iso_refinable_params.append(value) sf = xray.structure_factors.gradients_direct( xray_structure=structure, u_iso_refinable_params=u_iso_refinable_params, miller_set=y_obs, d_target_d_f_calc=target_result.derivatives(), n_parameters=0) d_analytical = sf.d_target_d_u_iso() d_numerical = finite_differences_scalar("tan_u_iso", target_ftor, structure) linear_regression_test(d_analytical, d_numerical, test_hard, verbose=verbose)
def __init__(self, fmodel, ncs_transformations_object=None, ncs_atom_selection = None, run_finite_grad_differences_test = False, max_iterations=100, sites = False, u_iso = False): """Implementing strict NCS to refinement minimization Arguments: fmodel : fmodel of the complete ASU ncs_transformation_object : information on the NCS to ASU transformations and chains. A multimer object ncs_atom_selection : boolean array for selection of atoms in the NCS. A flex bool array """ self.fmodel = fmodel self.fmodel.xray_structure.scatterers().flags_set_grads(state=False) self.x_target_functor = self.fmodel.target_functor() self.sites = sites self.u_iso = u_iso self.ncs_to_asu = ncs_transformations_object self.run_finite_grad_differences_test = run_finite_grad_differences_test if run_finite_grad_differences_test: # perform gradient calc test self.buffer_max_grad = flex.double() self.buffer_calc_grad = flex.double() # xray structure of NCS chains for self.x ncs_fmodel_xrs = self.fmodel.xray_structure.select(ncs_atom_selection) if(self.sites): self.x = ncs_fmodel_xrs.sites_cart().as_double() if(self.u_iso): assert ncs_fmodel_xrs.scatterers().size() == \ ncs_fmodel_xrs.use_u_iso().count(True) self.x = ncs_fmodel_xrs.extract_u_iso_or_u_equiv() # Use all scatterers for gradient calculations if(self.sites): xray.set_scatterer_grad_flags( scatterers = self.fmodel.xray_structure.scatterers(), site = True) if(self.u_iso): sel = flex.bool( self.fmodel.xray_structure.scatterers().size(), True).iselection() self.fmodel.xray_structure.scatterers().flags_set_grad_u_iso( iselection = sel) self.minimizer = scitbx.lbfgs.run( target_evaluator=self, termination_params=scitbx.lbfgs.termination_parameters( max_iterations=max_iterations), exception_handling_params=scitbx.lbfgs.exception_handling_parameters( ignore_line_search_failed_rounding_errors=True, ignore_line_search_failed_step_at_lower_bound=True, ignore_line_search_failed_maxfev=True)) self.fmodel.xray_structure.tidy_us() self.fmodel.xray_structure.apply_symmetry_sites() self.fmodel.update_xray_structure( xray_structure = self.fmodel.xray_structure, update_f_calc = True) self.tested = 0 if run_finite_grad_differences_test: if self.buffer_max_grad: print 'compare max_grad to calc_grad' for a,f in zip(self.buffer_max_grad, self.buffer_calc_grad): print '{0:10.5f} {1:10.5f} delta = {2:10.5f}'.format(a,f,abs(a-f)) print '-'*45 diff = flex.abs(self.buffer_max_grad - self.buffer_calc_grad) s = diff < 1.e-3 if(s.size()>0 and s.count(True)*100./s.size()>50): self.tested += 1
def exercise_constrained_lbfgs(xray_structure, constraints_list, t_celsius, d_min=0.5, shake_sites_rmsd=0.5, shake_u_iso_spread=0, shake_u_aniso_spread=0, grad_site=True, grad_u_iso=True, grad_u_aniso=False, grad_occupancy=False, grad_fp_fdp=False, lbfgs_m=5, lbfgs_max_iterations=1000, verbose=0): xs = xray_structure xray.set_scatterer_grad_flags(scatterers=xs.scatterers(), site=grad_site, u_iso=grad_u_iso, u_aniso=grad_u_aniso, occupancy=grad_occupancy, fp=grad_fp_fdp, fdp=grad_fp_fdp, tan_u_iso=False, param=0) xs0 = xs.deep_copy_scatterers() mi = xs0.build_miller_set(anomalous_flag=False, d_min=d_min) fo_sq = mi.structure_factors_from_scatterers( xs0, algorithm="direct").f_calc().norm() fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1)) fo_sq.set_observation_type_xray_intensity() y_obs = fo_sq #y_obs = fo_sq.f_sq_as_f() if grad_site: xs.shake_sites_in_place(rms_difference=shake_sites_rmsd) if not grad_u_aniso: shake_u_aniso_spread = 0 if not grad_u_iso: shake_u_iso_spread = 0 if grad_u_aniso or grad_u_iso: xs.shake_adp(spread=shake_u_iso_spread, aniso_spread=shake_u_aniso_spread) xs1 = xs.deep_copy_scatterers() core_params = scitbx.lbfgs.core_parameters(m=lbfgs_m, maxfev=100, xtol=1e-5) connectivity_table = smtbx.utils.connectivity_table(xs0) if constraints_list is None: from smtbx.development import generate_hydrogen_constraints constraints_list = generate_hydrogen_constraints( structure=xs0, connectivity_table=connectivity_table) reparametrisation = constraints.reparametrisation( xs, constraints_list, connectivity_table, temperature=t_celsius) lbfgs_termination_params=scitbx.lbfgs.termination_parameters( traditional_convergence_test=False, drop_convergence_test_max_drop_eps=1.e-20, drop_convergence_test_iteration_coefficient=1, min_iterations=500, max_iterations=lbfgs_max_iterations) minimizer = lbfgs( target_functor=xray.target_functors.unified_least_squares_residual(y_obs), xray_structure=xs, reparametrisation=reparametrisation, structure_factor_algorithm="direct", lbfgs_termination_params=lbfgs_termination_params, lbfgs_core_params=core_params, reference_structure=xs0, verbose=verbose) if verbose > 0: print "Total parameters: ", xs.n_parameters() print "Independent parameters: ", reparametrisation.n_independents print "Reference model: " xs0.show_angles(distance_cutoff=1.5) print print "Starting model: " xs1.show_angles(distance_cutoff=1.5) print print "Refined model: " xs.show_angles(distance_cutoff=1.5) print print "n_iter, n_fun: ", minimizer.minimizer.iter(), minimizer.minimizer.nfun() h_selection = xs.element_selection('H') diff = xray.meaningful_site_cart_differences( xs0.select(h_selection, negate=True), xs.select(h_selection, negate=True)) #diff = xray.meaningful_site_cart_differences(xs0, xs) #assert diff.max_absolute() < 1e-3 if verbose > 0: diff.show() print assert diff.max_absolute() < 2e-2, diff.max_absolute() diff = xray.meaningful_site_cart_differences(xs0, xs) if verbose > 0: diff.show() print # XXX why does this tolerance have to be so high? assert diff.max_absolute() < 0.5, diff.max_absolute()
def exercise_negative_parameters(verbose=0): structure_default = xray.structure( crystal_symmetry = crystal.symmetry( unit_cell=((10,13,17,75,80,85)), space_group_symbol="P 1"), scatterers=flex.xray_scatterer([ xray.scatterer(label="C", site=(0,0,0), u=0.25)])) negative_gaussian = eltbx.xray_scattering.gaussian((1,2), (2,3), -4) for i_trial in xrange(7): structure = structure_default.deep_copy_scatterers() scatterer = structure.scatterers()[0] if (i_trial == 1): scatterer.occupancy *= -1 elif (i_trial == 2): structure.scattering_type_registry(custom_dict={"C": negative_gaussian}) elif (i_trial == 3): scatterer.u_iso *= -1 elif (i_trial == 4): u_cart = adptbx.random_u_cart(u_scale=1, u_min=-1.1) assert max(adptbx.eigenvalues(u_cart)) < 0 u_star = adptbx.u_cart_as_u_star(structure.unit_cell(), u_cart) scatterer.u_star = u_star scatterer.flags.set_use_u_aniso_only() elif (i_trial == 5): scatterer.fp = -10 elif (i_trial == 6): scatterer.fp = -3 f_direct = structure.structure_factors( d_min=1, algorithm="direct", cos_sin_table=False).f_calc() f_fft = structure.structure_factors( d_min=1, algorithm="fft", quality_factor=1.e8, wing_cutoff=1.e-10).f_calc() if (i_trial == 2): assert negative_gaussian.at_d_star_sq(f_fft.d_star_sq().data()).all_lt(0) if (i_trial in [5,6]): f = structure.scattering_type_registry().gaussian_not_optional( scattering_type="C").at_d_star_sq(f_fft.d_star_sq().data()) if (i_trial == 5): assert flex.max(f) + scatterer.fp < 0 else: assert flex.max(f) + scatterer.fp > 0 assert flex.min(f) + scatterer.fp < 0 cc = flex.linear_correlation( abs(f_direct).data(), abs(f_fft).data()).coefficient() if (cc < 0.999): raise AssertionError("i_trial=%d, correlation=%.6g" % (i_trial, cc)) elif (0 or verbose): print "correlation=%.6g" % cc # # very simple test of gradient calculations with negative parameters structure_factor_gradients = \ cctbx.xray.structure_factors.gradients( miller_set=f_direct, cos_sin_table=False) target_functor = xray.target_functors.intensity_correlation( f_obs=abs(f_direct)) target_result = target_functor(f_fft, True) xray.set_scatterer_grad_flags(scatterers = structure.scatterers(), site = True, u_iso = True, u_aniso = True, occupancy = True, fp = True, fdp = True) for algorithm in ["direct", "fft"]: grads = structure_factor_gradients( xray_structure=structure, u_iso_refinable_params=None, miller_set=f_direct, d_target_d_f_calc=target_result.derivatives(), n_parameters = structure.n_parameters(), algorithm=algorithm).packed()
def __init__(self, structure, restraints_manager, temperature=300, protein_thermostat=False, n_steps=200, time_step=0.0005, initial_velocities_zero_fraction=0, vxyz=None, interleaved_minimization_params=None, n_print=20, fmodel=None, xray_target_weight=None, chem_target_weight=None, shift_update=0.0, xray_structure_last_updated=None, xray_gradient=None, reset_velocities=True, stop_cm_motion=False, update_f_calc=True, er_data=None, log=None, stop_at_diff=None, verbose=-1): adopt_init_args(self, locals()) assert self.n_print > 0 assert self.temperature >= 0.0 assert self.n_steps >= 0 assert self.time_step >= 0.0 assert self.log is not None or self.verbose < 1 xray.set_scatterer_grad_flags(scatterers=self.structure.scatterers(), site=True) self.structure_start = self.structure.deep_copy_scatterers() self.k_boltz = boltzmann_constant_akma self.ekcm = 0.0 self.timfac = akma_time_as_pico_seconds self.weights = self.structure.atomic_weights() if (vxyz is None): self.vxyz = flex.vec3_double(self.weights.size(), (0, 0, 0)) else: self.vxyz = vxyz if (self.er_data is not None and self.er_data.velocities is not None): self.vxyz = self.er_data.velocities if self.er_data is not None: self.er_data.geo_grad_rms = 0 self.er_data.xray_grad_rms = 0 if (self.fmodel is not None): if self.er_data is None: self.fmodel_copy = self.fmodel.deep_copy() else: self.fmodel_copy = self.fmodel if self.er_data.fix_scale_factor is not None: self.fmodel_copy.set_scale_switch = self.er_data.fix_scale_factor # self.target_functor = self.fmodel_copy.target_functor() assert self.chem_target_weight is not None assert self.xray_target_weight is not None if (self.xray_gradient is None): self.xray_gradient = self.xray_grads() # imp = self.interleaved_minimization_params self.interleaved_minimization_flag = (imp is not None and imp.number_of_iterations > 0) if (self.interleaved_minimization_flag): assert imp.time_step_factor > 0 self.time_step *= imp.time_step_factor if ("bonds" not in imp.restraints): raise Sorry( 'Invalid choice: %s.restraints: "bonds" must always be included.' % imp.__phil_path__()) self.interleaved_minimization_angles = "angles" in imp.restraints else: self.interleaved_minimization_angles = False # self.tstep = self.time_step / self.timfac self.show_gradient_rms = False # XXX debug option # self()
def exercise_constrained_lbfgs(xray_structure, constraints_list, t_celsius, d_min=0.5, shake_sites_rmsd=0.5, shake_u_iso_spread=0, shake_u_aniso_spread=0, grad_site=True, grad_u_iso=True, grad_u_aniso=False, grad_occupancy=False, grad_fp_fdp=False, lbfgs_m=5, lbfgs_max_iterations=1000, verbose=0): xs = xray_structure xray.set_scatterer_grad_flags(scatterers=xs.scatterers(), site=grad_site, u_iso=grad_u_iso, u_aniso=grad_u_aniso, occupancy=grad_occupancy, fp=grad_fp_fdp, fdp=grad_fp_fdp, tan_u_iso=False, param=0) xs0 = xs.deep_copy_scatterers() mi = xs0.build_miller_set(anomalous_flag=False, d_min=d_min) fo_sq = mi.structure_factors_from_scatterers( xs0, algorithm="direct").f_calc().norm() fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1)) fo_sq.set_observation_type_xray_intensity() y_obs = fo_sq #y_obs = fo_sq.f_sq_as_f() if grad_site: xs.shake_sites_in_place(rms_difference=shake_sites_rmsd) if not grad_u_aniso: shake_u_aniso_spread = 0 if not grad_u_iso: shake_u_iso_spread = 0 if grad_u_aniso or grad_u_iso: xs.shake_adp(spread=shake_u_iso_spread, aniso_spread=shake_u_aniso_spread) xs1 = xs.deep_copy_scatterers() core_params = scitbx.lbfgs.core_parameters(m=lbfgs_m, maxfev=100, xtol=1e-5) connectivity_table = smtbx.utils.connectivity_table(xs0) if constraints_list is None: from smtbx.development import generate_hydrogen_constraints constraints_list = generate_hydrogen_constraints( structure=xs0, connectivity_table=connectivity_table) reparametrisation = constraints.reparametrisation(xs, constraints_list, connectivity_table, temperature=t_celsius) lbfgs_termination_params = scitbx.lbfgs.termination_parameters( traditional_convergence_test=False, drop_convergence_test_max_drop_eps=1.e-20, drop_convergence_test_iteration_coefficient=1, min_iterations=500, max_iterations=lbfgs_max_iterations) minimizer = lbfgs( target_functor=xray.target_functors.unified_least_squares_residual( y_obs), xray_structure=xs, reparametrisation=reparametrisation, structure_factor_algorithm="direct", lbfgs_termination_params=lbfgs_termination_params, lbfgs_core_params=core_params, reference_structure=xs0, verbose=verbose) if verbose > 0: print "Total parameters: ", xs.n_parameters() print "Independent parameters: ", reparametrisation.n_independents print "Reference model: " xs0.show_angles(distance_cutoff=1.5) print print "Starting model: " xs1.show_angles(distance_cutoff=1.5) print print "Refined model: " xs.show_angles(distance_cutoff=1.5) print print "n_iter, n_fun: ", minimizer.minimizer.iter( ), minimizer.minimizer.nfun() h_selection = xs.element_selection('H') diff = xray.meaningful_site_cart_differences( xs0.select(h_selection, negate=True), xs.select(h_selection, negate=True)) #diff = xray.meaningful_site_cart_differences(xs0, xs) #assert diff.max_absolute() < 1e-3 if verbose > 0: diff.show() print assert diff.max_absolute() < 2e-2, diff.max_absolute() diff = xray.meaningful_site_cart_differences(xs0, xs) if verbose > 0: diff.show() print # XXX why does this tolerance have to be so high? assert diff.max_absolute() < 0.5, diff.max_absolute()
def exercise_negative_parameters(verbose=0): structure_default = xray.structure( crystal_symmetry=crystal.symmetry(unit_cell=((10, 13, 17, 75, 80, 85)), space_group_symbol="P 1"), scatterers=flex.xray_scatterer( [xray.scatterer(label="C", site=(0, 0, 0), u=0.25)])) negative_gaussian = eltbx.xray_scattering.gaussian((1, 2), (2, 3), -4) for i_trial in range(7): structure = structure_default.deep_copy_scatterers() scatterer = structure.scatterers()[0] if (i_trial == 1): scatterer.occupancy *= -1 elif (i_trial == 2): structure.scattering_type_registry( custom_dict={"C": negative_gaussian}) elif (i_trial == 3): scatterer.u_iso *= -1 elif (i_trial == 4): u_cart = adptbx.random_u_cart(u_scale=1, u_min=-1.1) assert max(adptbx.eigenvalues(u_cart)) < 0 u_star = adptbx.u_cart_as_u_star(structure.unit_cell(), u_cart) scatterer.u_star = u_star scatterer.flags.set_use_u_aniso_only() elif (i_trial == 5): scatterer.fp = -10 elif (i_trial == 6): scatterer.fp = -3 f_direct = structure.structure_factors(d_min=1, algorithm="direct", cos_sin_table=False).f_calc() f_fft = structure.structure_factors(d_min=1, algorithm="fft", quality_factor=1.e8, wing_cutoff=1.e-10).f_calc() if (i_trial == 2): assert negative_gaussian.at_d_star_sq( f_fft.d_star_sq().data()).all_lt(0) if (i_trial in [5, 6]): f = structure.scattering_type_registry().gaussian_not_optional( scattering_type="C").at_d_star_sq(f_fft.d_star_sq().data()) if (i_trial == 5): assert flex.max(f) + scatterer.fp < 0 else: assert flex.max(f) + scatterer.fp > 0 assert flex.min(f) + scatterer.fp < 0 cc = flex.linear_correlation(abs(f_direct).data(), abs(f_fft).data()).coefficient() if (cc < 0.999): raise AssertionError("i_trial=%d, correlation=%.6g" % (i_trial, cc)) elif (0 or verbose): print("correlation=%.6g" % cc) # # very simple test of gradient calculations with negative parameters structure_factor_gradients = \ cctbx.xray.structure_factors.gradients( miller_set=f_direct, cos_sin_table=False) target_functor = xray.target_functors.intensity_correlation( f_obs=abs(f_direct)) target_result = target_functor(f_fft, True) xray.set_scatterer_grad_flags(scatterers=structure.scatterers(), site=True, u_iso=True, u_aniso=True, occupancy=True, fp=True, fdp=True) for algorithm in ["direct", "fft"]: grads = structure_factor_gradients( xray_structure=structure, u_iso_refinable_params=None, miller_set=f_direct, d_target_d_f_calc=target_result.derivatives(), n_parameters=structure.n_parameters(), algorithm=algorithm).packed()
def exercise(target_functor, data_type, parameter_name, space_group_info, anomalous_flag, cartesian_flag, n_elements=9, d_min=2.5, shake_sigma=0.25, test_hard=True, verbose=0): assert data_type == 'F' or data_type == 'F^2' if (data_type == 'F^2' and not target_functor == xray.unified_least_squares_residual): return if (parameter_name != "site" and cartesian_flag == True): return if (parameter_name == "fdp" and not anomalous_flag): return structure_ideal = random_structure.xray_structure( space_group_info, elements=(("O","N","C")*(n_elements))[:n_elements], volume_per_atom=100, random_f_prime_d_min=d_min, random_f_double_prime=anomalous_flag, use_u_aniso = True, use_u_iso = False, random_u_cart_scale = 0.3, random_u_iso = False, random_occupancy=True) if(parameter_name in ["u_star", "u_iso"]): shake_sigma = shake_sigma / 2. random_structure.random_modify_adp_and_adp_flags( scatterers = structure_ideal.scatterers(), random_u_iso_scale = 0.3, random_u_iso_min = 0.0, parameter_name = parameter_name) rnd_f_calc = structure_ideal.structure_factors( anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct").f_calc() if data_type == 'F': y_obs = abs(rnd_f_calc) elif data_type == 'F^2': y_obs = rnd_f_calc.norm() y_obs.set_observation_type_xray_intensity() structure_shake = structure_ideal.random_modify_parameters( parameter_name, shake_sigma, vary_z_only=False) assert tuple(structure_ideal.special_position_indices()) \ == tuple(structure_shake.special_position_indices()) target_ftor = target_functor(y_obs) for structure in (structure_ideal, structure_shake)[:]: #SWITCH f_calc = y_obs.structure_factors_from_scatterers( xray_structure=structure, algorithm="direct").f_calc() target_result = target_ftor(f_calc, compute_derivatives=True) if (structure == structure_ideal): assert abs(target_result.target()) < 1.e-5 gradient_flags=xray.structure_factors.gradient_flags( site=(parameter_name=="site" or random.choice((False,True))), u_iso=(parameter_name=="u_iso" or random.choice((False,True))), u_aniso=(parameter_name=="u_star" or random.choice((False,True))), occupancy=(parameter_name=="occupancy" or random.choice((False,True))), fp=(parameter_name=="fp" or random.choice((False,True))), fdp=(parameter_name=="fdp" or (anomalous_flag and random.choice((False,True))))) xray.set_scatterer_grad_flags(scatterers = structure.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp) grad_flags_counts = xray.scatterer_grad_flags_counts(structure.scatterers()) sf = xray.structure_factors.gradients_direct( xray_structure=structure, u_iso_refinable_params=None, miller_set=y_obs, d_target_d_f_calc=target_result.derivatives(), n_parameters=0) if (parameter_name == "site"): d_analytical = sf.d_target_d_site_frac() if (cartesian_flag): # average d_analytical or d_numerical, but not both structure_ideal.apply_special_position_ops_d_target_d_site(d_analytical) if (cartesian_flag): d_analytical = sf.d_target_d_site_cart() d_numerical = finite_differences_site( cartesian_flag, target_ftor, structure) if (not cartesian_flag): # aver. d_analytical or d_numerical, but not both structure_ideal.apply_special_position_ops_d_target_d_site(d_numerical) elif (parameter_name == "u_star" and grad_flags_counts.use_u_aniso > 0): d_analytical = sf.d_target_d_u_star() d_numerical = finite_differences_u_star(target_ftor, structure) else: if (parameter_name == "occupancy"): d_analytical = sf.d_target_d_occupancy() elif (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0): d_analytical = sf.d_target_d_u_iso() elif (parameter_name == "fp"): d_analytical = sf.d_target_d_fp() elif (parameter_name == "fdp"): d_analytical = sf.d_target_d_fdp() else: raise RuntimeError d_numerical = finite_differences_scalar( parameter_name, target_ftor, structure) linear_regression_test(d_analytical, d_numerical, test_hard, verbose=verbose) if (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0): u_iso_refinable_params = flex.double() for scatterer in structure.scatterers(): scatterer.flags.set_tan_u_iso(True) scatterer.flags.set_grad_u_iso(gradient_flags.u_iso) param = random.randint(90,120) scatterer.flags.param= param value = math.tan(scatterer.u_iso*math.pi/adptbx.b_as_u(param)-math.pi/2) u_iso_refinable_params.append(value) sf = xray.structure_factors.gradients_direct( xray_structure=structure, u_iso_refinable_params = u_iso_refinable_params, miller_set=y_obs, d_target_d_f_calc=target_result.derivatives(), n_parameters=0) d_analytical = sf.d_target_d_u_iso() d_numerical = finite_differences_scalar("tan_u_iso",target_ftor, structure) linear_regression_test(d_analytical, d_numerical, test_hard, verbose=verbose)
def get_weight(fmodel=None, restraints_manager=None, sites=None, transformations=None, u_iso=None, ncs_restraints_group_list=None, refine_selection=None, minimized_obj=None): """ Calculates weights for refinements by slightly shaking the minimized parameters and taking the ratio: (restraint manager grad norm / parameters gradient norm) When calling this function during refinement macro cycle, the minimized object, "minimized_obj" , may contains fmodel, restraints_manager, refinement type info (sites, transformations, u_iso) and ncs_restraints_group_list. Args: fmodel : F-model object restraints_manager: Restraints manager object sites (bool): Refine by sites u_iso (bool): Refine using u_iso transformations (bool): Refine using transformations rotations, translations (matrix objects): ncs_restraints_group_list: list of ncs_restraint_group objects refine_selection (flex.size_t): selection of all ncs related copies and non ncs related parts to be included in selection (to be refined) minimized_obj:Minimization object containing all the other parameters above Returns: weight (int): Example: >>>get_weight(minimized_obj=minimized_obj) or >>>get_weight(fmodel=fmodel, restraints_manager=grm, sites=sites, transformations=transformations, u_iso=u_iso, ncs_restraints_group_list=ncs_restraints_group_list, refine_selection=refine_selection) """ grm = restraints_manager extended_ncs_selection = None # extract parameters from minimized_obj if minimized_obj: mo = minimized_obj fmodel = mo.fmodel grm = mo.grm sites = mo.sites transformations = mo.transformations u_iso = mo.u_iso ncs_restraints_group_list = mo.ncs_restraints_group_list # del: consider deleting the code below # if hasattr(mo,'extended_ncs_selection'): # extended_ncs_selection = mo.extended_ncs_selection # if not extended_ncs_selection: # extended_ncs_selection = get_extended_ncs_selection( # ncs_restraints_group_list=ncs_restraints_group_list, # refine_selection=refine_selection) # make sure sufficient input is provided assert bool(fmodel), 'F-model is not provided' assert bool(grm), 'F-restraints_manager is not provided' assert [sites, transformations, u_iso].count(True) == 1, 'Refinement type Error' # have_transforms = ncs_restraints_group_list != [] fmdc = fmodel.deep_copy() if sites: fmdc.xray_structure.shake_sites_in_place(mean_distance=0.3) elif u_iso: fmdc.xray_structure.shake_adp() elif transformations and have_transforms: x = concatenate_rot_tran( ncs_restraints_group_list=ncs_restraints_group_list) x = shake_transformations(x=x, shake_angles_sigma=0.035, shake_translation_sigma=0.5) fmdc.update_xray_structure(xray_structure=fmdc.xray_structure, update_f_calc=True) fmdc.xray_structure.scatterers().flags_set_grads(state=False) if sites or transformations: xray.set_scatterer_grad_flags( scatterers=fmdc.xray_structure.scatterers(), site=True) # fmodel gradients gxc = flex.vec3_double( fmdc.one_time_gradients_wrt_atomic_parameters(site=True).packed()) # restraints manager, energy sites gradients gc = grm.energies_sites(sites_cart=fmdc.xray_structure.sites_cart(), compute_gradients=True).gradients elif u_iso: # Create energies_site gradient, to create geometry_restraints_manager # plain_pair_sym_table needed for the energies_adp_iso import mmtbx.refinement.adp_refinement temp = mmtbx.refinement.adp_refinement.adp_restraints_master_params iso_restraints = temp.extract().iso gc = grm.energies_sites(sites_cart=fmdc.xray_structure.sites_cart(), compute_gradients=True).gradients xray.set_scatterer_grad_flags( scatterers=fmdc.xray_structure.scatterers(), u_iso=True) # fmodel gradients gxc = fmdc.one_time_gradients_wrt_atomic_parameters( u_iso=True).as_double() # manager restraints, energy sites gradients gc = grm.energies_adp_iso( xray_structure=fmdc.xray_structure, parameters=iso_restraints, use_u_local_only=iso_restraints.use_u_local_only, use_hd=False, compute_gradients=True).gradients if transformations and have_transforms: # Apply NCS relations to gradients gxc = compute_transform_grad( grad_wrt_xyz=gxc.as_double(), ncs_restraints_group_list=ncs_restraints_group_list, xyz_asu=fmdc.xray_structure.sites_cart(), x=x) gc = compute_transform_grad( grad_wrt_xyz=gc.as_double(), ncs_restraints_group_list=ncs_restraints_group_list, xyz_asu=fmdc.xray_structure.sites_cart(), x=x) weight = 1. gc_norm = gc.norm() gxc_norm = gxc.norm() if (gxc_norm != 0.0): weight = gc_norm / gxc_norm weight = min(weight, 1e6) # limit the weight max value return weight
def exercise(target_functor, data_type, space_group_info, anomalous_flag, gradient_flags, occupancy_penalty, n_elements=9, d_min=None, shake_sigma=0.1, verbose=0,tan_u_iso=False, param = 0): assert data_type == 'F' or data_type == 'F^2' if (data_type == 'F^2' and not target_functor == xray.unified_least_squares_residual): return structure_ideal = random_structure.xray_structure( space_group_info, elements=(("O","N","C")*(n_elements))[:n_elements],#("Se",)*n_elements, volume_per_atom=200, random_u_iso=True, random_u_cart_scale=.3, random_occupancy=True, use_u_aniso = True) random_structure.random_modify_adp_and_adp_flags( scatterers = structure_ideal.scatterers(), random_u_iso_scale = 0.3, random_u_iso_min = 0.0) xray.set_scatterer_grad_flags(scatterers = structure_ideal.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp, tan_u_iso = tan_u_iso, param = param) if(0): print for sc in structure_ideal.scatterers(): print sc.flags.use_u_iso(),sc.flags.grad_u_iso(),sc.flags.use_u_aniso(),\ sc.flags.grad_u_aniso(),sc.u_iso, sc.u_star,sc.flags.tan_u_iso(),\ sc.flags.param, sc.occupancy rnd_f_calc = structure_ideal.structure_factors( anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct", cos_sin_table=True).f_calc() if data_type == "F": y_obs = abs(rnd_f_calc) elif data_type == "F^2": y_obs = rnd_f_calc.norm() y_obs.set_observation_type_xray_intensity() else: raise "Error: invalid data type: %s" % data_type if (0 or verbose): print "structure_ideal:" structure_ideal.show_summary().show_scatterers() print "n_special_positions:", \ structure_ideal.special_position_indices().size() print structure_ideal_cp = structure_ideal.deep_copy_scatterers() structure_shake = structure_ideal if (gradient_flags.site): structure_shake = structure_shake.random_modify_parameters( "site", shake_sigma) if (gradient_flags.occupancy): structure_shake = structure_shake.random_modify_parameters( "occupancy", shake_sigma) if (occupancy_penalty is not None): structure_shake.scatterers()[-1].occupancy = 0 if (gradient_flags.u_aniso): shift_u_aniso(structure_shake, 0.001) if (gradient_flags.u_iso): shift_u_iso(structure_shake, 0.1) assert tuple(structure_ideal.special_position_indices()) \ == tuple(structure_shake.special_position_indices()) if (0 or verbose): print "structure_shake:" structure_shake.show_summary().show_scatterers() print for i_trial in xrange(10): try: minimizer = xray.minimization.lbfgs( target_functor=target_functor(y_obs), xray_structure=structure_shake, occupancy_penalty=occupancy_penalty, structure_factor_algorithm="direct") except RuntimeError, e: if (str(e).find("debye_waller_factor_exp: arg_limit exceeded") < 0): raise else: break
def exercise(target_functor, data_type, space_group_info, anomalous_flag, gradient_flags, occupancy_penalty, n_elements=9, d_min=None, shake_sigma=0.1, verbose=0,tan_u_iso=False, param = 0): assert data_type == 'F' or data_type == 'F^2' if (data_type == 'F^2' and not target_functor == xray.unified_least_squares_residual): return structure_ideal = random_structure.xray_structure( space_group_info, elements=(("O","N","C")*(n_elements))[:n_elements],#("Se",)*n_elements, volume_per_atom=200, random_u_iso=True, random_u_cart_scale=.3, random_occupancy=True, use_u_aniso = True) random_structure.random_modify_adp_and_adp_flags( scatterers = structure_ideal.scatterers(), random_u_iso_scale = 0.3, random_u_iso_min = 0.0) xray.set_scatterer_grad_flags(scatterers = structure_ideal.scatterers(), site = gradient_flags.site, u_iso = gradient_flags.u_iso, u_aniso = gradient_flags.u_aniso, occupancy = gradient_flags.occupancy, fp = gradient_flags.fp, fdp = gradient_flags.fdp, tan_u_iso = tan_u_iso, param = param) if(0): print() for sc in structure_ideal.scatterers(): print(sc.flags.use_u_iso(),sc.flags.grad_u_iso(),sc.flags.use_u_aniso(),\ sc.flags.grad_u_aniso(),sc.u_iso, sc.u_star,sc.flags.tan_u_iso(),\ sc.flags.param, sc.occupancy) rnd_f_calc = structure_ideal.structure_factors( anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct", cos_sin_table=True).f_calc() if data_type == "F": y_obs = abs(rnd_f_calc) elif data_type == "F^2": y_obs = rnd_f_calc.norm() y_obs.set_observation_type_xray_intensity() else: raise "Error: invalid data type: %s" % data_type if (0 or verbose): print("structure_ideal:") structure_ideal.show_summary().show_scatterers() print("n_special_positions:", \ structure_ideal.special_position_indices().size()) print() structure_ideal_cp = structure_ideal.deep_copy_scatterers() structure_shake = structure_ideal if (gradient_flags.site): structure_shake = structure_shake.random_modify_parameters( "site", shake_sigma) if (gradient_flags.occupancy): structure_shake = structure_shake.random_modify_parameters( "occupancy", shake_sigma) if (occupancy_penalty is not None): structure_shake.scatterers()[-1].occupancy = 0 if (gradient_flags.u_aniso): shift_u_aniso(structure_shake, 0.001) if (gradient_flags.u_iso): shift_u_iso(structure_shake, 0.1) assert tuple(structure_ideal.special_position_indices()) \ == tuple(structure_shake.special_position_indices()) if (0 or verbose): print("structure_shake:") structure_shake.show_summary().show_scatterers() print() for i_trial in range(10): try: minimizer = xray.minimization.lbfgs( target_functor=target_functor(y_obs), xray_structure=structure_shake, occupancy_penalty=occupancy_penalty, structure_factor_algorithm="direct") except RuntimeError as e: if (str(e).find("debye_waller_factor_exp: arg_limit exceeded") < 0): raise else: break else: raise RuntimeError("Too many xray.minimization.lbfgs failures.") if (0 or verbose): print("first:", minimizer.first_target_value) print("final:", minimizer.final_target_value) print() assert minimizer.final_target_value < minimizer.first_target_value if (0 or verbose): print("minimized structure_shake:") structure_shake.show_summary().show_scatterers() print() f_final = y_obs.structure_factors_from_scatterers( xray_structure=structure_shake, algorithm="direct", cos_sin_table=True).f_calc() if data_type == 'F': f_final = abs(f_final) elif data_type == 'F^2': f_final = f_final.norm() c = flex.linear_correlation(y_obs.data(), f_final.data()) assert c.is_well_defined() if (0 or verbose): label = gradient_flags.string_of_true() if (anomalous_flag): label += ",anomalous" print("correlation: %10.8f" % c.coefficient(), label) print() c_coefficient = c.coefficient() if(c_coefficient <= 0.999): print(c_coefficient) if data_type == 'F': assert c_coefficient > 0.999 elif data_type == 'F^2': assert c_coefficient > 0.9