def d_target_d_site_cart(self):
manager = self.manager
xray.set_scatterer_grad_flags(
scatterers=manager.xray_structure.scatterers(),
site=True)
return flex.vec3_double(
self.gradients_wrt_atomic_parameters().packed())
def __init__(self, fmodel, max_iterations=100, sites=False, u_iso=False):
self.fmodel = fmodel
self.fmodel.xray_structure.scatterers().flags_set_grads(state=False)
self.x_target_functor = self.fmodel.target_functor()
self.sites = sites
self.u_iso = u_iso
if (self.sites):
self.x = self.fmodel.xray_structure.sites_cart().as_double()
if (self.u_iso):
assert self.fmodel.xray_structure.scatterers().size() == \
self.fmodel.xray_structure.use_u_iso().count(True)
self.x = self.fmodel.xray_structure.extract_u_iso_or_u_equiv()
if (self.sites):
xray.set_scatterer_grad_flags(
scatterers=self.fmodel.xray_structure.scatterers(), site=True)
if (self.u_iso):
sel = flex.bool(self.fmodel.xray_structure.scatterers().size(),
True).iselection()
self.fmodel.xray_structure.scatterers().flags_set_grad_u_iso(
iselection=sel)
self.minimizer = scitbx.lbfgs.run(
target_evaluator=self,
termination_params=scitbx.lbfgs.termination_parameters(
max_iterations=max_iterations),
exception_handling_params=scitbx.lbfgs.
exception_handling_parameters(
ignore_line_search_failed_rounding_errors=True,
ignore_line_search_failed_step_at_lower_bound=True,
ignore_line_search_failed_maxfev=True))
self.fmodel.xray_structure.tidy_us()
self.fmodel.xray_structure.apply_symmetry_sites()
self.fmodel.update_xray_structure(
xray_structure=self.fmodel.xray_structure, update_f_calc=True)
def d_target_d_site_cart(self):
manager = self.manager
xray.set_scatterer_grad_flags(
scatterers=manager.xray_structure.scatterers(),
site=True)
return flex.vec3_double(
self.gradients_wrt_atomic_parameters().packed())
def initialize(self, fmodel=None):
self.not_hd_selection = ~self.fmodel.xray_structure.hd_selection(
) # XXX UGLY
assert fmodel is not None
self.fmodel = fmodel
self.fmodel.xray_structure.scatterers().flags_set_grads(state=False)
xray.set_scatterer_grad_flags(
scatterers=self.fmodel.xray_structure.scatterers(), site=True)
self.x = self.fmodel.xray_structure.sites_cart().as_double()
self.x_target_functor = self.fmodel.target_functor()
def __init__(self,
fmodel,
tlsos_initial,
refine_T,
refine_L,
refine_S,
selections,
selections_1d,
max_iterations,
run_finite_differences_test=False,
correct_adp=True):
adopt_init_args(self, locals())
fmodel.xray_structure.scatterers().flags_set_grads(state=False)
xray.set_scatterer_grad_flags(
scatterers=fmodel.xray_structure.scatterers(), u_aniso=True)
if (self.run_finite_differences_test): self.correct_adp = False
self.fmodel_copy = self.fmodel.deep_copy()
self.target_functor = self.fmodel_copy.target_functor()
self.run_finite_differences_test_counter = 0
self.T_initial = []
self.L_initial = []
self.S_initial = []
self.origins = []
for tlso_ in tlsos_initial:
self.T_initial.append(tlso_.t)
self.L_initial.append(tlso_.l)
self.S_initial.append(tlso_.s)
self.origins.append(tlso_.origin)
self.counter = 0
self.n_groups = len(self.T_initial)
self.dim_T = len(self.T_initial[0])
self.dim_L = len(self.L_initial[0])
self.dim_S = len(self.S_initial[0])
self.T_min = self.T_initial
self.L_min = self.L_initial
self.S_min = self.S_initial
self.x = self.pack(self.T_min, self.L_min, self.S_min)
self.minimizer = lbfgs.run(
target_evaluator=self,
core_params=lbfgs.core_parameters(maxfev=10),
termination_params=lbfgs.termination_parameters(
min_iterations=max_iterations,
max_calls=int(max_iterations * 1.5)),
exception_handling_params=lbfgs.exception_handling_parameters(
ignore_line_search_failed_step_at_lower_bound=True,
ignore_line_search_failed_step_at_upper_bound=True,
ignore_line_search_failed_maxfev=True))
self.compute_functional_and_gradients()
del self.x
self.tlsos_result = generate_tlsos(
selections=self.selections,
xray_structure=self.fmodel.xray_structure,
T=self.T_min,
L=self.L_min,
S=self.S_min)
def __init__(self,
fmodel,
tlsos_initial,
refine_T,
refine_L,
refine_S,
selections,
selections_1d,
max_iterations,
run_finite_differences_test = False,
correct_adp = True):
adopt_init_args(self, locals())
fmodel.xray_structure.scatterers().flags_set_grads(state=False)
xray.set_scatterer_grad_flags(scatterers = fmodel.xray_structure.scatterers(),
u_aniso = True)
if(self.run_finite_differences_test): self.correct_adp = False
self.fmodel_copy = self.fmodel.deep_copy()
self.target_functor = self.fmodel_copy.target_functor()
self.run_finite_differences_test_counter = 0
self.T_initial = []
self.L_initial = []
self.S_initial = []
self.origins = []
for tlso_ in tlsos_initial:
self.T_initial.append(tlso_.t)
self.L_initial.append(tlso_.l)
self.S_initial.append(tlso_.s)
self.origins.append(tlso_.origin)
self.counter = 0
self.n_groups = len(self.T_initial)
self.dim_T = len(self.T_initial[0])
self.dim_L = len(self.L_initial[0])
self.dim_S = len(self.S_initial[0])
self.T_min = self.T_initial
self.L_min = self.L_initial
self.S_min = self.S_initial
self.x = self.pack(self.T_min, self.L_min, self.S_min)
self.minimizer = lbfgs.run(
target_evaluator = self,
core_params = lbfgs.core_parameters(maxfev = 10),
termination_params = lbfgs.termination_parameters(
min_iterations = max_iterations,
max_calls = int(max_iterations*1.5)),
exception_handling_params = lbfgs.exception_handling_parameters(
ignore_line_search_failed_step_at_lower_bound = True,
ignore_line_search_failed_step_at_upper_bound = True,
ignore_line_search_failed_maxfev = True))
self.compute_functional_and_gradients()
del self.x
self.tlsos_result = generate_tlsos(
selections = self.selections,
xray_structure = self.fmodel.xray_structure,
T = self.T_min,
L = self.L_min,
S = self.S_min)
def __init__(self,
fmodel,
model,
xs_initial,
adp_nmas,
fsub_adp,
selections,
selections_1d,
max_iterations,
n_modes,
weight_nmre=1.0,
run_finite_differences_test=False,
correct_adp=True,
zero_mode_flag=True):
adopt_init_args(self, locals())
fmodel.xray_structure.scatterers().flags_set_grads(state=False)
xray.set_scatterer_grad_flags(
scatterers=fmodel.xray_structure.scatterers(), u_aniso=True)
if (self.run_finite_differences_test): self.correct_adp = False
self.fmodel_copy = self.fmodel.deep_copy()
self.target_functor = self.fmodel_copy.target_functor()
self.run_finite_differences_test_counter = 0
self.counter = 0
self.n_groups = len(self.xs_initial)
self.dim_x = len(self.xs_initial[0])
self.xs_min = self.xs_initial
self.x = self.pack(self.xs_min)
# for adp_nma_selected, selection in zip(adp_nmas, selections):
# weights_selected = self.fmodel.xray_structure.atomic_weights.select(selection)
# modes1d_selected = selected_modes_to_1D(modes = self.modes, n_modes = self.n_modes,
# selection = self.selection)
# assert len(modes1d_selected)/n_modes == len(weights_selected)
# adp_nma_selected = init_nm_adp(modes = modes1d_selected,
# weights = weights_selected,
# n_modes = n_modes,
# zero_mode_flag = zero_mode_flag)
# self.adp_nmas.append(adp_nma_selected)
self.minimizer = lbfgs.run(
target_evaluator=self,
core_params=lbfgs.core_parameters(maxfev=10),
termination_params=lbfgs.termination_parameters(
min_iterations=max_iterations,
max_calls=int(max_iterations * 1.5)),
exception_handling_params=lbfgs.exception_handling_parameters(
ignore_line_search_failed_step_at_lower_bound=True,
ignore_line_search_failed_step_at_upper_bound=True,
ignore_line_search_failed_maxfev=True))
self.compute_functional_and_gradients()
del self.x
self.xs_result = self.xs_min
def __init__(self,
fmodel,
states,
restraints_manager,
max_iterations=50,
sites = False,
u_iso = False,
restraints_weight=1,
data_weight=None,
restraints_weight_scale=1):
self.cntr = 0
self.rws = restraints_weight_scale
self.states = states
self.fmodel = fmodel
self.restraints_weight = restraints_weight
self.data_weight = data_weight
self.restraints_manager = restraints_manager
self.fmodel.xray_structure.scatterers().flags_set_grads(state=False)
self.x_target_functor = self.fmodel.target_functor()
self.sites = sites
self.u_iso = u_iso
if(self.sites):
self.x = self.fmodel.xray_structure.sites_cart().as_double()
if(self.u_iso):
assert self.fmodel.xray_structure.scatterers().size() == \
self.fmodel.xray_structure.use_u_iso().count(True)
self.x = self.fmodel.xray_structure.extract_u_iso_or_u_equiv()
if(self.sites):
xray.set_scatterer_grad_flags(
scatterers = self.fmodel.xray_structure.scatterers(),
site = True)
if(self.u_iso):
sel = flex.bool(
self.fmodel.xray_structure.scatterers().size(), True).iselection()
self.fmodel.xray_structure.scatterers().flags_set_grad_u_iso(
iselection = sel)
self.minimizer = scitbx.lbfgs.run(
target_evaluator=self,
termination_params=scitbx.lbfgs.termination_parameters(
max_iterations=max_iterations),
exception_handling_params=scitbx.lbfgs.exception_handling_parameters(
ignore_line_search_failed_rounding_errors=True,
ignore_line_search_failed_step_at_lower_bound=True,
ignore_line_search_failed_maxfev=True))
self.fmodel.xray_structure.tidy_us()
self.fmodel.xray_structure.apply_symmetry_sites()
self.fmodel.update_xray_structure(
xray_structure = self.fmodel.xray_structure,
update_f_calc = True)
def __init__(self,
fmodel,
model,
xs_initial,
adp_nmas,
selections,
selections_1d,
max_iterations,
n_modes,
weight_nmre = 1.0,
run_finite_differences_test = False,
correct_adp = True,
zero_mode_flag = True):
adopt_init_args(self, locals())
fmodel.xray_structure.scatterers().flags_set_grads(state=False)
xray.set_scatterer_grad_flags(scatterers = fmodel.xray_structure.scatterers(),
u_aniso = True)
if(self.run_finite_differences_test): self.correct_adp = False
self.fmodel_copy = self.fmodel.deep_copy()
self.target_functor = self.fmodel_copy.target_functor()
self.run_finite_differences_test_counter = 0
self.counter = 0
self.n_groups = len(self.xs_initial)
self.dim_x = len(self.xs_initial[0])
self.xs_min = self.xs_initial
self.x = self.pack(self.xs_min)
# for adp_nma_selected, selection in zip(adp_nmas, selections):
# weights_selected = self.fmodel.xray_structure.atomic_weights.select(selection)
# modes1d_selected = selected_modes_to_1D(modes = self.modes, n_modes = self.n_modes,
# selection = self.selection)
# assert len(modes1d_selected)/n_modes == len(weights_selected)
# adp_nma_selected = init_nm_adp(modes = modes1d_selected,
# weights = weights_selected,
# n_modes = n_modes,
# zero_mode_flag = zero_mode_flag)
# self.adp_nmas.append(adp_nma_selected)
self.minimizer = lbfgs.run(
target_evaluator = self,
core_params = lbfgs.core_parameters(),
termination_params = lbfgs.termination_parameters(
min_iterations = max_iterations,
max_calls = int(max_iterations*1.5)),
exception_handling_params = lbfgs.exception_handling_parameters(
ignore_line_search_failed_step_at_lower_bound = False,
ignore_line_search_failed_step_at_upper_bound = True,
ignore_line_search_failed_maxfev = False))
self.compute_functional_and_gradients()
del self.x
self.xs_result = self.xs_min
def site(structure_ideal, d_min, f_obs, verbose=0):
sh = shifted_site(f_obs, structure_ideal, 0, 0, 0.01)
if (0 or verbose):
print("site")
sh.structure_shifted.show_summary().show_scatterers()
print()
ls = xray.targets_least_squares_residual(f_obs.data(), sh.f_calc.data(),
True, 1)
gradient_flags = randomize_gradient_flags(
xray.structure_factors.gradient_flags(site=True),
f_obs.anomalous_flag())
xray.set_scatterer_grad_flags(scatterers=sh.structure_shifted.scatterers(),
site=gradient_flags.site,
u_iso=gradient_flags.u_iso,
u_aniso=gradient_flags.u_aniso,
occupancy=gradient_flags.occupancy,
fp=gradient_flags.fp,
fdp=gradient_flags.fdp)
sfd = xray.structure_factors.gradients_direct(
xray_structure=sh.structure_shifted,
u_iso_refinable_params=None,
miller_set=f_obs,
d_target_d_f_calc=ls.derivatives(),
n_parameters=0)
re = resampling(miller_set=f_obs)
map0 = re(xray_structure=sh.structure_shifted,
u_iso_refinable_params=None,
dp=miller.array(miller_set=f_obs, data=ls.derivatives()),
n_parameters=0,
verbose=verbose)
sfd_d_target_d_site_cart = sfd.d_target_d_site_cart()
top_gradient = None
for i_scatterer, scatterer in enumerate(sh.structure_shifted.scatterers()):
for i_xyz in (0, 1, 2):
direct_summ = sfd_d_target_d_site_cart[i_scatterer][i_xyz]
if (top_gradient is None): top_gradient = direct_summ
fast_gradie = map0.d_target_d_site_cart()[i_scatterer][i_xyz]
match = judge(scatterer, "site", direct_summ, fast_gradie,
top_gradient)
if (0 or verbose):
print("direct summ[%d][%d]: " % (i_scatterer, i_xyz),
direct_summ)
print("fast gradie[%d][%d]: " % (i_scatterer, i_xyz),
fast_gradie, match)
print()
assert not match.is_bad
sys.stdout.flush()
def __init__(self,
restraints_manager=None,
fmodel=None,
sites_cart=None,
wx=None,
wc=None,
update_gradient_threshold=0):
adopt_init_args(self, locals())
assert [self.fmodel, self.wx].count(None) in [0, 2]
assert [self.restraints_manager, self.wc].count(None) in [0, 2]
self.gx, self.gc = 0, 0
if (self.fmodel is not None):
self.x_target_functor = self.fmodel.target_functor()
xray.set_scatterer_grad_flags(
scatterers=self.fmodel.xray_structure.scatterers(), site=True)
self.gx = flex.vec3_double(self.x_target_functor(compute_gradients=True).\
gradients_wrt_atomic_parameters(site=True).packed())
def site(structure_ideal, d_min, f_obs, verbose=0):
sh = shifted_site(f_obs, structure_ideal, 0, 0, 0.01)
if (0 or verbose):
print "site"
sh.structure_shifted.show_summary().show_scatterers()
print
ls = xray.targets_least_squares_residual(
f_obs.data(), sh.f_calc.data(), True, 1)
gradient_flags = randomize_gradient_flags(
xray.structure_factors.gradient_flags(site=True),
f_obs.anomalous_flag())
xray.set_scatterer_grad_flags(scatterers = sh.structure_shifted.scatterers(),
site = gradient_flags.site,
u_iso = gradient_flags.u_iso,
u_aniso = gradient_flags.u_aniso,
occupancy = gradient_flags.occupancy,
fp = gradient_flags.fp,
fdp = gradient_flags.fdp)
sfd = xray.structure_factors.gradients_direct(
xray_structure=sh.structure_shifted,
u_iso_refinable_params=None,
miller_set=f_obs,
d_target_d_f_calc=ls.derivatives(),
n_parameters=0)
re = resampling(miller_set=f_obs)
map0 = re(
xray_structure=sh.structure_shifted,
u_iso_refinable_params=None,
dp=miller.array(miller_set=f_obs, data=ls.derivatives()),
n_parameters=0,
verbose=verbose)
sfd_d_target_d_site_cart = sfd.d_target_d_site_cart()
top_gradient = None
for i_scatterer,scatterer in enumerate(sh.structure_shifted.scatterers()):
for i_xyz in (0,1,2):
direct_summ = sfd_d_target_d_site_cart[i_scatterer][i_xyz]
if (top_gradient is None): top_gradient = direct_summ
fast_gradie = map0.d_target_d_site_cart()[i_scatterer][i_xyz]
match = judge(scatterer, "site", direct_summ, fast_gradie, top_gradient)
if (0 or verbose):
print "direct summ[%d][%d]: " % (i_scatterer,i_xyz), direct_summ
print "fast gradie[%d][%d]: " % (i_scatterer,i_xyz), fast_gradie, match
print
assert not match.is_bad
sys.stdout.flush()
def __init__(self,
restraints_manager = None,
fmodel = None,
sites_cart = None,
wx = None,
wc = None,
update_gradient_threshold = 0):
adopt_init_args(self, locals())
assert [self.fmodel, self.wx].count(None) in [0,2]
assert [self.restraints_manager, self.wc].count(None) in [0,2]
self.gx, self.gc = 0, 0
if(self.fmodel is not None):
self.x_target_functor = self.fmodel.target_functor()
xray.set_scatterer_grad_flags(
scatterers = self.fmodel.xray_structure.scatterers(),
site = True)
self.gx = flex.vec3_double(self.x_target_functor(compute_gradients=True).\
gradients_wrt_atomic_parameters(site=True).packed())
def __init__(self,
fmodel,
max_iterations=100,
sites = False,
u_iso = False):
self.fmodel = fmodel
self.fmodel.xray_structure.scatterers().flags_set_grads(state=False)
self.x_target_functor = self.fmodel.target_functor()
self.sites = sites
self.u_iso = u_iso
if(self.sites):
# Coordiantes
self.x = self.fmodel.xray_structure.sites_cart().as_double()
if(self.u_iso):
# anisotropic displacement factors
assert self.fmodel.xray_structure.scatterers().size() == \
self.fmodel.xray_structure.use_u_iso().count(True)
self.x = self.fmodel.xray_structure.extract_u_iso_or_u_equiv()
if(self.sites):
xray.set_scatterer_grad_flags(
scatterers = self.fmodel.xray_structure.scatterers(),
site = True)
if(self.u_iso):
sel = flex.bool(
self.fmodel.xray_structure.scatterers().size(), True).iselection()
self.fmodel.xray_structure.scatterers().flags_set_grad_u_iso(
iselection = sel)
self.minimizer = scitbx.lbfgs.run(
target_evaluator=self,
termination_params=scitbx.lbfgs.termination_parameters(
max_iterations=max_iterations),
exception_handling_params=scitbx.lbfgs.exception_handling_parameters(
ignore_line_search_failed_rounding_errors=True,
ignore_line_search_failed_step_at_lower_bound=True,
ignore_line_search_failed_maxfev=True))
self.fmodel.xray_structure.tidy_us()
self.fmodel.xray_structure.apply_symmetry_sites()
self.fmodel.update_xray_structure(
xray_structure = self.fmodel.xray_structure,
update_f_calc = True)
def __init__(self,
fmodel,
ncs_restraints_group_list,
refine_selection=None,
use_ncs_constraints=True,
restraints_manager=None,
data_weight=None,
refine_sites=False,
refine_u_iso=False,
refine_transformations=False,
iso_restraints=None,
use_hd=False):
adopt_init_args(self, locals())
asu_size = self.fmodel.xray_structure.sites_cart().size()
self.refine_selection = nu.get_refine_selection(
refine_selection=self.refine_selection, number_of_atoms=asu_size)
self.extended_ncs_selection = nu.get_extended_ncs_selection(
ncs_restraints_group_list=ncs_restraints_group_list,
refine_selection=self.refine_selection)
self.fmodel.xray_structure.scatterers().flags_set_grads(state=False)
self.x_target_functor = self.fmodel.target_functor()
self.xray_structure = self.fmodel.xray_structure
if self.refine_sites:
xray.set_scatterer_grad_flags(
scatterers=self.fmodel.xray_structure.scatterers(), site=True)
elif self.refine_u_iso:
xray.set_scatterer_grad_flags(
scatterers=self.fmodel.xray_structure.scatterers(), u_iso=True)
elif self.refine_transformations:
xray.set_scatterer_grad_flags(
scatterers=self.fmodel.xray_structure.scatterers(), site=True)
def finite_differences_test():
print "finite_differences_test: "
fmodel = get_fmodel_from_pdb(pdb_file_name="enk_rbr.pdb",
algorithm="direct",
d_min=2.0,
target="ls_wunit_k1")
xray.set_scatterer_grad_flags(
scatterers=fmodel.xray_structure.scatterers(), site=True)
for convention in ["zyz", "xyz"]:
rot_obj = scitbx.rigid_body.euler(phi=0,
psi=0,
the=0,
convention=convention)
size = fmodel.xray_structure.scatterers().size()
selections = [flex.bool(size, True).iselection()]
fmodel.xray_structure.apply_rigid_body_shift(rot=rot_obj.rot_mat(),
trans=[1, 2, 3])
fmodel.update_xray_structure(update_f_calc=True)
fmodel_copy = fmodel.deep_copy()
centers_of_mass = []
for s in selections:
xrs = fmodel_copy.xray_structure.select(s)
centers_of_mass.append(xrs.center_of_mass())
tg_obj = mmtbx.refinement.rigid_body.target_and_grads(
centers_of_mass=centers_of_mass,
sites_cart=fmodel_copy.xray_structure.sites_cart(),
target_functor=fmodel_copy.target_functor(),
rot_objs=[rot_obj],
selections=selections,
suppress_gradients=False)
assert approx_equal(tg_obj.target(), fmodel_copy.target_w())
g_rot, g_transl = tg_obj.gradients_wrt_r(), tg_obj.gradients_wrt_t()
fd_transl = fd_translation(fmodel_copy, e=0.0001)
assert approx_equal(list(g_transl[0]), fd_transl)
fd_rot = fd_rotation(fmodel=fmodel_copy,
e=0.0001,
convention=convention)
assert approx_equal(list(g_rot[0]), fd_rot)
def finite_differences_test():
print "finite_differences_test: "
fmodel = get_fmodel_from_pdb(pdb_file_name = "enk_rbr.pdb",
algorithm = "direct",
d_min = 2.0,
target = "ls_wunit_k1")
xray.set_scatterer_grad_flags(
scatterers = fmodel.xray_structure.scatterers(),
site = True)
for convention in ["zyz","xyz"]:
rot_obj = scitbx.rigid_body.euler(
phi = 0, psi = 0, the = 0, convention = convention)
size = fmodel.xray_structure.scatterers().size()
selections = [flex.bool(size, True).iselection()]
fmodel.xray_structure.apply_rigid_body_shift(
rot = rot_obj.rot_mat(), trans = [1,2,3])
fmodel.update_xray_structure(update_f_calc = True)
fmodel_copy = fmodel.deep_copy()
centers_of_mass = []
for s in selections:
xrs = fmodel_copy.xray_structure.select(s)
centers_of_mass.append(xrs.center_of_mass())
tg_obj = mmtbx.refinement.rigid_body.target_and_grads(
centers_of_mass = centers_of_mass,
sites_cart = fmodel_copy.xray_structure.sites_cart(),
target_functor = fmodel_copy.target_functor(),
rot_objs = [rot_obj],
selections = selections,
suppress_gradients = False)
assert approx_equal(tg_obj.target(),fmodel_copy.target_w())
g_rot, g_transl = tg_obj.gradients_wrt_r(), tg_obj.gradients_wrt_t()
fd_transl = fd_translation(fmodel_copy, e = 0.0001)
assert approx_equal(list(g_transl[0]), fd_transl)
fd_rot = fd_rotation(fmodel = fmodel_copy,
e = 0.0001,
convention = convention)
assert approx_equal(list(g_rot[0]), fd_rot)
def __init__(
self,
fmodel,
ncs_restraints_group_list,
refine_selection=None,
use_ncs_constraints=True,
restraints_manager=None,
data_weight=None,
refine_sites=False,
refine_u_iso=False,
refine_transformations=False,
iso_restraints = None,
use_hd = False):
adopt_init_args(self, locals())
asu_size = self.fmodel.xray_structure.sites_cart().size()
self.refine_selection = nu.get_refine_selection(
refine_selection=self.refine_selection,
number_of_atoms=asu_size)
self.extended_ncs_selection = nu.get_extended_ncs_selection(
ncs_restraints_group_list=ncs_restraints_group_list,
refine_selection=self.refine_selection)
self.fmodel.xray_structure.scatterers().flags_set_grads(state=False)
self.x_target_functor = self.fmodel.target_functor()
self.xray_structure = self.fmodel.xray_structure
if self.refine_sites:
xray.set_scatterer_grad_flags(
scatterers = self.fmodel.xray_structure.scatterers(),
site = True)
elif self.refine_u_iso:
xray.set_scatterer_grad_flags(
scatterers = self.fmodel.xray_structure.scatterers(),
u_iso = True)
elif self.refine_transformations:
xray.set_scatterer_grad_flags(
scatterers = self.fmodel.xray_structure.scatterers(),
site = True)
def get_intensity_structure(base,FE1_model,FE2_model):
"""Now used stuff learned from test_fmodel_stuff to calculate new data structure for use
in the program.
"""
# generate the list of all HKL to be used throughout.
C2_structures = get_C2_structures()
energy = 7070.0
GF_whole7070 = gen_fmodel_with_complex.from_structure(C2_structures[0],energy
).from_parameters(algorithm="fft")
GF_whole7070 = GF_whole7070.as_P1_primitive()
f_container7070 = GF_whole7070.get_fmodel()
Fmodel_whole7070 = f_container7070.f_model
Fmodel_indices = Fmodel_whole7070.indices() # common structure defines the indices
MS = Fmodel_whole7070.set() # avoid having to repeatedly calculate indices
CS = Fmodel_whole7070.crystal_symmetry() # same here
result = flex.double(flex.grid((Fmodel_indices.size(),500)))
GF_FE1 = gen_fmodel_with_complex.from_structure(C2_structures[2],energy
).from_parameters(algorithm="direct")
GF_FE1 = GF_FE1.as_P1_primitive()
GF_FE2 = gen_fmodel_with_complex.from_structure(C2_structures[3],energy
).from_parameters(algorithm="direct")
GF_FE2 = GF_FE2.as_P1_primitive()
for incr in range(100):
print ("incr is",incr)
energy = 7070.5 + incr
GF_FE1.reset_specific_at_energy(label_has="FE1",tables=FE1_model,newvalue=energy)
# not sure if I need this, takes a lot of time # f_container = GF_FE1.get_fmodel()
# not sure if I need this, takes a lot of time # Fcalc_FE1 = f_container.fmodel.f_calc()
GF_FE2.reset_specific_at_energy(label_has="FE2",tables=FE2_model,newvalue=energy)
# not sure if I need this, takes a lot of time # f_container = GF_FE2.get_fmodel()
# not sure if I need this, takes a lot of time # Fcalc_FE2 = f_container.fmodel.f_calc()
ALGO = structure_factors.from_scatterers(crystal_symmetry=CS,
d_min=GF_whole7070.params2.high_resolution)
#hack cctbx/xray/structure_factors/structure_factors_direct.h
"""
/*#if !defined(CCTBX_XRAY_STRUCTURE_FACTORS_DIRECT_NO_PRAGMA_OMP)
#if !defined(__DECCXX_VER) || (defined(_OPENMP) && _OPENMP > 199819)
#pragma omp parallel for schedule(static)
#endif
#endif
*/
#pragma omp parallel for
"""
from_scatterers_direct_fe1 = ALGO(xray_structure=GF_FE1.xray_structure,
miller_set=MS,algorithm="direct")
Fcalc_FE1_dir = from_scatterers_direct_fe1.f_calc().data()
from_scatterers_direct_fe2 = ALGO(xray_structure=GF_FE2.xray_structure,
miller_set=MS,algorithm="direct")
Fcalc_FE2_dir = from_scatterers_direct_fe2.f_calc().data()
# Get total Fcalc at energy:
F_bulk_non_Fe = base.matrix_copy_block(i_row=0,i_column=incr,
n_rows=Fmodel_indices.size(),n_columns=1)
Fcalc_FE1_dir.reshape(flex.grid((Fmodel_indices.size(),1)))
Fcalc_FE2_dir.reshape(flex.grid((Fmodel_indices.size(),1)))
Fcalc_total = F_bulk_non_Fe + Fcalc_FE1_dir + Fcalc_FE2_dir
result.matrix_paste_block_in_place(flex.norm(Fcalc_total),0,incr) # gives I = F * F
gradient_flags=xray.structure_factors.gradient_flags(
site=False,
u_iso=False,
u_aniso=False,
occupancy=False,
fp=True,
fdp=True)
xray.set_scatterer_grad_flags(scatterers = GF_FE1.xray_structure.scatterers(),
site = gradient_flags.site,
u_iso = gradient_flags.u_iso,
u_aniso = gradient_flags.u_aniso,
occupancy = gradient_flags.occupancy,
fp = gradient_flags.fp,
fdp = gradient_flags.fdp)
xray.set_scatterer_grad_flags(scatterers = GF_FE2.xray_structure.scatterers(),
site = gradient_flags.site,
u_iso = gradient_flags.u_iso,
u_aniso = gradient_flags.u_aniso,
occupancy = gradient_flags.occupancy,
fp = gradient_flags.fp,
fdp = gradient_flags.fdp)
#hack cctbx/xray/structure_factors/each_hkl_gradients_direct.h
#pragma omp parallel for (line 226)
sf1 = xray.ext.each_hkl_gradients_direct(
MS.unit_cell(), MS.space_group(), MS.indices(), GF_FE1.xray_structure.scatterers(), None,
GF_FE1.xray_structure.scattering_type_registry(), GF_FE1.xray_structure.site_symmetry_table(),
0)
sf2 = xray.ext.each_hkl_gradients_direct(
MS.unit_cell(), MS.space_group(), MS.indices(), GF_FE2.xray_structure.scatterers(), None,
GF_FE2.xray_structure.scattering_type_registry(), GF_FE2.xray_structure.site_symmetry_table(),
0)
sf1.d_fcalc_d_fp().reshape(flex.grid((Fmodel_indices.size(),1)))
sf1.d_fcalc_d_fdp().reshape(flex.grid((Fmodel_indices.size(),1)))
sf2.d_fcalc_d_fp().reshape(flex.grid((Fmodel_indices.size(),1)))
sf2.d_fcalc_d_fdp().reshape(flex.grid((Fmodel_indices.size(),1)))
parts = Fcalc_total.parts()
A = parts[0]
B = parts[1]
for offset,vec2 in zip([100,200,300,400],
[sf1.d_fcalc_d_fp(),sf1.d_fcalc_d_fdp(),sf2.d_fcalc_d_fp(),sf2.d_fcalc_d_fdp()]):
vparts = vec2.parts()
vA = vparts[0]
vB = vparts[1]
partial_I_partial_q = 2. * (A * vA + B * vB)
result.matrix_paste_block_in_place(partial_I_partial_q,0,incr + offset)
# result holds a table of structure factor intensities. Rows are Miller indices H.
# First 100 columns are I_H(energy, 100 channels). The next four groups of 100 columns
# are the partial derivatives of I_H with respect to fp(FE1), fdp(FE1), fp(FE2), and fdp(FE2)
# respectively all as a function of energy channel. Thus this is
# the energy-dependent portion of the calculation that is dependent on the iron model.
return result
def exercise(space_group_info,
n_elements = 10,
table = "wk1995",
d_min = 2.0,
k_sol = 0.35,
b_sol = 45.0,
b_cart = None,
quick=False,
verbose=0):
xray_structure = random_structure.xray_structure(
space_group_info = space_group_info,
elements =(("O","N","C")*(n_elements//3+1))[:n_elements],
volume_per_atom = 100,
min_distance = 1.5,
general_positions_only = True,
random_u_iso = False,
random_occupancy = False)
xray_structure.scattering_type_registry(table = table)
sg = xray_structure.space_group()
uc = xray_structure.unit_cell()
u_cart_1 = adptbx.random_u_cart(u_scale=5, u_min=5)
u_star_1 = adptbx.u_cart_as_u_star(uc, u_cart_1)
b_cart = adptbx.u_star_as_u_cart(uc, sg.average_u_star(u_star = u_star_1))
for anomalous_flag in [False, True]:
scatterers = xray_structure.scatterers()
if (anomalous_flag):
assert scatterers.size() >= 7
for i in [1,7]:
scatterers[i].fp = -0.2
scatterers[i].fdp = 5
have_non_zero_fdp = True
else:
for i in [1,7]:
scatterers[i].fp = 0
scatterers[i].fdp = 0
have_non_zero_fdp = False
f_obs = abs(xray_structure.structure_factors(
d_min = d_min,
anomalous_flag = anomalous_flag,
cos_sin_table = sfg_params.cos_sin_table,
algorithm = sfg_params.algorithm).f_calc())
f_obs_comp = f_obs.structure_factors_from_scatterers(
xray_structure = xray_structure,
algorithm = sfg_params.algorithm,
cos_sin_table = sfg_params.cos_sin_table).f_calc()
f_obs = abs(f_obs_comp)
flags = f_obs.generate_r_free_flags(fraction = 0.1,
max_free = 99999999)
#flags = flags.array(data = flex.bool(f_obs.data().size(), False))
xrs = xray_structure.deep_copy_scatterers()
xrs.shake_sites_in_place(rms_difference=0.3)
for target in mmtbx.refinement.targets.target_names:
if (quick):
if (target not in ["ls_wunit_k1", "ml", "mlhl", "ml_sad"]):
continue
if (target == "mlhl"):
if (have_non_zero_fdp): continue # XXX gradients not correct!
experimental_phases = generate_random_hl(miller_set=f_obs)
else:
experimental_phases = None
if (target == "ml_sad"
and (not anomalous_flag or mmtbx.refinement.targets.phaser is None)):
continue
print " ",target
xray.set_scatterer_grad_flags(
scatterers = xrs.scatterers(),
site = True)
ss = 1./flex.pow2(f_obs.d_spacings().data()) / 4.
u_star = adptbx.u_cart_as_u_star(
f_obs.unit_cell(), adptbx.b_as_u(b_cart))
k_anisotropic = mmtbx.f_model.ext.k_anisotropic(
f_obs.indices(), u_star)
k_mask = mmtbx.f_model.ext.k_mask(ss, k_sol, b_sol)
fmodel = mmtbx.f_model.manager(
xray_structure = xrs,
f_obs = f_obs,
r_free_flags = flags,
target_name = target,
abcd = experimental_phases,
sf_and_grads_accuracy_params = sfg_params,
k_mask = k_mask,
k_anisotropic = k_anisotropic,
mask_params = masks.mask_master_params.extract())
fmodel.update_xray_structure(
xray_structure=xrs,
update_f_calc=True,
update_f_mask=True)
xray.set_scatterer_grad_flags(
scatterers=fmodel.xray_structure.scatterers(),
site=True)
fmodel.update_xray_structure(update_f_calc=True)
t_f = fmodel.target_functor()
t_f.prepare_for_minimization()
gs = t_f(compute_gradients=True).d_target_d_site_cart().as_double()
gfd = finite_differences_site(target_functor=t_f)
cc = flex.linear_correlation(gs, gfd).coefficient()
if (0 or verbose):
print "ana:", list(gs)
print "fin:", list(gfd)
print "rat:", [f/a for a,f in zip(gs,gfd)]
print target, "corr:", cc, space_group_info
print
diff = gs - gfd
diff /= max(1, flex.max(flex.abs(gfd)))
tolerance = 1.2e-5
assert approx_equal(abs(flex.min(diff) ), 0.0, tolerance)
assert approx_equal(abs(flex.mean(diff)), 0.0, tolerance)
assert approx_equal(abs(flex.max(diff) ), 0.0, tolerance)
assert approx_equal(cc, 1.0, tolerance)
fmodel.model_error_ml()
def exercise_gradient_manager(structure_ideal, f_obs,
anomalous_flag,
verbose=0):
sh, ls = shift_all(structure_ideal, f_obs, anomalous_flag)
grad_manager = xray.structure_factors.gradients(
miller_set=f_obs,
quality_factor=100000,
wing_cutoff=1.e-10)
gradient_flags=xray.structure_factors.gradient_flags(default=True)
xray.set_scatterer_grad_flags(scatterers = sh.structure_shifted.scatterers(),
site = gradient_flags.site,
u_iso = gradient_flags.u_iso,
u_aniso = gradient_flags.u_aniso,
occupancy = gradient_flags.occupancy,
fp = gradient_flags.fp,
fdp = gradient_flags.fdp)
if (0):
print "exercise_gradient_manager"
print "gradient_flags.site ", gradient_flags.site
print "gradient_flags.u_iso ", gradient_flags.u_iso
print "gradient_flags.u_aniso ", gradient_flags.u_aniso
print "gradient_flags.occupancy ", gradient_flags.occupancy
print "gradient_flags.fp ", gradient_flags.fp
print "gradient_flags.fdp ", gradient_flags.fdp
cntr_use_u_iso = 0
cntr_use_u_aniso = 0
cntr_grad_u_iso = 0
cntr_grad_u_aniso = 0
for scatterer in sh.structure_shifted.scatterers():
if (scatterer.flags.use_u_iso()): cntr_use_u_iso += 1
if (scatterer.flags.use_u_aniso()): cntr_use_u_aniso += 1
if (scatterer.flags.grad_u_iso()): cntr_grad_u_iso += 1
if (scatterer.flags.grad_u_aniso()): cntr_grad_u_aniso += 1
print "use_u_iso ", cntr_use_u_iso,cntr_grad_u_iso
print "use_u_aniso ", cntr_use_u_aniso,cntr_grad_u_aniso
if (random.random() > 0.5):
n_parameters = 0
else:
n_parameters = xray.scatterer_grad_flags_counts(
sh.structure_shifted.scatterers()).n_parameters()
assert n_parameters == sh.structure_shifted.n_parameters()
gd = grad_manager(
xray_structure=sh.structure_shifted,
u_iso_refinable_params=None,
miller_set=f_obs,
d_target_d_f_calc=ls.derivatives(),
n_parameters=n_parameters,
algorithm="direct")
gf = grad_manager(
xray_structure=sh.structure_shifted,
u_iso_refinable_params=None,
miller_set=f_obs,
d_target_d_f_calc=ls.derivatives(),
n_parameters=n_parameters,
algorithm="fft")
grad_flags_counts = \
xray.scatterer_grad_flags_counts(sh.structure_shifted.scatterers())
if (n_parameters == 0):
d = gd.d_target_d_site_frac()
f = gf.d_target_d_site_frac()
linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose)
d = gd.d_target_d_site_cart()
f = gf.d_target_d_site_cart()
linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose)
if(grad_flags_counts.u_iso > 0):
d = gd.d_target_d_u_iso()
f = gf.d_target_d_u_iso()
linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose)
if(grad_flags_counts.u_aniso > 0):
linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose)
d = gd.d_target_d_u_cart()
f = gf.d_target_d_u_cart()
linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose)
d = gd.d_target_d_occupancy()
f = gf.d_target_d_occupancy()
linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose)
d = gd.d_target_d_fp()
f = gf.d_target_d_fp()
linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose)
if (anomalous_flag):
d = gd.d_target_d_fdp()
f = gf.d_target_d_fdp()
linear_regression_test(d, f, slope_tolerance=1.e-2, verbose=verbose)
else:
correlation = flex.linear_correlation(gd.packed(), gf.packed())
assert correlation.is_well_defined()
assert correlation.coefficient() > 0.995, correlation.coefficient()
def u_star(structure_ideal, d_min, f_obs, verbose=0):
sh = shifted_u_star(f_obs, structure_ideal, 0, 0, 0.0001)
if (0 or verbose):
print "u_star"
sh.structure_shifted.show_summary().show_scatterers()
print
ls = xray.targets_least_squares_residual(
f_obs.data(), sh.f_calc.data(), True, 1)
gradient_flags = randomize_gradient_flags(
xray.structure_factors.gradient_flags(u_aniso=True),
f_obs.anomalous_flag())
xray.set_scatterer_grad_flags(scatterers = sh.structure_shifted.scatterers(),
site = gradient_flags.site,
u_iso = gradient_flags.u_iso,
u_aniso = gradient_flags.u_aniso,
occupancy = gradient_flags.occupancy,
fp = gradient_flags.fp,
fdp = gradient_flags.fdp)
grad_flags_counts = xray.scatterer_grad_flags_counts(sh.structure_shifted.scatterers())
if(grad_flags_counts.n_parameters() > 0):
if (0):
print "u_aniso"
print "gradient_flags.site ", gradient_flags.site
print "gradient_flags.u_iso ", gradient_flags.u_iso
print "gradient_flags.u_aniso ", gradient_flags.u_aniso
print "gradient_flags.occupancy ", gradient_flags.occupancy
print "gradient_flags.fp ", gradient_flags.fp
print "gradient_flags.fdp ", gradient_flags.fdp
cntr_use_u_iso = 0
cntr_use_u_aniso = 0
cntr_grad_u_iso = 0
cntr_grad_u_aniso = 0
for scatterer in sh.structure_shifted.scatterers():
if (scatterer.flags.use_u_iso()): cntr_use_u_iso += 1
if (scatterer.flags.use_u_aniso()): cntr_use_u_aniso += 1
if (scatterer.flags.grad_u_iso()): cntr_grad_u_iso += 1
if (scatterer.flags.grad_u_aniso()): cntr_grad_u_aniso += 1
print "use_u_iso ", cntr_use_u_iso,cntr_grad_u_iso
print "use_u_aniso ", cntr_use_u_aniso,cntr_grad_u_aniso
sfd = xray.structure_factors.gradients_direct(
xray_structure=sh.structure_shifted,
u_iso_refinable_params=None,
miller_set=f_obs,
d_target_d_f_calc=ls.derivatives(),
n_parameters= 0
)
re = resampling(miller_set=f_obs)
map0 = re(
xray_structure=sh.structure_shifted,
u_iso_refinable_params=None,
dp=miller.array(miller_set=f_obs, data=ls.derivatives()),
n_parameters= 0,
verbose=verbose)
grad_flags_counts = xray.scatterer_grad_flags_counts(sh.structure_shifted.scatterers())
if(grad_flags_counts.u_aniso):
sfd_d_target_d_u_cart = sfd.d_target_d_u_cart()
map0_d_target_d_u_cart = map0.d_target_d_u_cart()
top_gradient = None
gradients_1 = []
for i_scatterer,scatterer in enumerate(sh.structure_shifted.scatterers()):
if(scatterer.flags.use_u_iso()): parameter_name = "u_iso"
if(scatterer.flags.use_u_aniso()): parameter_name = "u_star"
if(parameter_name == "u_iso" and scatterer.flags.grad_u_iso()):
direct_summ = sfd.d_target_d_u_iso()[i_scatterer]
if (top_gradient is None): top_gradient = direct_summ
fast_gradie = map0.d_target_d_u_iso()[i_scatterer]
sys.stdout.flush()
gradients_1.append([direct_summ, fast_gradie])
match = judge(scatterer, parameter_name, direct_summ, fast_gradie,
top_gradient)
if (0 or verbose):
print "direct summ[%d]: " % i_scatterer, direct_summ
print "fast gradie[%d]: " % i_scatterer, fast_gradie, match
print
assert not match.is_bad
if parameter_name == "u_star" and scatterer.flags.grad_u_aniso():
sfd_star = sfd.d_target_d_u_star()[i_scatterer]
sfd_cart = adptbx.grad_u_star_as_u_cart(
structure_ideal.unit_cell(), sfd_star)
assert approx_equal(
sfd_star,
adptbx.grad_u_cart_as_u_star(structure_ideal.unit_cell(), sfd_cart))
for ij in xrange(6):
direct_summ = sfd_d_target_d_u_cart[i_scatterer][ij]
if (top_gradient is None): top_gradient = direct_summ
fast_gradie = map0_d_target_d_u_cart[i_scatterer][ij]
gradients_1.append([direct_summ, fast_gradie])
match =judge(scatterer,"u_star",direct_summ,fast_gradie,top_gradient)
if (0 or verbose or match.is_bad):
print "direct summ[%d][%d]: " % (i_scatterer, ij), direct_summ
print "fast gradie[%d][%d]: " % (i_scatterer, ij),fast_gradie,match
print
assert not match.is_bad
# Making sure that gradients_1 = gradients_2
for i_scatterer,scatterer in enumerate(sh.structure_shifted.scatterers()):
if(not scatterer.flags.use_u_iso()):
scatterer.u_iso = -12345.0
if(not scatterer.flags.use_u_aniso()):
scatterer.u_star =(-999.,-999.,-999.,-999.,-999.,-999.)
sfd = xray.structure_factors.gradients_direct(
xray_structure=sh.structure_shifted,
u_iso_refinable_params=None,
miller_set=f_obs,
d_target_d_f_calc=ls.derivatives(),
n_parameters= 0
)
re = resampling(miller_set=f_obs)
map0 = re(
xray_structure=sh.structure_shifted,
u_iso_refinable_params=None,
dp=miller.array(miller_set=f_obs, data=ls.derivatives()),
n_parameters= 0,
verbose=verbose)
grad_flags_counts = \
xray.scatterer_grad_flags_counts(sh.structure_shifted.scatterers())
if(grad_flags_counts.u_aniso):
sfd_d_target_d_u_cart = sfd.d_target_d_u_cart()
map0_d_target_d_u_cart = map0.d_target_d_u_cart()
gradients_2 = []
for i_scatterer,scatterer in enumerate(sh.structure_shifted.scatterers()):
if(scatterer.flags.use_u_iso()): parameter_name = "u_iso"
if(scatterer.flags.use_u_aniso()): parameter_name = "u_star"
if(parameter_name == "u_iso" and scatterer.flags.grad_u_iso()):
direct_summ = sfd.d_target_d_u_iso()[i_scatterer]
fast_gradie = map0.d_target_d_u_iso()[i_scatterer]
gradients_2.append([direct_summ, fast_gradie])
if parameter_name == "u_star" and scatterer.flags.grad_u_aniso():
sfd_star = sfd.d_target_d_u_star()[i_scatterer]
sfd_cart = adptbx.grad_u_star_as_u_cart(
structure_ideal.unit_cell(), sfd_star)
assert approx_equal(
sfd_star,
adptbx.grad_u_cart_as_u_star(structure_ideal.unit_cell(), sfd_cart))
for ij in xrange(6):
direct_summ = sfd_d_target_d_u_cart[i_scatterer][ij]
fast_gradie = map0_d_target_d_u_cart[i_scatterer][ij]
gradients_2.append([direct_summ, fast_gradie])
for g1,g2 in zip(gradients_1, gradients_2):
assert approx_equal(g1, g2)
sys.stdout.flush()
def test_fmodel_stuff(energy,FE1_model,FE2_model):
# section 1. C2 models. a) whole pdb, b) no Fe, c) FE1 only, d) FE2 only
C2_structures = get_C2_structures()
print("C2 models validate OK")
# section 2.
if False:
for structure in C2_structures:
GF = gen_fmodel_with_complex.from_structure(structure,energy).from_parameters()
ID = GF.get_intensity_dictionary()
# section 3. Validate that the old script work2_for_aca_lsq/remake_range_intensities.py
# gives same intensities as the new gen_fmodel_with_complex class, for the
# simple case of the reduced model at energy=7125 eV
from LS49.work2_for_aca_lsq.remake_range_intensities import \
remake_intensities_at_energy as old_algorithm
print("old")
ID_ref = old_algorithm(energy,FE1_model,FE2_model)
print("new")
GF = gen_fmodel_with_complex.from_structure(C2_structures[0],energy).from_parameters()
GF.reset_specific_at_energy(label_has="FE1",tables=FE1_model,newvalue=energy)
GF.reset_specific_at_energy(label_has="FE2",tables=FE2_model,newvalue=energy)
GF = GF.as_P1_primitive()
ID_new = GF.get_intensity_dictionary()
for key in ID_ref:
assert ID_ref[key]==ID_new[key]
assert len(ID_ref)==len(ID_new)
print("whole structure validates OK")
# section 4. Validate that Fmodel(whole pdb) = Fbulk + Fcalc(non-Fe) + Fcalc(FE1) + Fcalc(FE2)
# For this validation it is necessary from cctbx.xray import structure_factors to carry the arithmetic using A+iB, not F*F
#GF_whole = GF #from above, but compute it again here to use algo=direct
GF_whole = gen_fmodel_with_complex.from_structure(C2_structures[0],energy).from_parameters(algorithm="fft")
GF_whole.reset_specific_at_energy(label_has="FE1",tables=FE1_model,newvalue=energy)
GF_whole.reset_specific_at_energy(label_has="FE2",tables=FE2_model,newvalue=energy)
GF_whole = GF_whole.as_P1_primitive()
f_container = GF_whole.get_fmodel()
Fmodel_whole = f_container.f_model
Fcalc_whole = f_container.fmodel.f_calc()
Fbulk = Fcalc_whole.customized_copy(data=Fmodel_whole.data()-Fcalc_whole.data())
GF_non_Fe = gen_fmodel_with_complex.from_structure(C2_structures[1],energy).from_parameters(algorithm="fft")
GF_non_Fe = GF_non_Fe.as_P1_primitive()
f_container = GF_non_Fe.get_fmodel()
Fcalc_non_Fe = f_container.fmodel.f_calc()
GF_FE1 = gen_fmodel_with_complex.from_structure(C2_structures[2],energy).from_parameters(algorithm="direct")
GF_FE1.reset_specific_at_energy(label_has="FE1",tables=FE1_model,newvalue=energy)
GF_FE1 = GF_FE1.as_P1_primitive()
f_container = GF_FE1.get_fmodel()
Fcalc_FE1 = f_container.fmodel.f_calc()
GF_FE2 = gen_fmodel_with_complex.from_structure(C2_structures[3],energy).from_parameters(algorithm="direct")
GF_FE2.reset_specific_at_energy(label_has="FE2",tables=FE2_model,newvalue=energy)
GF_FE2 = GF_FE2.as_P1_primitive()
f_container = GF_FE2.get_fmodel()
Fcalc_FE2 = f_container.fmodel.f_calc()
# subpoint 4a. Fcalc(all) = Fcalc(non-FE)+Fcalc(FE1)+Fcalc(FE2)
test4a = Fcalc_whole.customized_copy(data = Fcalc_whole.data() -
Fcalc_non_Fe.data() -
Fcalc_FE1.data() - Fcalc_FE2.data())
#print (list(test4a.data()))
if True:
n_outliers = 0
for ikey,key in enumerate(test4a.indices()):
diff0 = Fcalc_whole.data()[ikey]-Fcalc_non_Fe.data()[ikey]
diff1 = diff0 - Fcalc_FE1.data()[ikey]
test4a_diff = test4a.data()[ikey]
#It's not exact because the protein Fcalc's are done by FFT algorithm, but
# the lack of closure is generally < 1%.
if abs(test4a_diff) > 0.01 * abs(Fcalc_whole.data()[ikey]):
n_outliers += 1
print ("%18s %8.2f /%8.2f %8.2f /%8.2f %8.2f /%8.2f %8.2f"%(
key,abs(Fcalc_whole.data()[ikey]),
abs(Fcalc_non_Fe.data()[ikey]),
abs(diff0),
abs(Fcalc_FE1.data()[ikey]),
abs(diff1),
abs(Fcalc_FE2.data()[ikey]),
abs(test4a_diff)
))
print("Test 4a N_outliers: %d out of %d"%(n_outliers, len(test4a.indices())))
assert (n_outliers / len(test4a.indices())) < 0.01
# subpoint 4b. Fmodel(whole pdb) = Fbulk + Fcalc(non-Fe) + Fcalc(FE1) + Fcalc(FE2)
test4b = Fmodel_whole.customized_copy(data = Fmodel_whole.data() -
Fbulk.data() - Fcalc_non_Fe.data() -
Fcalc_FE1.data() - Fcalc_FE2.data())
n_outliers = 0
for ikey,key in enumerate(test4b.indices()):
test4b_diff = test4b.data()[ikey]
#It's not exact because the protein Fcalc's are done by FFT algorithm, but
# the lack of closure is generally < 1%.
if abs(test4b_diff) > 0.01 * abs(Fmodel_whole.data()[ikey]):
n_outliers += 1
print ("%18s %8.2f / %8.2f"%(key,abs(Fmodel_whole.data()[ikey]),
abs(test4b_diff)
))
print("Test 4b N_outliers: %d out of %d"%(n_outliers, len(test4b.indices())))
assert (n_outliers / len(test4b.indices())) < 0.01
print("Complex arithmetic validates OK")
# section 5. Repeat the section 4 assertions, but
# section 5a. Assign different FE1(f',f") and FE2(f',f") & calculate Fmodel(whole)
# section 5b. Assemble the same answer but with explicit Python-coded Fcalcs(FE1,FE2)
table = FE1_model
print(table.fp_fdp_at_wavelength(angstroms = 12398.425/energy))
new_FE1 = george_sherrell_proxy(-5,9)
new_FE2 = george_sherrell_proxy(-4,8)
GF_whole.reset_specific_at_energy(label_has="FE1",tables=new_FE1,newvalue=energy)
GF_whole.reset_specific_at_energy(label_has="FE2",tables=new_FE2,newvalue=energy)
f_container = GF_whole.get_fmodel()
Fmodel_whole_new_fpfdp = f_container.f_model
GF_FE1.reset_specific_at_energy(label_has="FE1",tables=new_FE1,newvalue=energy)
f_container = GF_FE1.get_fmodel()
Fcalc_FE1 = f_container.fmodel.f_calc()
GF_FE2.reset_specific_at_energy(label_has="FE2",tables=new_FE2,newvalue=energy)
f_container = GF_FE2.get_fmodel()
Fcalc_FE2 = f_container.fmodel.f_calc()
MS = Fmodel_whole_new_fpfdp.set() # avoid having to repeatedly calculate indices
CS = Fmodel_whole_new_fpfdp.crystal_symmetry() # same here
ALGO = structure_factors.from_scatterers(crystal_symmetry=CS,
d_min=GF_whole.params2.high_resolution)
from_scatterers_direct_fe1 = ALGO(xray_structure=GF_FE1.xray_structure,
miller_set=MS,algorithm="direct")
Fcalc_FE1_dir = from_scatterers_direct_fe1.f_calc()
from_scatterers_direct_fe2 = ALGO(xray_structure=GF_FE2.xray_structure,
miller_set=MS,algorithm="direct")
Fcalc_FE2_dir = from_scatterers_direct_fe2.f_calc()
for ikey,key in enumerate(MS.indices()):
#print ("%18s %8.2f %8.2f"%(key,abs(Fcalc_FE1.data()[ikey]-Fcalc_FE1_dir.data()[ikey]),
# abs(Fcalc_FE2.data()[ikey]-Fcalc_FE2_dir.data()[ikey])
#))
assert abs(Fcalc_FE1.data()[ikey]-Fcalc_FE1_dir.data()[ikey])==0.
assert abs(Fcalc_FE2.data()[ikey]-Fcalc_FE2_dir.data()[ikey])==0.
print("Test 5b, calculation with modified fp fdp (low-level interface) validates OK")
"""
Here we actually develop the Python code to give the A+iB for FE1 and FE2.
looks like we should develop a new direct class to calculate F, dF/dfp, dF/dfdp
all in one swoop, within a C++ extension module. Use this extension to calculate
Fcalc for FE1,FE2, instead of the conventional formalism.
actually, gradients direct already exists. So is it possible to just wrap existing
functions at a low-enough level to get the answer?
see tst_xray for usage.
"""
# section 6. Validate analytical derivatives' consistency with finite differences,
# using correlation coefficient as a score.
# section 6a. Get analytical derivatives from a low-level interface
gradient_flags=xray.structure_factors.gradient_flags(
site=False,
u_iso=False,
u_aniso=False,
occupancy=False,
fp=True,
fdp=True)
xray.set_scatterer_grad_flags(scatterers = GF_FE1.xray_structure.scatterers(),
site = gradient_flags.site,
u_iso = gradient_flags.u_iso,
u_aniso = gradient_flags.u_aniso,
occupancy = gradient_flags.occupancy,
fp = gradient_flags.fp,
fdp = gradient_flags.fdp)
xray.set_scatterer_grad_flags(scatterers = GF_FE2.xray_structure.scatterers(),
site = gradient_flags.site,
u_iso = gradient_flags.u_iso,
u_aniso = gradient_flags.u_aniso,
occupancy = gradient_flags.occupancy,
fp = gradient_flags.fp,
fdp = gradient_flags.fdp)
print ("gradients")
sf1 = xray.ext.each_hkl_gradients_direct(
MS.unit_cell(), MS.space_group(), MS.indices(), GF_FE1.xray_structure.scatterers(), None,
GF_FE1.xray_structure.scattering_type_registry(), GF_FE1.xray_structure.site_symmetry_table(),
0)
sf2 = xray.ext.each_hkl_gradients_direct(
MS.unit_cell(), MS.space_group(), MS.indices(), GF_FE2.xray_structure.scatterers(), None,
GF_FE2.xray_structure.scattering_type_registry(), GF_FE2.xray_structure.site_symmetry_table(),
0)
print("Test 6a, Get analytical derivatives from a low-level interface")
for excursion in [0.0001,0.001,0.01]:
# finite differences
incr_FE1_fp = george_sherrell_proxy(-5+excursion,9)
incr_FE1_fdp = george_sherrell_proxy(-5,9+excursion)
GF_FE1.reset_specific_at_energy(label_has="FE1",tables=incr_FE1_fp,newvalue=energy)
f_container = GF_FE1.get_fmodel()
incr_Fcalc_FE1_fp = f_container.fmodel.f_calc()
GF_FE1.reset_specific_at_energy(label_has="FE1",tables=incr_FE1_fdp,newvalue=energy)
f_container = GF_FE1.get_fmodel()
incr_Fcalc_FE1_fdp = f_container.fmodel.f_calc()
#analytical
fp_analytical = Fcalc_FE1.data() + sf1.d_fcalc_d_fp()*excursion
fdp_analytical = Fcalc_FE1.data() + sf1.d_fcalc_d_fdp()*excursion
diffs_fp = fp_analytical - incr_Fcalc_FE1_fp.data()
diffs_fdp = fdp_analytical - incr_Fcalc_FE1_fdp.data()
#validation
RMSDfp = math.sqrt( flex.sum( flex.pow( flex.abs(diffs_fp) ,2 ) ) )
RMSDfdp = math.sqrt(flex.sum(flex.pow( flex.abs(diffs_fdp) ,2)))
print ("with excursion %16.14f the RMSD are fp %16.14f and fdp %16.14f"%(excursion,RMSDfp,RMSDfdp))
assert RMSDfp < 1.E-10
assert RMSDfdp < 1.E-10
print("Test 6b, Validate analytical vs. finite differences")
def __init__(
self,
structure,
restraints_manager,
temperature=300,
protein_thermostat=False,
n_steps=200,
time_step=0.0005,
initial_velocities_zero_fraction=0,
vxyz=None,
interleaved_minimization_params=None,
n_print=20,
fmodel=None,
xray_target_weight=None,
chem_target_weight=None,
shift_update=0.0,
xray_structure_last_updated=None,
xray_gradient=None,
reset_velocities=True,
stop_cm_motion=False,
update_f_calc=True,
er_data=None,
log=None,
stop_at_diff=None,
verbose=-1,
):
adopt_init_args(self, locals())
assert self.n_print > 0
assert self.temperature >= 0.0
assert self.n_steps >= 0
assert self.time_step >= 0.0
assert self.log is not None or self.verbose < 1
xray.set_scatterer_grad_flags(scatterers=self.structure.scatterers(), site=True)
self.structure_start = self.structure.deep_copy_scatterers()
self.k_boltz = boltzmann_constant_akma
self.ekcm = 0.0
self.timfac = akma_time_as_pico_seconds
self.weights = self.structure.atomic_weights()
if vxyz is None:
self.vxyz = flex.vec3_double(self.weights.size(), (0, 0, 0))
else:
self.vxyz = vxyz
if self.er_data is not None and self.er_data.velocities is not None:
self.vxyz = self.er_data.velocities
if self.er_data is not None:
self.er_data.geo_grad_rms = 0
self.er_data.xray_grad_rms = 0
if self.fmodel is not None:
if self.er_data is None:
self.fmodel_copy = self.fmodel.deep_copy()
else:
self.fmodel_copy = self.fmodel
if self.er_data.fix_scale_factor is not None:
self.fmodel_copy.set_scale_switch = self.er_data.fix_scale_factor
#
self.target_functor = self.fmodel_copy.target_functor()
assert self.chem_target_weight is not None
assert self.xray_target_weight is not None
if self.xray_gradient is None:
self.xray_gradient = self.xray_grads()
#
imp = self.interleaved_minimization_params
self.interleaved_minimization_flag = imp is not None and imp.number_of_iterations > 0
if self.interleaved_minimization_flag:
assert imp.time_step_factor > 0
self.time_step *= imp.time_step_factor
if "bonds" not in imp.restraints:
raise Sorry('Invalid choice: %s.restraints: "bonds" must always be included.' % imp.__phil_path__())
self.interleaved_minimization_angles = "angles" in imp.restraints
else:
self.interleaved_minimization_angles = False
#
self.tstep = self.time_step / self.timfac
self.show_gradient_rms = False # XXX debug option
#
self()
def exercise(space_group_info,
n_elements=10,
table="wk1995",
d_min=2.0,
k_sol=0.35,
b_sol=45.0,
b_cart=None,
quick=False,
verbose=0):
xray_structure = random_structure.xray_structure(
space_group_info=space_group_info,
elements=(("O", "N", "C") * (n_elements // 3 + 1))[:n_elements],
volume_per_atom=100,
min_distance=1.5,
general_positions_only=True,
random_u_iso=False,
random_occupancy=False)
xray_structure.scattering_type_registry(table=table)
sg = xray_structure.space_group()
uc = xray_structure.unit_cell()
u_cart_1 = adptbx.random_u_cart(u_scale=5, u_min=5)
u_star_1 = adptbx.u_cart_as_u_star(uc, u_cart_1)
b_cart = adptbx.u_star_as_u_cart(uc, sg.average_u_star(u_star=u_star_1))
for anomalous_flag in [False, True]:
scatterers = xray_structure.scatterers()
if (anomalous_flag):
assert scatterers.size() >= 7
for i in [1, 7]:
scatterers[i].fp = -0.2
scatterers[i].fdp = 5
have_non_zero_fdp = True
else:
for i in [1, 7]:
scatterers[i].fp = 0
scatterers[i].fdp = 0
have_non_zero_fdp = False
f_obs = abs(
xray_structure.structure_factors(
d_min=d_min,
anomalous_flag=anomalous_flag,
cos_sin_table=sfg_params.cos_sin_table,
algorithm=sfg_params.algorithm).f_calc())
f_obs_comp = f_obs.structure_factors_from_scatterers(
xray_structure=xray_structure,
algorithm=sfg_params.algorithm,
cos_sin_table=sfg_params.cos_sin_table).f_calc()
f_obs = abs(f_obs_comp)
flags = f_obs.generate_r_free_flags(fraction=0.1, max_free=99999999)
#flags = flags.array(data = flex.bool(f_obs.data().size(), False))
xrs = xray_structure.deep_copy_scatterers()
xrs.shake_sites_in_place(rms_difference=0.3)
for target in mmtbx.refinement.targets.target_names:
if target == "mli": continue
if (quick):
if (target not in ["ls_wunit_k1", "ml", "mlhl", "ml_sad"]):
continue
if (target == "mlhl"):
if (have_non_zero_fdp): continue # XXX gradients not correct!
experimental_phases = generate_random_hl(miller_set=f_obs)
else:
experimental_phases = None
if (target == "ml_sad"
and (not anomalous_flag
or mmtbx.refinement.targets.phaser is None)):
continue
print(" ", target)
xray.set_scatterer_grad_flags(scatterers=xrs.scatterers(),
site=True)
ss = 1. / flex.pow2(f_obs.d_spacings().data()) / 4.
u_star = adptbx.u_cart_as_u_star(f_obs.unit_cell(),
adptbx.b_as_u(b_cart))
k_anisotropic = mmtbx.f_model.ext.k_anisotropic(
f_obs.indices(), u_star)
k_mask = mmtbx.f_model.ext.k_mask(ss, k_sol, b_sol)
fmodel = mmtbx.f_model.manager(
xray_structure=xrs,
f_obs=f_obs,
r_free_flags=flags,
target_name=target,
abcd=experimental_phases,
sf_and_grads_accuracy_params=sfg_params,
k_mask=k_mask,
k_anisotropic=k_anisotropic,
mask_params=masks.mask_master_params.extract())
fmodel.update_xray_structure(xray_structure=xrs,
update_f_calc=True,
update_f_mask=True)
xray.set_scatterer_grad_flags(
scatterers=fmodel.xray_structure.scatterers(), site=True)
fmodel.update_xray_structure(update_f_calc=True)
t_f = fmodel.target_functor()
t_f.prepare_for_minimization()
gs = t_f(compute_gradients=True).d_target_d_site_cart().as_double()
gfd = finite_differences_site(target_functor=t_f)
cc = flex.linear_correlation(gs, gfd).coefficient()
if (0 or verbose):
print("ana:", list(gs))
print("fin:", list(gfd))
print("rat:", [f / a for a, f in zip(gs, gfd)])
print(target, "corr:", cc, space_group_info)
print()
diff = gs - gfd
diff /= max(1, flex.max(flex.abs(gfd)))
tolerance = 1.2e-5
assert approx_equal(abs(flex.min(diff)), 0.0, tolerance)
assert approx_equal(abs(flex.mean(diff)), 0.0, tolerance)
assert approx_equal(abs(flex.max(diff)), 0.0, tolerance)
assert approx_equal(cc, 1.0, tolerance)
fmodel.model_error_ml()
def get_weight(fmodel=None,
restraints_manager=None,
sites=None,
transformations=None,
u_iso=None,
ncs_restraints_group_list=None,
refine_selection=None,
minimized_obj=None):
"""
Calculates weights for refinements by slightly shaking the minimized
parameters and taking the ratio:
(restraint manager grad norm / parameters gradient norm)
When calling this function during refinement macro cycle, the minimized
object, "minimized_obj" , may contains fmodel, restraints_manager,
refinement type info (sites, transformations, u_iso) and
ncs_restraints_group_list.
Args:
fmodel : F-model object
restraints_manager: Restraints manager object
sites (bool): Refine by sites
u_iso (bool): Refine using u_iso
transformations (bool): Refine using transformations
rotations, translations (matrix objects):
ncs_restraints_group_list: list of ncs_restraint_group objects
refine_selection (flex.size_t): selection of all ncs related copies and
non ncs related parts to be included in selection (to be refined)
minimized_obj:Minimization object containing all the other
parameters above
Returns:
weight (int):
Example:
>>>get_weight(minimized_obj=minimized_obj)
or
>>>get_weight(fmodel=fmodel,
restraints_manager=grm,
sites=sites,
transformations=transformations,
u_iso=u_iso,
ncs_restraints_group_list=ncs_restraints_group_list,
refine_selection=refine_selection)
"""
grm = restraints_manager
extended_ncs_selection = None
# extract parameters from minimized_obj
if minimized_obj:
mo = minimized_obj
fmodel = mo.fmodel
grm = mo.grm
sites = mo.sites
transformations = mo.transformations
u_iso = mo.u_iso
ncs_restraints_group_list = mo.ncs_restraints_group_list
# del: consider deleting the code below
# if hasattr(mo,'extended_ncs_selection'):
# extended_ncs_selection = mo.extended_ncs_selection
# if not extended_ncs_selection:
# extended_ncs_selection = get_extended_ncs_selection(
# ncs_restraints_group_list=ncs_restraints_group_list,
# refine_selection=refine_selection)
# make sure sufficient input is provided
assert bool(fmodel), 'F-model is not provided'
assert bool(grm), 'F-restraints_manager is not provided'
assert [sites,transformations,u_iso].count(True)==1, 'Refinement type Error'
#
have_transforms = ncs_restraints_group_list != []
fmdc = fmodel.deep_copy()
if sites:
fmdc.xray_structure.shake_sites_in_place(mean_distance=0.3)
elif u_iso:
fmdc.xray_structure.shake_adp()
elif transformations and have_transforms:
x = concatenate_rot_tran(
ncs_restraints_group_list = ncs_restraints_group_list)
x = shake_transformations(
x = x,
shake_angles_sigma=0.035,
shake_translation_sigma=0.5)
fmdc.update_xray_structure(xray_structure = fmdc.xray_structure,
update_f_calc=True)
fmdc.xray_structure.scatterers().flags_set_grads(state=False)
if sites or transformations:
xray.set_scatterer_grad_flags(
scatterers = fmdc.xray_structure.scatterers(),
site = True)
# fmodel gradients
gxc = flex.vec3_double(fmdc.one_time_gradients_wrt_atomic_parameters(
site = True).packed())
# restraints manager, energy sites gradients
gc = grm.energies_sites(
sites_cart = fmdc.xray_structure.sites_cart(),
compute_gradients = True).gradients
elif u_iso:
# Create energies_site gradient, to create geometry_restraints_manager
# plain_pair_sym_table needed for the energies_adp_iso
import mmtbx.refinement.adp_refinement
temp = mmtbx.refinement.adp_refinement.adp_restraints_master_params
iso_restraints = temp.extract().iso
gc = grm.energies_sites(
sites_cart = fmdc.xray_structure.sites_cart(),
compute_gradients = True).gradients
xray.set_scatterer_grad_flags(
scatterers = fmdc.xray_structure.scatterers(),
u_iso = True)
# fmodel gradients
gxc = fmdc.one_time_gradients_wrt_atomic_parameters(
u_iso = True).as_double()
# manager restraints, energy sites gradients
gc = grm.energies_adp_iso(
xray_structure = fmdc.xray_structure,
parameters = iso_restraints,
use_u_local_only = iso_restraints.use_u_local_only,
use_hd = False,
compute_gradients = True).gradients
if transformations and have_transforms:
# Apply NCS relations to gradients
gxc = compute_transform_grad(
grad_wrt_xyz = gxc.as_double(),
ncs_restraints_group_list = ncs_restraints_group_list,
xyz_asu = fmdc.xray_structure.sites_cart(),
x = x)
gc = compute_transform_grad(
grad_wrt_xyz = gc.as_double(),
ncs_restraints_group_list = ncs_restraints_group_list,
xyz_asu = fmdc.xray_structure.sites_cart(),
x = x)
weight = 1.
gc_norm = gc.norm()
gxc_norm = gxc.norm()
if(gxc_norm != 0.0):
weight = gc_norm / gxc_norm
weight =min(weight,1e6) # limit the weight max value
return weight
def exercise(target_functor,
data_type,
parameter_name,
space_group_info,
anomalous_flag,
cartesian_flag,
n_elements=9,
d_min=2.5,
shake_sigma=0.25,
test_hard=True,
verbose=0):
assert data_type == 'F' or data_type == 'F^2'
if (data_type == 'F^2'
and not target_functor == xray.unified_least_squares_residual):
return
if (parameter_name != "site" and cartesian_flag == True): return
if (parameter_name == "fdp" and not anomalous_flag): return
structure_ideal = random_structure.xray_structure(
space_group_info,
elements=(("O", "N", "C") * (n_elements))[:n_elements],
volume_per_atom=100,
random_f_prime_d_min=d_min,
random_f_double_prime=anomalous_flag,
use_u_aniso=True,
use_u_iso=False,
random_u_cart_scale=0.3,
random_u_iso=False,
random_occupancy=True)
if (parameter_name in ["u_star", "u_iso"]): shake_sigma = shake_sigma / 2.
random_structure.random_modify_adp_and_adp_flags(
scatterers=structure_ideal.scatterers(),
random_u_iso_scale=0.3,
random_u_iso_min=0.0,
parameter_name=parameter_name)
rnd_f_calc = structure_ideal.structure_factors(
anomalous_flag=anomalous_flag, d_min=d_min,
algorithm="direct").f_calc()
if data_type == 'F':
y_obs = abs(rnd_f_calc)
elif data_type == 'F^2':
y_obs = rnd_f_calc.norm()
y_obs.set_observation_type_xray_intensity()
structure_shake = structure_ideal.random_modify_parameters(
parameter_name, shake_sigma, vary_z_only=False)
assert tuple(structure_ideal.special_position_indices()) \
== tuple(structure_shake.special_position_indices())
target_ftor = target_functor(y_obs)
for structure in (structure_ideal, structure_shake)[:]: #SWITCH
f_calc = y_obs.structure_factors_from_scatterers(
xray_structure=structure, algorithm="direct").f_calc()
target_result = target_ftor(f_calc, compute_derivatives=True)
if (structure == structure_ideal):
assert abs(target_result.target()) < 1.e-5
gradient_flags = xray.structure_factors.gradient_flags(
site=(parameter_name == "site" or random.choice((False, True))),
u_iso=(parameter_name == "u_iso" or random.choice((False, True))),
u_aniso=(parameter_name == "u_star" or random.choice((False, True))),
occupancy=(parameter_name == "occupancy" or random.choice(
(False, True))),
fp=(parameter_name == "fp" or random.choice((False, True))),
fdp=(parameter_name == "fdp" or (anomalous_flag and random.choice(
(False, True)))))
xray.set_scatterer_grad_flags(scatterers=structure.scatterers(),
site=gradient_flags.site,
u_iso=gradient_flags.u_iso,
u_aniso=gradient_flags.u_aniso,
occupancy=gradient_flags.occupancy,
fp=gradient_flags.fp,
fdp=gradient_flags.fdp)
grad_flags_counts = xray.scatterer_grad_flags_counts(
structure.scatterers())
sf = xray.structure_factors.gradients_direct(
xray_structure=structure,
u_iso_refinable_params=None,
miller_set=y_obs,
d_target_d_f_calc=target_result.derivatives(),
n_parameters=0)
if (parameter_name == "site"):
d_analytical = sf.d_target_d_site_frac()
if (cartesian_flag
): # average d_analytical or d_numerical, but not both
structure_ideal.apply_special_position_ops_d_target_d_site(
d_analytical)
if (cartesian_flag):
d_analytical = sf.d_target_d_site_cart()
d_numerical = finite_differences_site(cartesian_flag, target_ftor,
structure)
if (not cartesian_flag
): # aver. d_analytical or d_numerical, but not both
structure_ideal.apply_special_position_ops_d_target_d_site(
d_numerical)
elif (parameter_name == "u_star" and grad_flags_counts.use_u_aniso > 0):
d_analytical = sf.d_target_d_u_star()
d_numerical = finite_differences_u_star(target_ftor, structure)
else:
if (parameter_name == "occupancy"):
d_analytical = sf.d_target_d_occupancy()
elif (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0):
d_analytical = sf.d_target_d_u_iso()
elif (parameter_name == "fp"):
d_analytical = sf.d_target_d_fp()
elif (parameter_name == "fdp"):
d_analytical = sf.d_target_d_fdp()
else:
raise RuntimeError
d_numerical = finite_differences_scalar(parameter_name, target_ftor,
structure)
linear_regression_test(d_analytical,
d_numerical,
test_hard,
verbose=verbose)
if (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0):
u_iso_refinable_params = flex.double()
for scatterer in structure.scatterers():
scatterer.flags.set_tan_u_iso(True)
scatterer.flags.set_grad_u_iso(gradient_flags.u_iso)
param = random.randint(90, 120)
scatterer.flags.param = param
value = math.tan(scatterer.u_iso * math.pi / adptbx.b_as_u(param) -
math.pi / 2)
u_iso_refinable_params.append(value)
sf = xray.structure_factors.gradients_direct(
xray_structure=structure,
u_iso_refinable_params=u_iso_refinable_params,
miller_set=y_obs,
d_target_d_f_calc=target_result.derivatives(),
n_parameters=0)
d_analytical = sf.d_target_d_u_iso()
d_numerical = finite_differences_scalar("tan_u_iso", target_ftor,
structure)
linear_regression_test(d_analytical,
d_numerical,
test_hard,
verbose=verbose)
def __init__(self,
fmodel,
ncs_transformations_object=None,
ncs_atom_selection = None,
run_finite_grad_differences_test = False,
max_iterations=100,
sites = False,
u_iso = False):
"""Implementing strict NCS to refinement minimization
Arguments:
fmodel : fmodel of the complete ASU
ncs_transformation_object : information on the NCS to ASU
transformations and chains. A multimer object
ncs_atom_selection : boolean array for selection of atoms in the NCS.
A flex bool array
"""
self.fmodel = fmodel
self.fmodel.xray_structure.scatterers().flags_set_grads(state=False)
self.x_target_functor = self.fmodel.target_functor()
self.sites = sites
self.u_iso = u_iso
self.ncs_to_asu = ncs_transformations_object
self.run_finite_grad_differences_test = run_finite_grad_differences_test
if run_finite_grad_differences_test:
# perform gradient calc test
self.buffer_max_grad = flex.double()
self.buffer_calc_grad = flex.double()
# xray structure of NCS chains for self.x
ncs_fmodel_xrs = self.fmodel.xray_structure.select(ncs_atom_selection)
if(self.sites):
self.x = ncs_fmodel_xrs.sites_cart().as_double()
if(self.u_iso):
assert ncs_fmodel_xrs.scatterers().size() == \
ncs_fmodel_xrs.use_u_iso().count(True)
self.x = ncs_fmodel_xrs.extract_u_iso_or_u_equiv()
# Use all scatterers for gradient calculations
if(self.sites):
xray.set_scatterer_grad_flags(
scatterers = self.fmodel.xray_structure.scatterers(),
site = True)
if(self.u_iso):
sel = flex.bool(
self.fmodel.xray_structure.scatterers().size(), True).iselection()
self.fmodel.xray_structure.scatterers().flags_set_grad_u_iso(
iselection = sel)
self.minimizer = scitbx.lbfgs.run(
target_evaluator=self,
termination_params=scitbx.lbfgs.termination_parameters(
max_iterations=max_iterations),
exception_handling_params=scitbx.lbfgs.exception_handling_parameters(
ignore_line_search_failed_rounding_errors=True,
ignore_line_search_failed_step_at_lower_bound=True,
ignore_line_search_failed_maxfev=True))
self.fmodel.xray_structure.tidy_us()
self.fmodel.xray_structure.apply_symmetry_sites()
self.fmodel.update_xray_structure(
xray_structure = self.fmodel.xray_structure,
update_f_calc = True)
self.tested = 0
if run_finite_grad_differences_test:
if self.buffer_max_grad:
print 'compare max_grad to calc_grad'
for a,f in zip(self.buffer_max_grad, self.buffer_calc_grad):
print '{0:10.5f} {1:10.5f} delta = {2:10.5f}'.format(a,f,abs(a-f))
print '-'*45
diff = flex.abs(self.buffer_max_grad - self.buffer_calc_grad)
s = diff < 1.e-3
if(s.size()>0 and s.count(True)*100./s.size()>50):
self.tested += 1
def exercise_constrained_lbfgs(xray_structure,
constraints_list,
t_celsius,
d_min=0.5,
shake_sites_rmsd=0.5,
shake_u_iso_spread=0,
shake_u_aniso_spread=0,
grad_site=True,
grad_u_iso=True,
grad_u_aniso=False,
grad_occupancy=False,
grad_fp_fdp=False,
lbfgs_m=5,
lbfgs_max_iterations=1000,
verbose=0):
xs = xray_structure
xray.set_scatterer_grad_flags(scatterers=xs.scatterers(),
site=grad_site,
u_iso=grad_u_iso,
u_aniso=grad_u_aniso,
occupancy=grad_occupancy,
fp=grad_fp_fdp,
fdp=grad_fp_fdp,
tan_u_iso=False,
param=0)
xs0 = xs.deep_copy_scatterers()
mi = xs0.build_miller_set(anomalous_flag=False, d_min=d_min)
fo_sq = mi.structure_factors_from_scatterers(
xs0, algorithm="direct").f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))
fo_sq.set_observation_type_xray_intensity()
y_obs = fo_sq
#y_obs = fo_sq.f_sq_as_f()
if grad_site:
xs.shake_sites_in_place(rms_difference=shake_sites_rmsd)
if not grad_u_aniso: shake_u_aniso_spread = 0
if not grad_u_iso: shake_u_iso_spread = 0
if grad_u_aniso or grad_u_iso:
xs.shake_adp(spread=shake_u_iso_spread, aniso_spread=shake_u_aniso_spread)
xs1 = xs.deep_copy_scatterers()
core_params = scitbx.lbfgs.core_parameters(m=lbfgs_m, maxfev=100, xtol=1e-5)
connectivity_table = smtbx.utils.connectivity_table(xs0)
if constraints_list is None:
from smtbx.development import generate_hydrogen_constraints
constraints_list = generate_hydrogen_constraints(
structure=xs0, connectivity_table=connectivity_table)
reparametrisation = constraints.reparametrisation(
xs,
constraints_list,
connectivity_table,
temperature=t_celsius)
lbfgs_termination_params=scitbx.lbfgs.termination_parameters(
traditional_convergence_test=False,
drop_convergence_test_max_drop_eps=1.e-20,
drop_convergence_test_iteration_coefficient=1,
min_iterations=500,
max_iterations=lbfgs_max_iterations)
minimizer = lbfgs(
target_functor=xray.target_functors.unified_least_squares_residual(y_obs),
xray_structure=xs,
reparametrisation=reparametrisation,
structure_factor_algorithm="direct",
lbfgs_termination_params=lbfgs_termination_params,
lbfgs_core_params=core_params,
reference_structure=xs0,
verbose=verbose)
if verbose > 0:
print "Total parameters: ", xs.n_parameters()
print "Independent parameters: ", reparametrisation.n_independents
print "Reference model: "
xs0.show_angles(distance_cutoff=1.5)
print
print "Starting model: "
xs1.show_angles(distance_cutoff=1.5)
print
print "Refined model: "
xs.show_angles(distance_cutoff=1.5)
print
print "n_iter, n_fun: ", minimizer.minimizer.iter(), minimizer.minimizer.nfun()
h_selection = xs.element_selection('H')
diff = xray.meaningful_site_cart_differences(
xs0.select(h_selection, negate=True),
xs.select(h_selection, negate=True))
#diff = xray.meaningful_site_cart_differences(xs0, xs)
#assert diff.max_absolute() < 1e-3
if verbose > 0:
diff.show()
print
assert diff.max_absolute() < 2e-2, diff.max_absolute()
diff = xray.meaningful_site_cart_differences(xs0, xs)
if verbose > 0:
diff.show()
print
# XXX why does this tolerance have to be so high?
assert diff.max_absolute() < 0.5, diff.max_absolute()
def exercise_negative_parameters(verbose=0):
structure_default = xray.structure(
crystal_symmetry = crystal.symmetry(
unit_cell=((10,13,17,75,80,85)),
space_group_symbol="P 1"),
scatterers=flex.xray_scatterer([
xray.scatterer(label="C", site=(0,0,0), u=0.25)]))
negative_gaussian = eltbx.xray_scattering.gaussian((1,2), (2,3), -4)
for i_trial in xrange(7):
structure = structure_default.deep_copy_scatterers()
scatterer = structure.scatterers()[0]
if (i_trial == 1):
scatterer.occupancy *= -1
elif (i_trial == 2):
structure.scattering_type_registry(custom_dict={"C": negative_gaussian})
elif (i_trial == 3):
scatterer.u_iso *= -1
elif (i_trial == 4):
u_cart = adptbx.random_u_cart(u_scale=1, u_min=-1.1)
assert max(adptbx.eigenvalues(u_cart)) < 0
u_star = adptbx.u_cart_as_u_star(structure.unit_cell(), u_cart)
scatterer.u_star = u_star
scatterer.flags.set_use_u_aniso_only()
elif (i_trial == 5):
scatterer.fp = -10
elif (i_trial == 6):
scatterer.fp = -3
f_direct = structure.structure_factors(
d_min=1, algorithm="direct", cos_sin_table=False).f_calc()
f_fft = structure.structure_factors(
d_min=1, algorithm="fft",
quality_factor=1.e8, wing_cutoff=1.e-10).f_calc()
if (i_trial == 2):
assert negative_gaussian.at_d_star_sq(f_fft.d_star_sq().data()).all_lt(0)
if (i_trial in [5,6]):
f = structure.scattering_type_registry().gaussian_not_optional(
scattering_type="C").at_d_star_sq(f_fft.d_star_sq().data())
if (i_trial == 5):
assert flex.max(f) + scatterer.fp < 0
else:
assert flex.max(f) + scatterer.fp > 0
assert flex.min(f) + scatterer.fp < 0
cc = flex.linear_correlation(
abs(f_direct).data(),
abs(f_fft).data()).coefficient()
if (cc < 0.999):
raise AssertionError("i_trial=%d, correlation=%.6g" % (i_trial, cc))
elif (0 or verbose):
print "correlation=%.6g" % cc
#
# very simple test of gradient calculations with negative parameters
structure_factor_gradients = \
cctbx.xray.structure_factors.gradients(
miller_set=f_direct,
cos_sin_table=False)
target_functor = xray.target_functors.intensity_correlation(
f_obs=abs(f_direct))
target_result = target_functor(f_fft, True)
xray.set_scatterer_grad_flags(scatterers = structure.scatterers(),
site = True,
u_iso = True,
u_aniso = True,
occupancy = True,
fp = True,
fdp = True)
for algorithm in ["direct", "fft"]:
grads = structure_factor_gradients(
xray_structure=structure,
u_iso_refinable_params=None,
miller_set=f_direct,
d_target_d_f_calc=target_result.derivatives(),
n_parameters = structure.n_parameters(),
algorithm=algorithm).packed()
def __init__(self,
structure,
restraints_manager,
temperature=300,
protein_thermostat=False,
n_steps=200,
time_step=0.0005,
initial_velocities_zero_fraction=0,
vxyz=None,
interleaved_minimization_params=None,
n_print=20,
fmodel=None,
xray_target_weight=None,
chem_target_weight=None,
shift_update=0.0,
xray_structure_last_updated=None,
xray_gradient=None,
reset_velocities=True,
stop_cm_motion=False,
update_f_calc=True,
er_data=None,
log=None,
stop_at_diff=None,
verbose=-1):
adopt_init_args(self, locals())
assert self.n_print > 0
assert self.temperature >= 0.0
assert self.n_steps >= 0
assert self.time_step >= 0.0
assert self.log is not None or self.verbose < 1
xray.set_scatterer_grad_flags(scatterers=self.structure.scatterers(),
site=True)
self.structure_start = self.structure.deep_copy_scatterers()
self.k_boltz = boltzmann_constant_akma
self.ekcm = 0.0
self.timfac = akma_time_as_pico_seconds
self.weights = self.structure.atomic_weights()
if (vxyz is None):
self.vxyz = flex.vec3_double(self.weights.size(), (0, 0, 0))
else:
self.vxyz = vxyz
if (self.er_data is not None and self.er_data.velocities is not None):
self.vxyz = self.er_data.velocities
if self.er_data is not None:
self.er_data.geo_grad_rms = 0
self.er_data.xray_grad_rms = 0
if (self.fmodel is not None):
if self.er_data is None:
self.fmodel_copy = self.fmodel.deep_copy()
else:
self.fmodel_copy = self.fmodel
if self.er_data.fix_scale_factor is not None:
self.fmodel_copy.set_scale_switch = self.er_data.fix_scale_factor
#
self.target_functor = self.fmodel_copy.target_functor()
assert self.chem_target_weight is not None
assert self.xray_target_weight is not None
if (self.xray_gradient is None):
self.xray_gradient = self.xray_grads()
#
imp = self.interleaved_minimization_params
self.interleaved_minimization_flag = (imp is not None
and imp.number_of_iterations > 0)
if (self.interleaved_minimization_flag):
assert imp.time_step_factor > 0
self.time_step *= imp.time_step_factor
if ("bonds" not in imp.restraints):
raise Sorry(
'Invalid choice: %s.restraints: "bonds" must always be included.'
% imp.__phil_path__())
self.interleaved_minimization_angles = "angles" in imp.restraints
else:
self.interleaved_minimization_angles = False
#
self.tstep = self.time_step / self.timfac
self.show_gradient_rms = False # XXX debug option
#
self()
def exercise_constrained_lbfgs(xray_structure,
constraints_list,
t_celsius,
d_min=0.5,
shake_sites_rmsd=0.5,
shake_u_iso_spread=0,
shake_u_aniso_spread=0,
grad_site=True,
grad_u_iso=True,
grad_u_aniso=False,
grad_occupancy=False,
grad_fp_fdp=False,
lbfgs_m=5,
lbfgs_max_iterations=1000,
verbose=0):
xs = xray_structure
xray.set_scatterer_grad_flags(scatterers=xs.scatterers(),
site=grad_site,
u_iso=grad_u_iso,
u_aniso=grad_u_aniso,
occupancy=grad_occupancy,
fp=grad_fp_fdp,
fdp=grad_fp_fdp,
tan_u_iso=False,
param=0)
xs0 = xs.deep_copy_scatterers()
mi = xs0.build_miller_set(anomalous_flag=False, d_min=d_min)
fo_sq = mi.structure_factors_from_scatterers(
xs0, algorithm="direct").f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))
fo_sq.set_observation_type_xray_intensity()
y_obs = fo_sq
#y_obs = fo_sq.f_sq_as_f()
if grad_site:
xs.shake_sites_in_place(rms_difference=shake_sites_rmsd)
if not grad_u_aniso: shake_u_aniso_spread = 0
if not grad_u_iso: shake_u_iso_spread = 0
if grad_u_aniso or grad_u_iso:
xs.shake_adp(spread=shake_u_iso_spread,
aniso_spread=shake_u_aniso_spread)
xs1 = xs.deep_copy_scatterers()
core_params = scitbx.lbfgs.core_parameters(m=lbfgs_m,
maxfev=100,
xtol=1e-5)
connectivity_table = smtbx.utils.connectivity_table(xs0)
if constraints_list is None:
from smtbx.development import generate_hydrogen_constraints
constraints_list = generate_hydrogen_constraints(
structure=xs0, connectivity_table=connectivity_table)
reparametrisation = constraints.reparametrisation(xs,
constraints_list,
connectivity_table,
temperature=t_celsius)
lbfgs_termination_params = scitbx.lbfgs.termination_parameters(
traditional_convergence_test=False,
drop_convergence_test_max_drop_eps=1.e-20,
drop_convergence_test_iteration_coefficient=1,
min_iterations=500,
max_iterations=lbfgs_max_iterations)
minimizer = lbfgs(
target_functor=xray.target_functors.unified_least_squares_residual(
y_obs),
xray_structure=xs,
reparametrisation=reparametrisation,
structure_factor_algorithm="direct",
lbfgs_termination_params=lbfgs_termination_params,
lbfgs_core_params=core_params,
reference_structure=xs0,
verbose=verbose)
if verbose > 0:
print "Total parameters: ", xs.n_parameters()
print "Independent parameters: ", reparametrisation.n_independents
print "Reference model: "
xs0.show_angles(distance_cutoff=1.5)
print
print "Starting model: "
xs1.show_angles(distance_cutoff=1.5)
print
print "Refined model: "
xs.show_angles(distance_cutoff=1.5)
print
print "n_iter, n_fun: ", minimizer.minimizer.iter(
), minimizer.minimizer.nfun()
h_selection = xs.element_selection('H')
diff = xray.meaningful_site_cart_differences(
xs0.select(h_selection, negate=True),
xs.select(h_selection, negate=True))
#diff = xray.meaningful_site_cart_differences(xs0, xs)
#assert diff.max_absolute() < 1e-3
if verbose > 0:
diff.show()
print
assert diff.max_absolute() < 2e-2, diff.max_absolute()
diff = xray.meaningful_site_cart_differences(xs0, xs)
if verbose > 0:
diff.show()
print
# XXX why does this tolerance have to be so high?
assert diff.max_absolute() < 0.5, diff.max_absolute()
def exercise_negative_parameters(verbose=0):
structure_default = xray.structure(
crystal_symmetry=crystal.symmetry(unit_cell=((10, 13, 17, 75, 80, 85)),
space_group_symbol="P 1"),
scatterers=flex.xray_scatterer(
[xray.scatterer(label="C", site=(0, 0, 0), u=0.25)]))
negative_gaussian = eltbx.xray_scattering.gaussian((1, 2), (2, 3), -4)
for i_trial in range(7):
structure = structure_default.deep_copy_scatterers()
scatterer = structure.scatterers()[0]
if (i_trial == 1):
scatterer.occupancy *= -1
elif (i_trial == 2):
structure.scattering_type_registry(
custom_dict={"C": negative_gaussian})
elif (i_trial == 3):
scatterer.u_iso *= -1
elif (i_trial == 4):
u_cart = adptbx.random_u_cart(u_scale=1, u_min=-1.1)
assert max(adptbx.eigenvalues(u_cart)) < 0
u_star = adptbx.u_cart_as_u_star(structure.unit_cell(), u_cart)
scatterer.u_star = u_star
scatterer.flags.set_use_u_aniso_only()
elif (i_trial == 5):
scatterer.fp = -10
elif (i_trial == 6):
scatterer.fp = -3
f_direct = structure.structure_factors(d_min=1,
algorithm="direct",
cos_sin_table=False).f_calc()
f_fft = structure.structure_factors(d_min=1,
algorithm="fft",
quality_factor=1.e8,
wing_cutoff=1.e-10).f_calc()
if (i_trial == 2):
assert negative_gaussian.at_d_star_sq(
f_fft.d_star_sq().data()).all_lt(0)
if (i_trial in [5, 6]):
f = structure.scattering_type_registry().gaussian_not_optional(
scattering_type="C").at_d_star_sq(f_fft.d_star_sq().data())
if (i_trial == 5):
assert flex.max(f) + scatterer.fp < 0
else:
assert flex.max(f) + scatterer.fp > 0
assert flex.min(f) + scatterer.fp < 0
cc = flex.linear_correlation(abs(f_direct).data(),
abs(f_fft).data()).coefficient()
if (cc < 0.999):
raise AssertionError("i_trial=%d, correlation=%.6g" %
(i_trial, cc))
elif (0 or verbose):
print("correlation=%.6g" % cc)
#
# very simple test of gradient calculations with negative parameters
structure_factor_gradients = \
cctbx.xray.structure_factors.gradients(
miller_set=f_direct,
cos_sin_table=False)
target_functor = xray.target_functors.intensity_correlation(
f_obs=abs(f_direct))
target_result = target_functor(f_fft, True)
xray.set_scatterer_grad_flags(scatterers=structure.scatterers(),
site=True,
u_iso=True,
u_aniso=True,
occupancy=True,
fp=True,
fdp=True)
for algorithm in ["direct", "fft"]:
grads = structure_factor_gradients(
xray_structure=structure,
u_iso_refinable_params=None,
miller_set=f_direct,
d_target_d_f_calc=target_result.derivatives(),
n_parameters=structure.n_parameters(),
algorithm=algorithm).packed()
def exercise(target_functor, data_type, parameter_name, space_group_info,
anomalous_flag,
cartesian_flag,
n_elements=9,
d_min=2.5,
shake_sigma=0.25,
test_hard=True, verbose=0):
assert data_type == 'F' or data_type == 'F^2'
if (data_type == 'F^2'
and not target_functor == xray.unified_least_squares_residual): return
if (parameter_name != "site" and cartesian_flag == True): return
if (parameter_name == "fdp" and not anomalous_flag): return
structure_ideal = random_structure.xray_structure(
space_group_info,
elements=(("O","N","C")*(n_elements))[:n_elements],
volume_per_atom=100,
random_f_prime_d_min=d_min,
random_f_double_prime=anomalous_flag,
use_u_aniso = True,
use_u_iso = False,
random_u_cart_scale = 0.3,
random_u_iso = False,
random_occupancy=True)
if(parameter_name in ["u_star", "u_iso"]): shake_sigma = shake_sigma / 2.
random_structure.random_modify_adp_and_adp_flags(
scatterers = structure_ideal.scatterers(),
random_u_iso_scale = 0.3,
random_u_iso_min = 0.0,
parameter_name = parameter_name)
rnd_f_calc = structure_ideal.structure_factors(
anomalous_flag=anomalous_flag, d_min=d_min, algorithm="direct").f_calc()
if data_type == 'F':
y_obs = abs(rnd_f_calc)
elif data_type == 'F^2':
y_obs = rnd_f_calc.norm()
y_obs.set_observation_type_xray_intensity()
structure_shake = structure_ideal.random_modify_parameters(
parameter_name, shake_sigma, vary_z_only=False)
assert tuple(structure_ideal.special_position_indices()) \
== tuple(structure_shake.special_position_indices())
target_ftor = target_functor(y_obs)
for structure in (structure_ideal, structure_shake)[:]: #SWITCH
f_calc = y_obs.structure_factors_from_scatterers(
xray_structure=structure,
algorithm="direct").f_calc()
target_result = target_ftor(f_calc, compute_derivatives=True)
if (structure == structure_ideal):
assert abs(target_result.target()) < 1.e-5
gradient_flags=xray.structure_factors.gradient_flags(
site=(parameter_name=="site" or random.choice((False,True))),
u_iso=(parameter_name=="u_iso" or random.choice((False,True))),
u_aniso=(parameter_name=="u_star" or random.choice((False,True))),
occupancy=(parameter_name=="occupancy" or random.choice((False,True))),
fp=(parameter_name=="fp" or random.choice((False,True))),
fdp=(parameter_name=="fdp" or (anomalous_flag
and random.choice((False,True)))))
xray.set_scatterer_grad_flags(scatterers = structure.scatterers(),
site = gradient_flags.site,
u_iso = gradient_flags.u_iso,
u_aniso = gradient_flags.u_aniso,
occupancy = gradient_flags.occupancy,
fp = gradient_flags.fp,
fdp = gradient_flags.fdp)
grad_flags_counts = xray.scatterer_grad_flags_counts(structure.scatterers())
sf = xray.structure_factors.gradients_direct(
xray_structure=structure,
u_iso_refinable_params=None,
miller_set=y_obs,
d_target_d_f_calc=target_result.derivatives(),
n_parameters=0)
if (parameter_name == "site"):
d_analytical = sf.d_target_d_site_frac()
if (cartesian_flag): # average d_analytical or d_numerical, but not both
structure_ideal.apply_special_position_ops_d_target_d_site(d_analytical)
if (cartesian_flag):
d_analytical = sf.d_target_d_site_cart()
d_numerical = finite_differences_site(
cartesian_flag, target_ftor, structure)
if (not cartesian_flag): # aver. d_analytical or d_numerical, but not both
structure_ideal.apply_special_position_ops_d_target_d_site(d_numerical)
elif (parameter_name == "u_star" and grad_flags_counts.use_u_aniso > 0):
d_analytical = sf.d_target_d_u_star()
d_numerical = finite_differences_u_star(target_ftor, structure)
else:
if (parameter_name == "occupancy"):
d_analytical = sf.d_target_d_occupancy()
elif (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0):
d_analytical = sf.d_target_d_u_iso()
elif (parameter_name == "fp"):
d_analytical = sf.d_target_d_fp()
elif (parameter_name == "fdp"):
d_analytical = sf.d_target_d_fdp()
else:
raise RuntimeError
d_numerical = finite_differences_scalar(
parameter_name, target_ftor, structure)
linear_regression_test(d_analytical, d_numerical, test_hard,
verbose=verbose)
if (parameter_name == "u_iso" and grad_flags_counts.use_u_iso > 0):
u_iso_refinable_params = flex.double()
for scatterer in structure.scatterers():
scatterer.flags.set_tan_u_iso(True)
scatterer.flags.set_grad_u_iso(gradient_flags.u_iso)
param = random.randint(90,120)
scatterer.flags.param= param
value = math.tan(scatterer.u_iso*math.pi/adptbx.b_as_u(param)-math.pi/2)
u_iso_refinable_params.append(value)
sf = xray.structure_factors.gradients_direct(
xray_structure=structure,
u_iso_refinable_params = u_iso_refinable_params,
miller_set=y_obs,
d_target_d_f_calc=target_result.derivatives(),
n_parameters=0)
d_analytical = sf.d_target_d_u_iso()
d_numerical = finite_differences_scalar("tan_u_iso",target_ftor, structure)
linear_regression_test(d_analytical, d_numerical, test_hard,
verbose=verbose)
def get_weight(fmodel=None,
restraints_manager=None,
sites=None,
transformations=None,
u_iso=None,
ncs_restraints_group_list=None,
refine_selection=None,
minimized_obj=None):
"""
Calculates weights for refinements by slightly shaking the minimized
parameters and taking the ratio:
(restraint manager grad norm / parameters gradient norm)
When calling this function during refinement macro cycle, the minimized
object, "minimized_obj" , may contains fmodel, restraints_manager,
refinement type info (sites, transformations, u_iso) and
ncs_restraints_group_list.
Args:
fmodel : F-model object
restraints_manager: Restraints manager object
sites (bool): Refine by sites
u_iso (bool): Refine using u_iso
transformations (bool): Refine using transformations
rotations, translations (matrix objects):
ncs_restraints_group_list: list of ncs_restraint_group objects
refine_selection (flex.size_t): selection of all ncs related copies and
non ncs related parts to be included in selection (to be refined)
minimized_obj:Minimization object containing all the other
parameters above
Returns:
weight (int):
Example:
>>>get_weight(minimized_obj=minimized_obj)
or
>>>get_weight(fmodel=fmodel,
restraints_manager=grm,
sites=sites,
transformations=transformations,
u_iso=u_iso,
ncs_restraints_group_list=ncs_restraints_group_list,
refine_selection=refine_selection)
"""
grm = restraints_manager
extended_ncs_selection = None
# extract parameters from minimized_obj
if minimized_obj:
mo = minimized_obj
fmodel = mo.fmodel
grm = mo.grm
sites = mo.sites
transformations = mo.transformations
u_iso = mo.u_iso
ncs_restraints_group_list = mo.ncs_restraints_group_list
# del: consider deleting the code below
# if hasattr(mo,'extended_ncs_selection'):
# extended_ncs_selection = mo.extended_ncs_selection
# if not extended_ncs_selection:
# extended_ncs_selection = get_extended_ncs_selection(
# ncs_restraints_group_list=ncs_restraints_group_list,
# refine_selection=refine_selection)
# make sure sufficient input is provided
assert bool(fmodel), 'F-model is not provided'
assert bool(grm), 'F-restraints_manager is not provided'
assert [sites, transformations,
u_iso].count(True) == 1, 'Refinement type Error'
#
have_transforms = ncs_restraints_group_list != []
fmdc = fmodel.deep_copy()
if sites:
fmdc.xray_structure.shake_sites_in_place(mean_distance=0.3)
elif u_iso:
fmdc.xray_structure.shake_adp()
elif transformations and have_transforms:
x = concatenate_rot_tran(
ncs_restraints_group_list=ncs_restraints_group_list)
x = shake_transformations(x=x,
shake_angles_sigma=0.035,
shake_translation_sigma=0.5)
fmdc.update_xray_structure(xray_structure=fmdc.xray_structure,
update_f_calc=True)
fmdc.xray_structure.scatterers().flags_set_grads(state=False)
if sites or transformations:
xray.set_scatterer_grad_flags(
scatterers=fmdc.xray_structure.scatterers(), site=True)
# fmodel gradients
gxc = flex.vec3_double(
fmdc.one_time_gradients_wrt_atomic_parameters(site=True).packed())
# restraints manager, energy sites gradients
gc = grm.energies_sites(sites_cart=fmdc.xray_structure.sites_cart(),
compute_gradients=True).gradients
elif u_iso:
# Create energies_site gradient, to create geometry_restraints_manager
# plain_pair_sym_table needed for the energies_adp_iso
import mmtbx.refinement.adp_refinement
temp = mmtbx.refinement.adp_refinement.adp_restraints_master_params
iso_restraints = temp.extract().iso
gc = grm.energies_sites(sites_cart=fmdc.xray_structure.sites_cart(),
compute_gradients=True).gradients
xray.set_scatterer_grad_flags(
scatterers=fmdc.xray_structure.scatterers(), u_iso=True)
# fmodel gradients
gxc = fmdc.one_time_gradients_wrt_atomic_parameters(
u_iso=True).as_double()
# manager restraints, energy sites gradients
gc = grm.energies_adp_iso(
xray_structure=fmdc.xray_structure,
parameters=iso_restraints,
use_u_local_only=iso_restraints.use_u_local_only,
use_hd=False,
compute_gradients=True).gradients
if transformations and have_transforms:
# Apply NCS relations to gradients
gxc = compute_transform_grad(
grad_wrt_xyz=gxc.as_double(),
ncs_restraints_group_list=ncs_restraints_group_list,
xyz_asu=fmdc.xray_structure.sites_cart(),
x=x)
gc = compute_transform_grad(
grad_wrt_xyz=gc.as_double(),
ncs_restraints_group_list=ncs_restraints_group_list,
xyz_asu=fmdc.xray_structure.sites_cart(),
x=x)
weight = 1.
gc_norm = gc.norm()
gxc_norm = gxc.norm()
if (gxc_norm != 0.0):
weight = gc_norm / gxc_norm
weight = min(weight, 1e6) # limit the weight max value
return weight
def exercise(target_functor, data_type, space_group_info, anomalous_flag,
gradient_flags, occupancy_penalty,
n_elements=9, d_min=None, shake_sigma=0.1,
verbose=0,tan_u_iso=False, param = 0):
assert data_type == 'F' or data_type == 'F^2'
if (data_type == 'F^2'
and not target_functor == xray.unified_least_squares_residual): return
structure_ideal = random_structure.xray_structure(
space_group_info,
elements=(("O","N","C")*(n_elements))[:n_elements],#("Se",)*n_elements,
volume_per_atom=200,
random_u_iso=True,
random_u_cart_scale=.3,
random_occupancy=True,
use_u_aniso = True)
random_structure.random_modify_adp_and_adp_flags(
scatterers = structure_ideal.scatterers(),
random_u_iso_scale = 0.3,
random_u_iso_min = 0.0)
xray.set_scatterer_grad_flags(scatterers = structure_ideal.scatterers(),
site = gradient_flags.site,
u_iso = gradient_flags.u_iso,
u_aniso = gradient_flags.u_aniso,
occupancy = gradient_flags.occupancy,
fp = gradient_flags.fp,
fdp = gradient_flags.fdp,
tan_u_iso = tan_u_iso,
param = param)
if(0):
print
for sc in structure_ideal.scatterers():
print sc.flags.use_u_iso(),sc.flags.grad_u_iso(),sc.flags.use_u_aniso(),\
sc.flags.grad_u_aniso(),sc.u_iso, sc.u_star,sc.flags.tan_u_iso(),\
sc.flags.param, sc.occupancy
rnd_f_calc = structure_ideal.structure_factors(
anomalous_flag=anomalous_flag,
d_min=d_min,
algorithm="direct",
cos_sin_table=True).f_calc()
if data_type == "F":
y_obs = abs(rnd_f_calc)
elif data_type == "F^2":
y_obs = rnd_f_calc.norm()
y_obs.set_observation_type_xray_intensity()
else:
raise "Error: invalid data type: %s" % data_type
if (0 or verbose):
print "structure_ideal:"
structure_ideal.show_summary().show_scatterers()
print "n_special_positions:", \
structure_ideal.special_position_indices().size()
print
structure_ideal_cp = structure_ideal.deep_copy_scatterers()
structure_shake = structure_ideal
if (gradient_flags.site):
structure_shake = structure_shake.random_modify_parameters(
"site", shake_sigma)
if (gradient_flags.occupancy):
structure_shake = structure_shake.random_modify_parameters(
"occupancy", shake_sigma)
if (occupancy_penalty is not None):
structure_shake.scatterers()[-1].occupancy = 0
if (gradient_flags.u_aniso):
shift_u_aniso(structure_shake, 0.001)
if (gradient_flags.u_iso):
shift_u_iso(structure_shake, 0.1)
assert tuple(structure_ideal.special_position_indices()) \
== tuple(structure_shake.special_position_indices())
if (0 or verbose):
print "structure_shake:"
structure_shake.show_summary().show_scatterers()
print
for i_trial in xrange(10):
try:
minimizer = xray.minimization.lbfgs(
target_functor=target_functor(y_obs),
xray_structure=structure_shake,
occupancy_penalty=occupancy_penalty,
structure_factor_algorithm="direct")
except RuntimeError, e:
if (str(e).find("debye_waller_factor_exp: arg_limit exceeded") < 0):
raise
else:
break
def exercise(target_functor, data_type, space_group_info, anomalous_flag,
gradient_flags, occupancy_penalty,
n_elements=9, d_min=None, shake_sigma=0.1,
verbose=0,tan_u_iso=False, param = 0):
assert data_type == 'F' or data_type == 'F^2'
if (data_type == 'F^2'
and not target_functor == xray.unified_least_squares_residual): return
structure_ideal = random_structure.xray_structure(
space_group_info,
elements=(("O","N","C")*(n_elements))[:n_elements],#("Se",)*n_elements,
volume_per_atom=200,
random_u_iso=True,
random_u_cart_scale=.3,
random_occupancy=True,
use_u_aniso = True)
random_structure.random_modify_adp_and_adp_flags(
scatterers = structure_ideal.scatterers(),
random_u_iso_scale = 0.3,
random_u_iso_min = 0.0)
xray.set_scatterer_grad_flags(scatterers = structure_ideal.scatterers(),
site = gradient_flags.site,
u_iso = gradient_flags.u_iso,
u_aniso = gradient_flags.u_aniso,
occupancy = gradient_flags.occupancy,
fp = gradient_flags.fp,
fdp = gradient_flags.fdp,
tan_u_iso = tan_u_iso,
param = param)
if(0):
print()
for sc in structure_ideal.scatterers():
print(sc.flags.use_u_iso(),sc.flags.grad_u_iso(),sc.flags.use_u_aniso(),\
sc.flags.grad_u_aniso(),sc.u_iso, sc.u_star,sc.flags.tan_u_iso(),\
sc.flags.param, sc.occupancy)
rnd_f_calc = structure_ideal.structure_factors(
anomalous_flag=anomalous_flag,
d_min=d_min,
algorithm="direct",
cos_sin_table=True).f_calc()
if data_type == "F":
y_obs = abs(rnd_f_calc)
elif data_type == "F^2":
y_obs = rnd_f_calc.norm()
y_obs.set_observation_type_xray_intensity()
else:
raise "Error: invalid data type: %s" % data_type
if (0 or verbose):
print("structure_ideal:")
structure_ideal.show_summary().show_scatterers()
print("n_special_positions:", \
structure_ideal.special_position_indices().size())
print()
structure_ideal_cp = structure_ideal.deep_copy_scatterers()
structure_shake = structure_ideal
if (gradient_flags.site):
structure_shake = structure_shake.random_modify_parameters(
"site", shake_sigma)
if (gradient_flags.occupancy):
structure_shake = structure_shake.random_modify_parameters(
"occupancy", shake_sigma)
if (occupancy_penalty is not None):
structure_shake.scatterers()[-1].occupancy = 0
if (gradient_flags.u_aniso):
shift_u_aniso(structure_shake, 0.001)
if (gradient_flags.u_iso):
shift_u_iso(structure_shake, 0.1)
assert tuple(structure_ideal.special_position_indices()) \
== tuple(structure_shake.special_position_indices())
if (0 or verbose):
print("structure_shake:")
structure_shake.show_summary().show_scatterers()
print()
for i_trial in range(10):
try:
minimizer = xray.minimization.lbfgs(
target_functor=target_functor(y_obs),
xray_structure=structure_shake,
occupancy_penalty=occupancy_penalty,
structure_factor_algorithm="direct")
except RuntimeError as e:
if (str(e).find("debye_waller_factor_exp: arg_limit exceeded") < 0):
raise
else:
break
else:
raise RuntimeError("Too many xray.minimization.lbfgs failures.")
if (0 or verbose):
print("first:", minimizer.first_target_value)
print("final:", minimizer.final_target_value)
print()
assert minimizer.final_target_value < minimizer.first_target_value
if (0 or verbose):
print("minimized structure_shake:")
structure_shake.show_summary().show_scatterers()
print()
f_final = y_obs.structure_factors_from_scatterers(
xray_structure=structure_shake,
algorithm="direct",
cos_sin_table=True).f_calc()
if data_type == 'F':
f_final = abs(f_final)
elif data_type == 'F^2':
f_final = f_final.norm()
c = flex.linear_correlation(y_obs.data(), f_final.data())
assert c.is_well_defined()
if (0 or verbose):
label = gradient_flags.string_of_true()
if (anomalous_flag):
label += ",anomalous"
print("correlation: %10.8f" % c.coefficient(), label)
print()
c_coefficient = c.coefficient()
if(c_coefficient <= 0.999):
print(c_coefficient)
if data_type == 'F':
assert c_coefficient > 0.999
elif data_type == 'F^2':
assert c_coefficient > 0.9