def compute_functional_and_gradients(O):
O.number_of_function_evaluations += 1
O.unpack_parameters()
try:
f_calc = O.f_obs.structure_factors_from_scatterers(
xray_structure=O.xray_structure,
algorithm="direct",
cos_sin_table=False).f_calc()
except RuntimeError as e:
print("RuntimeError f_calc:", e)
sys.stdout.flush()
raise RuntimeError("f_calc")
ls = xray.targets_least_squares(compute_scale_using_all_data=False,
obs_type="F",
obs=O.f_obs.data(),
weights=O.weights,
r_free_flags=None,
f_calc=f_calc.data(),
derivatives_depth=2,
scale_factor=1)
gact = O.xray_structure.grads_and_curvs_target_simple(
miller_indices=O.f_obs.indices(),
da_db=ls.gradients_work(),
daa_dbb_dab=ls.hessians_work())
g_all = gact.grads
g_active = flex.double()
i_all = 0
sstab = O.xray_structure.site_symmetry_table()
for i_sc, sc in enumerate(O.xray_structure.scatterers()):
assert sc.flags.use_u_iso()
assert not sc.flags.use_u_aniso()
site_symmetry = sstab.get(i_sc)
if (site_symmetry.is_point_group_1()):
np = 3
else:
np = site_symmetry.site_constraints().n_independent_params()
g_active.extend(g_all[i_all:i_all + np + 1])
np += 4 # u_iso, occ, fp, fdp
i_all += np
assert i_all == g_all.size()
assert g_active.size() == O.x.size()
O.f_last = ls.target_work()
O.g_last = g_active / O.p_as_x
return O.f_last, O.g_last
def e_pot(O, sites_moved):
if (O.last_sites_moved is None
or O.last_sites_moved.id() != sites_moved.id()):
O.last_sites_moved = sites_moved
xs = O.xray_structure
assert len(sites_moved) == xs.scatterers().size()
xs.set_sites_cart(sites_cart=sites_moved)
f_calc = O.f_obs.structure_factors_from_scatterers(
xray_structure=xs, algorithm="direct",
cos_sin_table=False).f_calc()
from cctbx import xray
tg = xray.targets_least_squares(compute_scale_using_all_data=True,
obs_type="F",
obs=O.f_obs.data(),
weights=None,
r_free_flags=None,
f_calc=f_calc.data(),
derivatives_depth=0,
scale_factor=0)
O.f = tg.target_work()
return O.f
def e_pot(O, sites_moved):
if ( O.last_sites_moved is None
or O.last_sites_moved.id() != sites_moved.id()):
O.last_sites_moved = sites_moved
xs = O.xray_structure
assert len(sites_moved) == xs.scatterers().size()
xs.set_sites_cart(sites_cart=sites_moved)
f_calc = O.f_obs.structure_factors_from_scatterers(
xray_structure=xs, algorithm="direct", cos_sin_table=False).f_calc()
from cctbx import xray
tg = xray.targets_least_squares(
compute_scale_using_all_data=True,
obs_type="F",
obs=O.f_obs.data(),
weights=None,
r_free_flags=None,
f_calc=f_calc.data(),
derivatives_depth=0,
scale_factor=0)
O.f = tg.target_work()
return O.f
def get_curvs_work(f_obs, weights, xray_structure, lim_eps):
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure=xray_structure, algorithm="direct",
cos_sin_table=False).f_calc()
ls = xray.targets_least_squares(compute_scale_using_all_data=False,
obs_type="F",
obs=abs(f_calc).data(),
weights=weights,
r_free_flags=None,
f_calc=f_calc.data(),
derivatives_depth=2,
scale_factor=1)
gact = xray_structure.grads_and_curvs_target_simple(
miller_indices=f_obs.indices(),
da_db=ls.gradients_work(),
daa_dbb_dab=ls.hessians_work())
c_all = gact.curvs
c_active_site = flex.double()
c_active_u_iso = flex.double()
i_all = 0
sstab = xray_structure.site_symmetry_table()
for i_sc, sc in enumerate(xray_structure.scatterers()):
assert sc.flags.use_u_iso()
assert not sc.flags.use_u_aniso()
site_symmetry = sstab.get(i_sc)
if (site_symmetry.is_point_group_1()):
np = 3
else:
np = site_symmetry.site_constraints().n_independent_params()
c_active_site.extend(c_all[i_all:i_all + np])
c_active_u_iso.append(c_all[i_all + np])
np += 4 # u_iso, occ, fp, fdp
i_all += np
assert i_all == c_all.size()
#
cf_site = curv_filter(curvs=c_active_site, lim_eps=lim_eps)
cf_u_iso = curv_filter(curvs=c_active_u_iso, lim_eps=lim_eps)
#
result = flex.double()
i_all = 0
sstab = xray_structure.site_symmetry_table()
for i_sc, sc in enumerate(xray_structure.scatterers()):
assert sc.flags.use_u_iso()
assert not sc.flags.use_u_aniso()
site_symmetry = sstab.get(i_sc)
if (site_symmetry.is_point_group_1()):
np = 3
else:
np = site_symmetry.site_constraints().n_independent_params()
result.extend(cf_site.apply(c_all[i_all:i_all + np]))
result.extend(cf_u_iso.apply(c_all[i_all + np:i_all + np + 1]))
np += 4 # u_iso, occ, fp, fdp
i_all += np
assert i_all == c_all.size()
print "curv site below limit: %d of %d" % (
cf_site.n_below_limit, cf_site.n_below_limit + cf_site.n_above_limit)
print "curv u_iso below limit: %d of %d" % (cf_u_iso.n_below_limit,
cf_u_iso.n_below_limit +
cf_u_iso.n_above_limit)
sys.stdout.flush()
return result
class ls_refinement(object):
def __init__(O,
f_obs,
xray_structure,
params,
lbfgs_termination_params=None,
lbfgs_exception_handling_params=None,
reference_structure=None):
O.f_obs = f_obs
O.weights = flex.double(f_obs.data().size(), 1)
O.xray_structure = xray_structure
O.params = params
O.reference_structure = reference_structure
O.curvs_work = get_curvs_work(f_obs=O.f_obs,
weights=O.weights,
xray_structure=O.xray_structure,
lim_eps=O.params.curv_filter_lim_eps)
if (O.params.curvature_rescaling):
O.p_as_x = flex.sqrt(O.curvs_work)
else:
O.p_as_x = flex.double(O.curvs_work.size(), 1)
O.pack_parameters()
O.number_of_function_evaluations = -1
O.number_of_lbfgs_iterations = -1
O.f_start, O.g_start = O.compute_functional_and_gradients()
O.history = []
O.callback_after_step(minimizer=None)
if (params.minimizer == "lbfgs"):
O.minimizer = scitbx.lbfgs.run(
target_evaluator=O,
termination_params=lbfgs_termination_params,
exception_handling_params=lbfgs_exception_handling_params)
elif (params.minimizer == "lbfgs_raw"):
O.run_lbfgs_raw()
elif (params.minimizer == "lbfgsb"):
O.run_lbfgsb()
O.f_final, O.g_final = O.compute_functional_and_gradients()
def pack_parameters(O):
O.x = flex.double()
O.l = flex.double()
O.u = flex.double()
O.nbd = flex.int()
sstab = O.xray_structure.site_symmetry_table()
for i_sc, sc in enumerate(O.xray_structure.scatterers()):
assert sc.flags.use_u_iso()
assert not sc.flags.use_u_aniso()
#
site_symmetry = sstab.get(i_sc)
if (site_symmetry.is_point_group_1()):
p = sc.site
else:
p = site_symmetry.site_constraints().independent_params(
all_params=sc.site)
O.x.extend(flex.double(p))
O.l.resize(O.x.size(), 0)
O.u.resize(O.x.size(), 0)
O.nbd.resize(O.x.size(), 0)
#
O.x.append(sc.u_iso)
O.l.append(0)
O.u.append(0)
O.nbd.append(1)
O.x *= O.p_as_x
def unpack_parameters(O):
p = O.x / O.p_as_x
ix = 0
sstab = O.xray_structure.site_symmetry_table()
for i_sc, sc in enumerate(O.xray_structure.scatterers()):
site_symmetry = sstab.get(i_sc)
if (site_symmetry.is_point_group_1()):
sc.site = tuple(p[ix:ix + 3])
ix += 3
else:
constr = site_symmetry.site_constraints()
np = constr.n_independent_params()
sc.site = constr.all_params(independent_params=tuple(p[ix:ix +
np]))
ix += np
sc.u_iso = p[ix]
ix += 1
assert ix == O.x.size()
def adjust_stp(O,
stp,
csd,
large_shift_site=0.1,
large_shift_u_iso_factor=0.5,
min_large_shift_u_iso=b_as_u(1)):
print "adjust_stp", stp
assert csd.size() == O.x.size()
max_stp = 100
ix = 0
uc = O.xray_structure.unit_cell()
sstab = O.xray_structure.site_symmetry_table()
for i_sc, sc in enumerate(O.xray_structure.scatterers()):
site_symmetry = sstab.get(i_sc)
if (site_symmetry.is_point_group_1()):
np = 3
for i, ucp in enumerate(uc.parameters()[:3]):
lsx = large_shift_site / ucp \
* O.p_as_x[ix+i]
if (abs(stp * csd[ix + i]) > lsx):
max_stp = min(max_stp, abs(lsx / csd[ix]))
else:
constr = site_symmetry.site_constraints()
np = constr.n_independent_params()
raise RuntimeError(
"SPECIAL POSITION LARGE SHIFT NOT IMPLEMENTED.")
ix += np
u_iso = O.x[ix] / O.p_as_x[ix]
lux = max(min_large_shift_u_iso, abs(u_iso) * large_shift_u_iso_factor) \
* O.p_as_x[ix]
if (abs(stp * csd[ix]) > lux):
max_stp = min(max_stp, abs(lux / csd[ix]))
ix += 1
assert ix == O.x.size()
print "max_stp:", max_stp
sys.stdout.flush()
if (O.params.use_max_stp):
return min(max_stp, stp)
return stp
def compute_functional_and_gradients(O):
O.number_of_function_evaluations += 1
O.unpack_parameters()
try:
f_calc = O.f_obs.structure_factors_from_scatterers(
xray_structure=O.xray_structure,
algorithm="direct",
cos_sin_table=False).f_calc()
except RuntimeError, e:
print "RuntimeError f_calc:", e
sys.stdout.flush()
raise RuntimeError("f_calc")
ls = xray.targets_least_squares(compute_scale_using_all_data=False,
obs_type="F",
obs=O.f_obs.data(),
weights=O.weights,
r_free_flags=None,
f_calc=f_calc.data(),
derivatives_depth=2,
scale_factor=1)
gact = O.xray_structure.grads_and_curvs_target_simple(
miller_indices=O.f_obs.indices(),
da_db=ls.gradients_work(),
daa_dbb_dab=ls.hessians_work())
g_all = gact.grads
g_active = flex.double()
i_all = 0
sstab = O.xray_structure.site_symmetry_table()
for i_sc, sc in enumerate(O.xray_structure.scatterers()):
assert sc.flags.use_u_iso()
assert not sc.flags.use_u_aniso()
site_symmetry = sstab.get(i_sc)
if (site_symmetry.is_point_group_1()):
np = 3
else:
np = site_symmetry.site_constraints().n_independent_params()
g_active.extend(g_all[i_all:i_all + np + 1])
np += 4 # u_iso, occ, fp, fdp
i_all += np
assert i_all == g_all.size()
assert g_active.size() == O.x.size()
O.f_last = ls.target_work()
O.g_last = g_active / O.p_as_x
return O.f_last, O.g_last
def __get_tg(O, f_cbs, derivatives_depth=2, target=Auto, weights=Auto):
if (target is Auto): target = O.params.target
if (weights is Auto): weights = O.weights
if (target.obs_type == "F"):
obs = O.f_obs
else:
obs = O.i_obs
if (target.type == "ls"):
if (target.weighting_scheme in ["unit", "shelxl_wght_once"]):
return xray.targets_least_squares(
compute_scale_using_all_data=True,
obs_type=target.obs_type,
obs=obs.data(),
weights=O.weights,
r_free_flags=None,
f_calc=f_cbs.data(),
derivatives_depth=derivatives_depth,
scale_factor=0)
if (target.weighting_scheme != "shelxl_wght"):
raise RuntimeError(
"Unknown: target.weighting_scheme = %s" % target.weighting_scheme)
assert target.obs_type == "I"
if (derivatives_depth == 0):
return O.calc_shelxl_wght_ls(f_cbs=f_cbs.data(), need="t")
return xray.targets.shelxl_wght_ls(
f_obs=O.f_obs.data(),
i_obs=O.i_obs.data(),
i_sig=O.i_obs.sigmas(),
f_calc=f_cbs.data(),
i_calc=None,
wa=0.1,
wb=0)
elif (target.type == "cc"):
return xray.targets_correlation(
obs_type=target.obs_type,
obs=obs.data(),
weights=O.weights,
r_free_flags=None,
f_calc=f_cbs.data(),
derivatives_depth=derivatives_depth)
elif (target.type == "r1"):
assert target.obs_type == "F"
assert target.weighting_scheme == "unit"
from cctbx.xray.targets import r1
return r1.target(
f_obs=O.f_obs.data(),
f_calc=f_cbs.data())
elif (target.type == "ml"):
assert derivatives_depth in [0,1]
if (O.epsilons is None):
raise RuntimeError(
"target.type=ml can only be used if bulk_solvent_correction=True")
return xray.targets_maximum_likelihood_criterion(
f_obs=O.f_obs.data(),
r_free_flags=O.r_free_flags.data(),
f_calc=f_cbs.data(),
alpha=O.alpha_beta[0].data(),
beta=O.alpha_beta[1].data(),
scale_factor=1,
epsilons=O.epsilons.data().as_double(),
centric_flags=O.centric_flags.data(),
compute_gradients=bool(derivatives_depth))
raise RuntimeError("Unknown target_type: %s" % target.type)
def get_curvs_work(f_obs, weights, xray_structure, lim_eps):
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure=xray_structure,
algorithm="direct",
cos_sin_table=False).f_calc()
ls = xray.targets_least_squares(
compute_scale_using_all_data=False,
obs_type="F",
obs=abs(f_calc).data(),
weights=weights,
r_free_flags=None,
f_calc=f_calc.data(),
derivatives_depth=2,
scale_factor=1)
gact = xray_structure.grads_and_curvs_target_simple(
miller_indices=f_obs.indices(),
da_db=ls.gradients_work(),
daa_dbb_dab=ls.hessians_work())
c_all = gact.curvs
c_active_site = flex.double()
c_active_u_iso = flex.double()
i_all = 0
sstab = xray_structure.site_symmetry_table()
for i_sc,sc in enumerate(xray_structure.scatterers()):
assert sc.flags.use_u_iso()
assert not sc.flags.use_u_aniso()
site_symmetry = sstab.get(i_sc)
if (site_symmetry.is_point_group_1()):
np = 3
else:
np = site_symmetry.site_constraints().n_independent_params()
c_active_site.extend(c_all[i_all:i_all+np])
c_active_u_iso.append(c_all[i_all+np])
np += 4 # u_iso, occ, fp, fdp
i_all += np
assert i_all == c_all.size()
#
cf_site = curv_filter(curvs=c_active_site, lim_eps=lim_eps)
cf_u_iso = curv_filter(curvs=c_active_u_iso, lim_eps=lim_eps)
#
result = flex.double()
i_all = 0
sstab = xray_structure.site_symmetry_table()
for i_sc,sc in enumerate(xray_structure.scatterers()):
assert sc.flags.use_u_iso()
assert not sc.flags.use_u_aniso()
site_symmetry = sstab.get(i_sc)
if (site_symmetry.is_point_group_1()):
np = 3
else:
np = site_symmetry.site_constraints().n_independent_params()
result.extend(cf_site.apply(c_all[i_all:i_all+np]))
result.extend(cf_u_iso.apply(c_all[i_all+np:i_all+np+1]))
np += 4 # u_iso, occ, fp, fdp
i_all += np
assert i_all == c_all.size()
print "curv site below limit: %d of %d" % (
cf_site.n_below_limit,
cf_site.n_below_limit + cf_site.n_above_limit)
print "curv u_iso below limit: %d of %d" % (
cf_u_iso.n_below_limit,
cf_u_iso.n_below_limit + cf_u_iso.n_above_limit)
sys.stdout.flush()
return result
def __get_tg(O, f_cbs, derivatives_depth=2, target=Auto, weights=Auto):
if (target is Auto): target = O.params.target
if (weights is Auto): weights = O.weights
if (target.obs_type == "F"):
obs = O.f_obs
else:
obs = O.i_obs
if (target.type == "ls"):
if (target.weighting_scheme in ["unit", "shelxl_wght_once"]):
return xray.targets_least_squares(
compute_scale_using_all_data=True,
obs_type=target.obs_type,
obs=obs.data(),
weights=O.weights,
r_free_flags=None,
f_calc=f_cbs.data(),
derivatives_depth=derivatives_depth,
scale_factor=0)
if (target.weighting_scheme != "shelxl_wght"):
raise RuntimeError(
"Unknown: target.weighting_scheme = %s" % target.weighting_scheme)
assert target.obs_type == "I"
if (derivatives_depth == 0):
return O.calc_shelxl_wght_ls(f_cbs=f_cbs.data(), need="t")
return xray.targets.shelxl_wght_ls(
f_obs=O.f_obs.data(),
i_obs=O.i_obs.data(),
i_sig=O.i_obs.sigmas(),
f_calc=f_cbs.data(),
i_calc=None,
wa=0.1,
wb=0)
elif (target.type == "cc"):
return xray.targets_correlation(
obs_type=target.obs_type,
obs=obs.data(),
weights=O.weights,
r_free_flags=None,
f_calc=f_cbs.data(),
derivatives_depth=derivatives_depth)
elif (target.type == "r1"):
assert target.obs_type == "F"
assert target.weighting_scheme == "unit"
from cctbx.xray.targets import r1
return r1.target(
f_obs=O.f_obs.data(),
f_calc=f_cbs.data())
elif (target.type == "ml"):
assert derivatives_depth in [0,1]
if (O.epsilons is None):
raise RuntimeError(
"target.type=ml can only be used if bulk_solvent_correction=True")
return xray.mlf_target_and_gradients(
f_obs=O.f_obs.data(),
r_free_flags=O.r_free_flags.data(),
f_calc=f_cbs.data(),
alpha=O.alpha_beta[0].data(),
beta=O.alpha_beta[1].data(),
scale_factor=1,
epsilons=O.epsilons.data().as_double(),
centric_flags=O.centric_flags.data(),
compute_gradients=bool(derivatives_depth))
raise RuntimeError("Unknown target_type: %s" % target.type)
