def run(self):
"""Execute the script."""
# Parse the command line
self.params, _ = self.parser.parse_args(show_diff_phil=True)
if self.params.hklin is None:
self.parser.print_help()
sys.exit()
self._extract_data_from_mtz()
if self.params.fit_hyperbola:
# fit by NLLS Levenberg Marquardt algorithm
hyperbola_fit = HyperbolaFit(self.fc, self.fobs)
hyperbola_fit.restart()
normal_eqns_solving.levenberg_marquardt_iterations(
hyperbola_fit,
track_all=True,
gradient_threshold=1e-8,
step_threshold=1e-8,
tau=1e-4,
n_max_iterations=200,
)
intercept = hyperbola_fit.param[0]
print(
"Model fit described by the formula: |Fo|^2 = sqrt(|Fc|^2 + |Fe|^2)"
)
print("where |Fe| = {:.5f}\n".format(sqrt(intercept)))
print("Goodness of fit:")
gof = hyperbola_fit.goodness_of_fit()
print("SSE: {:.5g}".format(gof["SSE"]))
print("R-square: {:.5f}".format(gof["R-square"]))
print("RMSE: {:.2f}".format(gof["RMSE"]))
print()
# Set the model_fit function using the determined intercept
def hyperbola(x, c):
return flex.sqrt(flex.pow2(x) + c)
from functools import partial
self.model_fit = partial(hyperbola, c=intercept)
if self.params.plot_filename:
self._plot()
return
def __init__(self,x_obs,y_obs,w_obs,initial):
self.counter = 0
self.x = initial.deep_copy()
self.helper = levenberg_helper(initial_estimates = self.x)
self.helper.set_cpp_data(x_obs,y_obs,w_obs)
self.helper.restart()
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls = self.helper,
n_max_iterations = 5000,
track_all=True,
step_threshold = 0.0001
)
###### get esd's
self.helper.build_up()
upper = self.helper.step_equations().normal_matrix_packed_u()
nm_elem = flex.double(25)
self.c = flex.double(5)
ctr = 0
for x in xrange(5):
x_0 = ctr
for y in xrange(4,x-1,-1):
nm_elem[ 5*x+y ] = upper[x_0+(y-x)]
ctr += 1
if x!= y:
nm_elem[ 5*y+x ] = upper[x_0+(y-x)]
else:
self.c[x]=upper[x_0+(y-x)]
NM = sqr(nm_elem)
self.helper.solve()
#print list(self.helper.step_equations().cholesky_factor_packed_u())
error_matrix = NM.inverse()
self.error_diagonal = [error_matrix(a,a) for a in xrange(5)]
print "End of minimization: Converged", self.helper.counter,"cycles"
def __init__(self,x_obs,y_obs,w_obs,initial):
self.counter = 0
self.x = initial.deep_copy()
self.helper = levenberg_helper(initial_estimates = self.x)
self.helper.set_cpp_data(x_obs,y_obs,w_obs)
self.helper.restart()
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls = self.helper,
n_max_iterations = 5000,
track_all=True,
step_threshold = 0.0001
)
###### get esd's
self.helper.build_up()
upper = self.helper.step_equations().normal_matrix_packed_u()
nm_elem = flex.double(25)
self.c = flex.double(5)
ctr = 0
for x in xrange(5):
x_0 = ctr
for y in xrange(4,x-1,-1):
nm_elem[ 5*x+y ] = upper[x_0+(y-x)]
ctr += 1
if x!= y:
nm_elem[ 5*y+x ] = upper[x_0+(y-x)]
else:
self.c[x]=upper[x_0+(y-x)]
NM = sqr(nm_elem)
self.helper.solve()
#print list(self.helper.step_equations().cholesky_factor_packed_u())
error_matrix = NM.inverse()
self.error_diagonal = [error_matrix(a,a) for a in xrange(5)]
print "End of minimization: Converged", self.helper.counter,"cycles"
def exercise(self, fixed_twin_fraction):
# Create a shaken structure xs ready for refinement
xs0 = self.structure
emma_ref = xs0.as_emma_model()
xs = xs0.deep_copy_scatterers()
xs.shake_sites_in_place(rms_difference=0.15)
xs.shake_adp()
for sc in xs.scatterers():
sc.flags.set_use_u_iso(False).set_use_u_aniso(True)
sc.flags.set_grad_site(True).set_grad_u_aniso(True)
# Setup L.S. problem
connectivity_table = smtbx.utils.connectivity_table(xs)
shaken_twin_fraction = (self.twin_fraction if fixed_twin_fraction else
self.twin_fraction +
0.1 * flex.random_double())
# 2nd domain in __init__
twin_components = (xray.twin_component(twin_law=self.twin_law.r(),
value=shaken_twin_fraction,
grad=not fixed_twin_fraction), )
reparametrisation = constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=connectivity_table,
twin_fractions=twin_components)
obs = self.fo_sq.as_xray_observations(twin_components=twin_components)
ls = least_squares.crystallographic_ls(
obs,
reparametrisation,
weighting_scheme=least_squares.unit_weighting(),
origin_fixing_restraints_type=origin_fixing_restraints.
atomic_number_weighting)
# Refine till we get back the original structure (so we hope)
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls, gradient_threshold=1e-12, step_threshold=1e-6, track_all=True)
# Now let's start to check it all worked
assert ls.n_parameters == 63 if fixed_twin_fraction else 64
match = emma.model_matches(emma_ref,
xs.as_emma_model()).refined_matches[0]
assert match.rt.r == matrix.identity(3)
for pair in match.pairs:
assert approx_equal(match.calculate_shortest_dist(pair),
0,
eps=1e-4), pair
if fixed_twin_fraction:
assert ls.twin_fractions[0].value == self.twin_fraction
else:
assert approx_equal(ls.twin_fractions[0].value,
self.twin_fraction,
eps=1e-2)
assert approx_equal(ls.scale_factor(), 1, eps=1e-5)
assert approx_equal(ls.objective(), 0)
def exercise_levenberg_marquardt(non_linear_ls):
non_linear_ls.restart()
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls,
track_all=True,
gradient_threshold=1e-12,
step_threshold=1e-12,
tau=1e-8,
n_max_iterations=200)
print "L-M: %i iterations" % iterations.n_iterations
def exercise(self, fixed_twin_fraction):
# Create a shaken structure xs ready for refinement
xs0 = self.structure
emma_ref = xs0.as_emma_model()
xs = xs0.deep_copy_scatterers()
xs.shake_sites_in_place(rms_difference=0.15)
xs.shake_adp()
for sc in xs.scatterers():
sc.flags.set_use_u_iso(False).set_use_u_aniso(True)
sc.flags.set_grad_site(True).set_grad_u_aniso(True)
# Setup L.S. problem
connectivity_table = smtbx.utils.connectivity_table(xs)
shaken_twin_fraction = (
self.twin_fraction if fixed_twin_fraction else
self.twin_fraction + 0.1*flex.random_double())
# 2nd domain in __init__
twin_components = (xray.twin_component(
twin_law=self.twin_law.r(), value=shaken_twin_fraction,
grad=not fixed_twin_fraction),)
reparametrisation = constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=connectivity_table,
twin_fractions=twin_components)
obs = self.fo_sq.as_xray_observations(twin_components=twin_components)
ls = least_squares.crystallographic_ls(
obs, reparametrisation,
weighting_scheme=least_squares.unit_weighting(),
origin_fixing_restraints_type=
origin_fixing_restraints.atomic_number_weighting)
# Refine till we get back the original structure (so we hope)
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls,
gradient_threshold=1e-12,
step_threshold=1e-6,
track_all=True)
# Now let's start to check it all worked
assert ls.n_parameters == 63 if fixed_twin_fraction else 64
match = emma.model_matches(emma_ref, xs.as_emma_model()).refined_matches[0]
assert match.rt.r == matrix.identity(3)
for pair in match.pairs:
assert approx_equal(match.calculate_shortest_dist(pair), 0, eps=1e-4), pair
if fixed_twin_fraction:
assert ls.twin_fractions[0].value == self.twin_fraction
else:
assert approx_equal(ls.twin_fractions[0].value, self.twin_fraction,
eps=1e-2)
assert approx_equal(ls.scale_factor(), 1, eps=1e-5)
assert approx_equal(ls.objective(), 0)
def exercise_levenberg_marquardt(non_linear_ls):
non_linear_ls.restart()
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls,
track_all=True,
gradient_threshold=1e-08,
step_threshold=1e-08,
tau=1e-08,
n_max_iterations=200)
# from IPython import embed; embed(); exit()
print "L-M: %i iterations" % iterations.n_iterations
def run_smtbx_ls(mode, cod_id, i_obs, f_obs, xray_structure, params):
import smtbx.refinement
fo_sq = i_obs
assert fo_sq.sigmas() is not None
sel = (fo_sq.data() == 0) & (fo_sq.sigmas() == 0)
fo_sq = fo_sq.select(~sel)
fo_sq.select(fo_sq.sigmas() <= 0).show_array()
assert fo_sq.sigmas().all_gt(0)
if (1): # work around bug currently in smtbx weighting scheme implementation
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.data().size(), 1))
xobs = fo_sq.as_xray_observations()
tm = user_plus_sys_time()
rm = smtbx.refinement.model(
fo_sq=xobs,
xray_structure=xray_structure,
constraints=[],
restraints_manager=smtbx.refinement.restraints.manager(),
weighting_scheme=smtbx.refinement.least_squares.unit_weighting())
ls = rm.least_squares()
if (mode == "simple"):
for i_cycle in range(params.ls_simple_iterations):
ls.build_up()
try:
ls.solve_and_step_forward()
except RuntimeError as e:
if (str(e).find("cholesky.failure") <= 0): raise
print('Aborting run_smtbx_ls("simple"): cholesky.failure: %s' \
% cod_id)
break
for sc in xray_structure.scatterers():
if (sc.u_iso <= 0 or sc.u_iso > 1):
sc.u_iso = 0.05
show_cc_r1(params, "ls%02d" % (i_cycle+1), f_obs, xray_structure)
tm.show_elapsed(prefix="time smtbx_ls_simple_iterations: ")
elif (mode == "lm"):
from scitbx.lstbx import normal_eqns_solving
thresh = 1e-6
try:
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls,
gradient_threshold=thresh,
step_threshold=thresh,
tau=1e-7)
except RuntimeError as e:
if (not str(e).startswith(
"cctbx::adptbx::debye_waller_factor_exp: arg_limit exceeded")):
raise
print('Aborting run_smtbx_ls("lm"):' \
' debye_waller_factor_exp failure: %s' % cod_id)
show_cc_r1(params, "smtbx_lm", f_obs, xray_structure)
tm.show_elapsed(prefix="time levenberg_marquardt_iterations: ")
else:
raise RuntimeError('Unknown run_smtbx_ls(mode="%s")' % mode)
def exercise_levenberg_marquardt(non_linear_ls):
non_linear_ls.restart()
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls,
track_all=True,
gradient_threshold=1e-8,
step_threshold=1e-8,
tau=1e-4,
n_max_iterations=200)
assert non_linear_ls.n_equations == non_linear_ls.n_data
assert approx_equal(non_linear_ls.x, non_linear_ls.arg_min, eps=5e-4)
print "L-M: %i iterations" % iterations.n_iterations
def run_minimzer(self, values, sels, **kwargs):
self.refinery = sdfac_refinery(self.scaler, self,
self.scaler.miller_set.indices(), sels,
self.log)
self.helper = sdfac_helper(current_x=values.reference,
parameterization=self.parameterization,
refinery=self.refinery,
out=self.log)
self.iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls=self.helper,
track_all=True,
gradient_threshold=1e-08,
step_threshold=1e-08,
tau=1e-08,
n_max_iterations=200)
return self
def run_minimzer(self, values, sels, **kwargs):
from xfel.merging.algorithms.error_model.sdfac_refine_levmar import sdfac_helper
from scitbx.lstbx import normal_eqns_solving
self.refinery = sdfac_propagate_refinery(self.scaler, self, self.scaler.miller_set.indices(), sels, self.log)
self.helper = sdfac_helper(current_x = values.reference,
parameterization = self.parameterization, refinery = self.refinery,
out = self.log )
self.iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls = self.helper,
track_all=True,
gradient_threshold=1e-08,
step_threshold=1e-08,
tau=1e-08,
n_max_iterations=200)
return self
def fit_3_gaussians(self, histogram):
fitted_gaussians = []
low_idx = self.work_params.fit_limits[0]
high_idx = self.work_params.fit_limits[1]
slot_centers = flex.double(
range(self.work_params.first_slot_value,
self.work_params.first_slot_value + len(histogram)))
free_x = slot_centers[low_idx:high_idx]
slots = flex.double(histogram.astype(np.float64))
free_y = slots[low_idx:high_idx]
total_population = flex.sum(free_y)
# zero_mean = 0. # originally intended mean=0
maxidx = flex.max_index(
free_y) # but if dark subtraction (pedstal correction) is off
zero_mean = free_x[maxidx] # use this non-zero maximum instead
zero_amplitude = flex.max(free_y)
assert 1. / zero_amplitude #guard against division by zero
zero_sigma = self.work_params.gaussian_3.zero_sigma
inelastic_amplitude = 0.001
elastic_amplitude = 0.001
elastic_sigma = self.work_params.gaussian_3.zero_sigma
#elastic_mean = zero_mean+ zero_sigma*self.work_params.gaussian_3.elastic_gain_to_sigma #**
helper = self.helper_3_gaussian_factory(
initial_estimates=(zero_mean, 1.0, zero_sigma, inelastic_amplitude,
elastic_amplitude, elastic_sigma),
constants=flex.double((
self.work_params.gaussian_3.inelastic_gain_to_sigma,
self.work_params.gaussian_3.elastic_gain_to_sigma,
)),
free_x=free_x,
free_y=free_y / zero_amplitude
) # put y values on 0->1 scale for normal eqn solving
helper.restart()
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls=helper, n_max_iterations=7, gradient_threshold=1.E-3)
fitted_gaussians = helper.as_gaussians()
for item in fitted_gaussians:
item.params = (item.params[0] * zero_amplitude, item.params[1],
item.params[2]) # convert back to full scale
return fitted_gaussians
def exercise_levenberg_marquardt(non_linear_ls, plot=False):
non_linear_ls.restart()
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls,
track_all=True,
gradient_threshold=1e-8,
step_threshold=1e-8,
tau=1e-4,
n_max_iterations=200)
assert non_linear_ls.n_equations == non_linear_ls.n_data
assert approx_equal(non_linear_ls.x, non_linear_ls.arg_min, eps=5e-4)
print "L-M: %i iterations" % iterations.n_iterations
if plot:
f = open('plot.nb', 'w')
print >>f, "g=%s;" % iterations.gradient_norm_history.mathematica_form()
print >>f, "\[Mu]=%s;" % iterations.mu_history.mathematica_form()
print >>f, "ListLogPlot[{g,\[Mu]},Joined->True]"
f.close()
def exercise_levenberg_marquardt(non_linear_ls, plot=False):
non_linear_ls.restart()
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls,
track_all=True,
gradient_threshold=1e-8,
step_threshold=1e-8,
tau=1e-4,
n_max_iterations=200)
assert non_linear_ls.n_equations == non_linear_ls.n_data
assert approx_equal(non_linear_ls.x, non_linear_ls.arg_min, eps=5e-4)
print("L-M: %i iterations" % iterations.n_iterations)
if plot:
f = open('plot.nb', 'w')
print("g=%s;" % iterations.gradient_norm_history.mathematica_form(), file=f)
print("\[Mu]=%s;" % iterations.mu_history.mathematica_form(), file=f)
print("ListLogPlot[{g,\[Mu]},Joined->True]", file=f)
f.close()
def fit_3_gaussians(self,histogram):
fitted_gaussians = []
low_idx = self.work_params.fit_limits[0]
high_idx = self.work_params.fit_limits[1]
slot_centers = flex.double(xrange(self.work_params.first_slot_value,
self.work_params.first_slot_value + len(histogram)))
free_x = slot_centers[low_idx:high_idx]
slots = flex.double(histogram.astype(np.float64))
free_y = slots[low_idx:high_idx]
total_population = flex.sum(free_y)
# zero_mean = 0. # originally intended mean=0
maxidx = flex.max_index(free_y) # but if dark subtraction (pedstal correction) is off
zero_mean = free_x[maxidx] # use this non-zero maximum instead
zero_amplitude = flex.max(free_y)
assert 1./zero_amplitude #guard against division by zero
zero_sigma = self.work_params.gaussian_3.zero_sigma
inelastic_amplitude = 0.001
elastic_amplitude = 0.001
elastic_sigma = self.work_params.gaussian_3.zero_sigma
#elastic_mean = zero_mean+ zero_sigma*self.work_params.gaussian_3.elastic_gain_to_sigma #**
helper = self.helper_3_gaussian_factory(initial_estimates =
(zero_mean, 1.0, zero_sigma, inelastic_amplitude, elastic_amplitude, elastic_sigma),
constants=flex.double((self.work_params.gaussian_3.inelastic_gain_to_sigma,
self.work_params.gaussian_3.elastic_gain_to_sigma,
)),
free_x = free_x,
free_y = free_y/zero_amplitude) # put y values on 0->1 scale for normal eqn solving
helper.restart()
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls = helper,
n_max_iterations = 7,
gradient_threshold = 1.E-3)
fitted_gaussians = helper.as_gaussians()
for item in fitted_gaussians: item.params = (item.params[0] * zero_amplitude,
item.params[1], item.params[2]) # convert back to full scale
return fitted_gaussians
def fit_one_histogram_two_gaussians(self, histogram):
fitted_gaussians = []
GAIN_TO_SIGMA = self.work_params.fudge_factor.gain_to_sigma
low_idx = self.work_params.fit_limits[0]
high_idx = self.work_params.fit_limits[1]
slot_centers = flex.double(
range(self.work_params.first_slot_value,
self.work_params.first_slot_value + len(histogram)))
free_x = slot_centers[low_idx:high_idx]
#print list(free_x)
slots = flex.double(histogram.astype(np.float64))
free_y = slots[low_idx:high_idx]
# zero_mean = 0. # originally intended mean=0
maxidx = flex.max_index(
free_y) # but if dark subtraction (pedstal correction) is off
zero_mean = free_x[maxidx] # use this non-zero maximum instead
zero_amplitude = flex.max(free_y)
assert 1. / zero_amplitude #guard against division by zero
total_population = flex.sum(free_y)
zero_sigma = self.work_params.estimated_gain / GAIN_TO_SIGMA
one_amplitude = 0.001
helper = self.per_pixel_helper_factory(
initial_estimates=(zero_mean, 1.0, zero_sigma, one_amplitude),
GAIN_TO_SIGMA=GAIN_TO_SIGMA,
free_x=free_x,
free_y=free_y / zero_amplitude
) # put y values on 0->1 scale for normal eqn solving
helper.restart()
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls=helper, n_max_iterations=7, gradient_threshold=1.E-3)
print("current values after iterations", list(helper.x), end=' ')
fitted_gaussians = helper.as_gaussians()
for item in fitted_gaussians:
item.params = (item.params[0] * zero_amplitude, item.params[1],
item.params[2]) # convert back to full scale
return fitted_gaussians
def run():
import libtbx.utils
libtbx.utils.show_times_at_exit()
import sys
from libtbx.option_parser import option_parser
command_line = (option_parser().option(
None, "--normal_eqns_solving_method",
default='naive').option(None,
"--fix_random_seeds",
action='store_true',
default='naive')).process(args=sys.argv[1:])
opts = command_line.options
if opts.fix_random_seeds:
import random
random.seed(1)
flex.set_random_seed(1)
gradient_threshold = 1e-8
step_threshold = 1e-8
if opts.normal_eqns_solving_method == 'naive':
m = lambda eqns: normal_eqns_solving.naive_iterations(
eqns,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold)
elif opts.normal_eqns_solving_method == 'levenberg-marquardt':
m = lambda eqns: normal_eqns_solving.levenberg_marquardt_iterations(
eqns,
gradient_threshold=gradient_threshold,
step_threshold=gradient_threshold,
tau=1e-7)
else:
raise RuntimeError("Unknown method %s" %
opts.normal_eqns_solving_method)
for t in [
saturated_test_case(m),
sucrose_test_case(m),
symmetry_equivalent_test_case(m),
fpfdp_test_case(m),
constrained_fpfdp_test_case(m),
scalar_scaled_adp_test_case(m),
]:
t.run()
def fit_one_histogram_two_gaussians(self,histogram):
fitted_gaussians = []
GAIN_TO_SIGMA = self.work_params.fudge_factor.gain_to_sigma
low_idx = self.work_params.fit_limits[0]
high_idx = self.work_params.fit_limits[1]
slot_centers = flex.double(xrange(self.work_params.first_slot_value,
self.work_params.first_slot_value + len(histogram)))
free_x = slot_centers[low_idx:high_idx]
#print list(free_x)
slots = flex.double(histogram.astype(np.float64))
free_y = slots[low_idx:high_idx]
# zero_mean = 0. # originally intended mean=0
maxidx = flex.max_index(free_y) # but if dark subtraction (pedstal correction) is off
zero_mean = free_x[maxidx] # use this non-zero maximum instead
zero_amplitude = flex.max(free_y)
assert 1./zero_amplitude #guard against division by zero
total_population = flex.sum(free_y)
zero_sigma = self.work_params.estimated_gain / GAIN_TO_SIGMA
one_amplitude = 0.001
helper = self.per_pixel_helper_factory(initial_estimates =
(zero_mean, 1.0, zero_sigma, one_amplitude),
GAIN_TO_SIGMA=GAIN_TO_SIGMA,
free_x = free_x,
free_y = free_y/zero_amplitude) # put y values on 0->1 scale for normal eqn solving
helper.restart()
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls = helper,
n_max_iterations = 7,
gradient_threshold = 1.E-3)
print "current values after iterations", list(helper.x),
fitted_gaussians = helper.as_gaussians()
for item in fitted_gaussians: item.params = (item.params[0] * zero_amplitude,
item.params[1], item.params[2]) # convert back to full scale
return fitted_gaussians
def run():
import libtbx.utils
libtbx.utils.show_times_at_exit()
import sys
from libtbx.option_parser import option_parser
command_line = (option_parser()
.option(None, "--normal_eqns_solving_method",
default='naive')
.option(None, "--fix_random_seeds",
action='store_true',
default='naive')
).process(args=sys.argv[1:])
opts = command_line.options
if opts.fix_random_seeds:
import random
random.seed(1)
flex.set_random_seed(1)
gradient_threshold=1e-8
step_threshold=1e-8
if opts.normal_eqns_solving_method == 'naive':
m = lambda eqns: normal_eqns_solving.naive_iterations(
eqns,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold)
elif opts.normal_eqns_solving_method == 'levenberg-marquardt':
m = lambda eqns: normal_eqns_solving.levenberg_marquardt_iterations(
eqns,
gradient_threshold=gradient_threshold,
step_threshold=gradient_threshold,
tau=1e-7)
else:
raise RuntimeError("Unknown method %s" % opts.normal_eqns_solving_method)
for t in [
saturated_test_case(m),
sucrose_test_case(m),
symmetry_equivalent_test_case(m),
]:
t.run()
def run(args):
processed = iotbx.phil.process_command_line(
args=args, master_string=master_phil_str)
args = processed.remaining_args
work_params = processed.work.extract().xscale
processed.work.show()
assert len(args) == 1
observations=args[0]
choose_method=work_params.method
assert choose_method in ("G_only", "G_cycles", "G_wI", "G_wI_LM", "test_G_wI_LM_C++")
if choose_method=="G_only": method=1
if choose_method=="G_cycles": method=2
if choose_method=="G_wI": method=3
if choose_method=="G_wI_LM": method=4
# if choose_method=="test_G_wI_LM_C++": method=5
eps = work_params.eps
ref_G = work_params.ref_G
ref_I = work_params.ref_I
cycles = work_params.n_cycles
mi=None
f=open(observations)
(I, W, H, Miller_list, F)=read_cxi_merge(f)
t1=time.time()
if method==1:
from cxi_xdr_xes.XScale.minimizers.scaling_minimizer_G import xscale
print "Loading objects......."
n_frames = max(F)+1
(MI, Fr) = Load_object(I, W, H, Miller_list, F, n_frames)
fit=xscale(MI, Fr, n_frames, eps)
Gm=fit.x
(Ih,mi)=read_objects(MI)
if method==2:
from cxi_xdr_xes.XScale.minimizers.scaling_minimizer_G_cycles import xscale
N_miller=max(H)+1
Gm=flex.double(max(F)+1,1)
t1=time.time()
Ih=get_Ih(H, I, W, F, Gm, N_miller)
for i in range(cycles):
fit=xscale(I, W, H, F, flex.double(Gm), flex.double(Ih), eps)
Gm=fit.x
Ih=get_Ih(H, I, W, F, Gm, N_miller)
if method==3:
from cxi_xdr_xes.XScale.minimizers.scaling_minimizer_wI_G import xscale
N_miller=max(H)+1
G=flex.double(max(F)+1,1)
Ih=get_Ih(H, I, W, F, G, N_miller)
fit=xscale(I, W, H, F, flex.double(G), flex.double(Ih), eps)
Ih=fit.x[len(G):]*flex.double(Ih)
Gm=fit.x[:len(G)]
if method==4:
from scitbx.lstbx import normal_eqns_solving
from cxi_xdr_xes.XScale.minimizers.scaling_LM_minimizer_wI_G_cpp import xscale
N_miller=max(H)+1
G=flex.double(max(F)+1,1)
Ih=get_Ih(H, I, W, F, G, N_miller)
fit=xscale(I, W, H, F, G, Ih)
iterations = normal_eqns_solving.levenberg_marquardt_iterations(
non_linear_ls = fit,
gradient_threshold = eps)
Ih=fit.x[len(G):]*flex.double(Ih)
Gm=fit.x[:len(G)]
print "Total time", time.time()-t1
if ref_G or ref_I:
correlation(ref_G, ref_I, Gm, Ih, mi, False)
def exercise(self):
xs0 = self.structure
xs = xs0.deep_copy_scatterers()
k1, s1, li1, o1, o2 = xs.scatterers()
self.shake_point_group_3(k1)
self.shake_point_group_3(s1)
self.shake_point_group_3(li1)
self.shake_point_group_3(o1)
o2.site = tuple(
[ x*(1 + random.uniform(-self.delta_site, self.delta_site))
for x in o2.site])
o2.u_star = tuple(
[ u*(1 + random.uniform(-self.delta_u_star, self.delta_u_star))
for u in o2.u_star])
for sc in xs.scatterers():
sc.flags.set_use_u_iso(False).set_use_u_aniso(True)
sc.flags.set_grad_site(True).set_grad_u_aniso(True)
connectivity_table = smtbx.utils.connectivity_table(xs)
reparametrisation = constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=connectivity_table)
ls = least_squares.crystallographic_ls(
self.fo_sq.as_xray_observations(), reparametrisation,
weighting_scheme=least_squares.unit_weighting(),
origin_fixing_restraints_type=
origin_fixing_restraints.atomic_number_weighting)
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls,
gradient_threshold=1e-12,
step_threshold=1e-7,
track_all=True)
## Test whether refinement brought back the shaked structure to its
## original state
match = emma.model_matches(xs0.as_emma_model(),
xs.as_emma_model()).refined_matches[0]
assert match.rt.r == matrix.identity(3)
assert not match.singles1 and not match.singles2
assert match.rms < 1e-6
delta_u_carts= ( xs.scatterers().extract_u_cart(xs.unit_cell())
- xs0.scatterers().extract_u_cart(xs.unit_cell())).norms()
assert flex.abs(delta_u_carts) < 1e-6
assert approx_equal(ls.scale_factor(), 1, eps=1e-4)
## Test covariance matrix
jac_tr = reparametrisation.jacobian_transpose_matching_grad_fc()
cov = ls.covariance_matrix(
jacobian_transpose=jac_tr, normalised_by_goof=False)\
.matrix_packed_u_as_symmetric()
m, n = cov.accessor().focus()
# x,y for point group 3 sites are fixed: no variance or correlation
for i in (0, 9, 18, 27,):
assert cov.matrix_copy_block(i, 0, 2, n) == 0
# u_star coefficients u13 and u23 for point group 3 sites are fixed
# to 0: again no variance or correlation with any other param
for i in (7, 16, 25, 34,):
assert cov.matrix_copy_block(i, 0, 2, n).as_1d()\
.all_approx_equal(0., 1e-20)
# u_star coefficients u11, u22 and u12 for point group 3 sites
# are totally correlated, with variances in ratios 1:1:1/2
for i in (3, 12, 21, 30,):
assert cov[i, i] != 0
assert approx_equal(cov[i, i], cov[i+1, i+1], eps=1e-15)
assert approx_equal(cov[i, i+1]/cov[i, i], 1, eps=1e-12)
assert approx_equal(cov[i, i+3]/cov[i, i], 0.5, eps=1e-12)
if (str(e).find("cholesky.failure") <= 0): raise
print 'Aborting run_smtbx_ls("simple"): cholesky.failure: %s' \
% cod_id
break
for sc in xray_structure.scatterers():
if (sc.u_iso <= 0 or sc.u_iso > 1):
sc.u_iso = 0.05
show_cc_r1(params, "ls%02d" % (i_cycle+1), f_obs, xray_structure)
tm.show_elapsed(prefix="time smtbx_ls_simple_iterations: ")
elif (mode == "lm"):
from scitbx.lstbx import normal_eqns_solving
thresh = 1e-6
try:
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls,
gradient_threshold=thresh,
step_threshold=thresh,
tau=1e-7)
except RuntimeError, e:
if (not str(e).startswith(
"cctbx::adptbx::debye_waller_factor_exp: arg_limit exceeded")):
raise
print 'Aborting run_smtbx_ls("lm"):' \
' debye_waller_factor_exp failure: %s' % cod_id
show_cc_r1(params, "smtbx_lm", f_obs, xray_structure)
tm.show_elapsed(prefix="time levenberg_marquardt_iterations: ")
else:
raise RuntimeError('Unknown run_smtbx_ls(mode="%s")' % mode)
def remove_tmp_files(file_names):
for fn in file_names:
yc_i = self.model(self.x, t[i], only_value=True)
residuals[i] = yc_i - self.yo[i]
self.add_residuals(residuals, weights=None)
else:
# each row = derivatives of one L.S. term
derivatives = flex.double(flex.grid(m, n))
for i in xrange(m):
yc_i, der_yc_i = self.model(self.x, t[i], only_value=False)
residuals[i] = yc_i - self.yo[i]
for j in xrange(n):
derivatives[i, j] = der_yc_i[j]
self.add_equations(residuals, derivatives, weights=None)
test = my_fit(t, yo, model, flex.double(x_0))
runs = normal_eqns_solving.levenberg_marquardt_iterations(
test,
track_all=True,
gradient_threshold=1e-8,
step_threshold=1e-8,
tau=1e-4,
n_max_iterations=200)
# other choices:`
# - naive_iterations
# - naive_iterations_with_damping
# - naive_iterations_with_damping_and_shift_limit
print "#iterations: {}".format(runs.n_iterations)
print "Fitted parameters: {}".format(tuple(test.x))
print
print 'OK'
def exercise_restrained_refinement(options):
import random
random.seed(1)
flex.set_random_seed(1)
xs0 = smtbx.development.random_xray_structure(
sgtbx.space_group_info('P1'),
n_scatterers=options.n_scatterers,
elements="random")
for sc in xs0.scatterers():
sc.flags.set_grad_site(True)
sc0 = xs0.scatterers()
uc = xs0.unit_cell()
mi = xs0.build_miller_set(anomalous_flag=False, d_min=options.resolution)
fo_sq = mi.structure_factors_from_scatterers(
xs0, algorithm="direct").f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))
i, j, k, l = random.sample(xrange(options.n_scatterers), 4)
bond_proxies = geometry_restraints.shared_bond_simple_proxy()
w = 1e9
d_ij = uc.distance(sc0[i].site, sc0[j].site) * 0.8
bond_proxies.append(
geom.bond_simple_proxy(i_seqs=(i, j), distance_ideal=d_ij, weight=w))
d_jk = uc.distance(sc0[j].site, sc0[k].site) * 0.85
bond_proxies.append(
geom.bond_simple_proxy(i_seqs=(j, k), distance_ideal=d_jk, weight=w))
d_ki = min(
uc.distance(sc0[k].site, sc0[i].site) * 0.9, (d_ij + d_jk) * 0.8)
bond_proxies.append(
geom.bond_simple_proxy(i_seqs=(k, i), distance_ideal=d_ki, weight=w))
d_jl = uc.distance(sc0[j].site, sc0[l].site) * 0.9
bond_proxies.append(
geom.bond_simple_proxy(i_seqs=(j, l), distance_ideal=d_jl, weight=w))
d_lk = min(
uc.distance(sc0[l].site, sc0[k].site) * 0.8, 0.75 * (d_jk + d_jl))
bond_proxies.append(
geom.bond_simple_proxy(i_seqs=(l, k), distance_ideal=d_jl, weight=w))
restraints_manager = restraints.manager(bond_proxies=bond_proxies)
xs1 = xs0.deep_copy_scatterers()
xs1.shake_sites_in_place(rms_difference=0.1)
def ls_problem():
xs = xs1.deep_copy_scatterers()
reparametrisation = constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=smtbx.utils.connectivity_table(xs),
temperature=20)
return least_squares.crystallographic_ls(
fo_sq.as_xray_observations(),
reparametrisation=reparametrisation,
restraints_manager=restraints_manager)
gradient_threshold, step_threshold = 1e-6, 1e-6
eps = 5e-3
ls = ls_problem()
t = wall_clock_time()
cycles = normal_eqns_solving.naive_iterations(
ls,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold,
track_all=True)
if options.verbose:
print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles,
t.elapsed())
sc = ls.xray_structure.scatterers()
for p in bond_proxies:
d = uc.distance(*[sc[i_pair].site for i_pair in p.i_seqs])
assert approx_equal(d, p.distance_ideal, eps)
ls = ls_problem()
t = wall_clock_time()
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold,
tau=1e-3,
track_all=True)
if options.verbose:
print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles,
t.elapsed())
sc = ls.xray_structure.scatterers()
sc = ls.xray_structure.scatterers()
for p in bond_proxies:
d = uc.distance(*[sc[i].site for i in p.i_seqs])
assert approx_equal(d, p.distance_ideal, eps)
ls.solve_and_step_forward()
except RuntimeError, e:
if (str(e).find("cholesky.failure") <= 0): raise
print 'Aborting run_smtbx_ls("simple"): cholesky.failure: %s' \
% cod_id
break
for sc in xray_structure.scatterers():
if (sc.u_iso <= 0 or sc.u_iso > 1):
sc.u_iso = 0.05
show_cc_r1(params, "ls%02d" % (i_cycle + 1), f_obs, xray_structure)
tm.show_elapsed(prefix="time smtbx_ls_simple_iterations: ")
elif (mode == "lm"):
from scitbx.lstbx import normal_eqns_solving
thresh = 1e-6
try:
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls, gradient_threshold=thresh, step_threshold=thresh, tau=1e-7)
except RuntimeError, e:
if (not str(e).startswith(
"cctbx::adptbx::debye_waller_factor_exp: arg_limit exceeded"
)):
raise
print 'Aborting run_smtbx_ls("lm"):' \
' debye_waller_factor_exp failure: %s' % cod_id
show_cc_r1(params, "smtbx_lm", f_obs, xray_structure)
tm.show_elapsed(prefix="time levenberg_marquardt_iterations: ")
else:
raise RuntimeError('Unknown run_smtbx_ls(mode="%s")' % mode)
def remove_tmp_files(file_names):
for fn in file_names:
def exercise_restrained_refinement(options):
import random
random.seed(1)
flex.set_random_seed(1)
xs0 = smtbx.development.random_xray_structure(
sgtbx.space_group_info('P1'),
n_scatterers=options.n_scatterers,
elements="random")
for sc in xs0.scatterers():
sc.flags.set_grad_site(True)
sc0 = xs0.scatterers()
uc = xs0.unit_cell()
mi = xs0.build_miller_set(anomalous_flag=False, d_min=options.resolution)
fo_sq = mi.structure_factors_from_scatterers(
xs0, algorithm="direct").f_calc().norm()
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.size(), 1))
i, j, k, l = random.sample(xrange(options.n_scatterers), 4)
bond_proxies = geometry_restraints.shared_bond_simple_proxy()
w = 1e9
d_ij = uc.distance(sc0[i].site, sc0[j].site)*0.8
bond_proxies.append(geom.bond_simple_proxy(
i_seqs=(i, j),
distance_ideal=d_ij,
weight=w))
d_jk = uc.distance(sc0[j].site, sc0[k].site)*0.85
bond_proxies.append(geom.bond_simple_proxy(
i_seqs=(j, k),
distance_ideal=d_jk,
weight=w))
d_ki = min(uc.distance(sc0[k].site, sc0[i].site)*0.9, (d_ij + d_jk)*0.8)
bond_proxies.append(geom.bond_simple_proxy(
i_seqs=(k, i),
distance_ideal=d_ki,
weight=w))
d_jl = uc.distance(sc0[j].site, sc0[l].site)*0.9
bond_proxies.append(geom.bond_simple_proxy(
i_seqs=(j, l),
distance_ideal=d_jl,
weight=w))
d_lk = min(uc.distance(sc0[l].site, sc0[k].site)*0.8, 0.75*(d_jk + d_jl))
bond_proxies.append(geom.bond_simple_proxy(
i_seqs=(l, k),
distance_ideal=d_jl,
weight=w))
restraints_manager = restraints.manager(bond_proxies=bond_proxies)
xs1 = xs0.deep_copy_scatterers()
xs1.shake_sites_in_place(rms_difference=0.1)
def ls_problem():
xs = xs1.deep_copy_scatterers()
reparametrisation = constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=smtbx.utils.connectivity_table(xs),
temperature=20)
return least_squares.crystallographic_ls(
fo_sq.as_xray_observations(),
reparametrisation=reparametrisation,
restraints_manager=restraints_manager)
gradient_threshold, step_threshold = 1e-6, 1e-6
eps = 5e-3
ls = ls_problem()
t = wall_clock_time()
cycles = normal_eqns_solving.naive_iterations(
ls,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold,
track_all=True)
if options.verbose:
print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed())
sc = ls.xray_structure.scatterers()
for p in bond_proxies:
d = uc.distance(*[ sc[i_pair].site for i_pair in p.i_seqs ])
assert approx_equal(d, p.distance_ideal, eps)
ls = ls_problem()
t = wall_clock_time()
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls,
gradient_threshold=gradient_threshold,
step_threshold=step_threshold,
tau=1e-3,
track_all=True)
if options.verbose:
print "%i %s steps in %.6f s" % (cycles.n_iterations, cycles, t.elapsed())
sc = ls.xray_structure.scatterers()
sc = ls.xray_structure.scatterers()
for p in bond_proxies:
d = uc.distance(*[ sc[i].site for i in p.i_seqs ])
assert approx_equal(d, p.distance_ideal, eps)
def exercise(self):
xs0 = self.structure
xs = xs0.deep_copy_scatterers()
k1, s1, li1, o1, o2 = xs.scatterers()
self.shake_point_group_3(k1)
self.shake_point_group_3(s1)
self.shake_point_group_3(li1)
self.shake_point_group_3(o1)
o2.site = tuple(
[ x*(1 + random.uniform(-self.delta_site, self.delta_site))
for x in o2.site])
o2.u_star = tuple(
[ u*(1 + random.uniform(-self.delta_u_star, self.delta_u_star))
for u in o2.u_star])
for sc in xs.scatterers():
sc.flags.set_use_u_iso(False).set_use_u_aniso(True)
sc.flags.set_grad_site(True).set_grad_u_aniso(True)
connectivity_table = smtbx.utils.connectivity_table(xs)
reparametrisation = constraints.reparametrisation(
structure=xs,
constraints=[],
connectivity_table=connectivity_table)
ls = least_squares.crystallographic_ls(
self.fo_sq.as_xray_observations(), reparametrisation,
weighting_scheme=least_squares.unit_weighting(),
origin_fixing_restraints_type=
origin_fixing_restraints.atomic_number_weighting)
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls,
gradient_threshold=1e-12,
step_threshold=1e-7,
track_all=True)
## Test whether refinement brought back the shaked structure to its
## original state
match = emma.model_matches(xs0.as_emma_model(),
xs.as_emma_model()).refined_matches[0]
assert match.rt.r == matrix.identity(3)
assert not match.singles1 and not match.singles2
assert match.rms < 1e-6
delta_u_carts= ( xs.scatterers().extract_u_cart(xs.unit_cell())
- xs0.scatterers().extract_u_cart(xs.unit_cell())).norms()
assert flex.abs(delta_u_carts) < 1e-6
assert approx_equal(ls.scale_factor(), 1, eps=1e-4)
## Test covariance matrix
jac_tr = reparametrisation.jacobian_transpose_matching_grad_fc()
cov = ls.covariance_matrix(
jacobian_transpose=jac_tr, normalised_by_goof=False)\
.matrix_packed_u_as_symmetric()
m, n = cov.accessor().focus()
# x,y for point group 3 sites are fixed: no variance or correlation
for i in (0, 9, 18, 27,):
assert cov.matrix_copy_block(i, 0, 2, n) == 0
# u_star coefficients u13 and u23 for point group 3 sites are fixed
# to 0: again no variance or correlation with any other param
for i in (7, 16, 25, 34,):
assert cov.matrix_copy_block(i, 0, 2, n).as_1d()\
.all_approx_equal(0., 1e-20)
# u_star coefficients u11, u22 and u12 for point group 3 sites
# are totally correlated, with variances in ratios 1:1:1/2
for i in (3, 12, 21, 30,):
assert cov[i, i] != 0
assert approx_equal(cov[i, i], cov[i+1, i+1], eps=1e-15)
assert approx_equal(cov[i, i+1]/cov[i, i], 1, eps=1e-12)
assert approx_equal(cov[i, i+3]/cov[i, i], 0.5, eps=1e-12)
