def __init__(self, xray_structure,
miller_set,
manager=None,
cos_sin_table=False,
algorithm="direct"):
"""Evaluate structure factors via direct calculations
:type xray_structure: cctbx.xray.structure
:param xray_structure: the X-ray structure to evaluate the structure factors of
:type miller_set: cctbx.miller.set
:param miller_set: the set of miller indices to evaluate the structure factors at
:type manager: cctbx.xray.structure_factors.manager
:param manager: ? (TODO: doc)
:type algorithm: string
:param algorithm: the name of the evaluation method, either \
"direct", "use_alt_parallel"
:type cos_sin_table: boolean
:param cos_sin_table: If set to 'True' a precalculated cos_sin_table is used \
instead of actual trigonometric functions. Using a manager overrides this \
setting with the one specified in the manager.
:rtype: cctbx.xray.structure_factors.from_scatterers_direct
:retruns: an instance `e` of `cctbx.xray.structure_factors.from_scatterers_direct` \
providing the evaluated structure factors as `e.f_calc()`
"""
time_all = user_plus_sys_time()
managed_calculation_base.__init__(self,
manager, xray_structure, miller_set, algorithm="direct")
if hasattr(algorithm,"use_alt_parallel"):
#enumeration of indices() is fast compared to direct summation
from cctbx.xray.structure_factors.from_scatterers_direct_parallel import pprocess
pprocess(self,algorithm)
return
timer = user_plus_sys_time()
if (manager is not None):
cos_sin_table = manager.cos_sin_table()
if (cos_sin_table == True):
cos_sin_table = default_cos_sin_table
elif (cos_sin_table == False):
cos_sin_table = None
if (cos_sin_table is None):
self._results = ext.structure_factors_direct(
self._miller_set.unit_cell(),
self._miller_set.space_group(),
self._miller_set.indices(),
self._xray_structure.scatterers(),
self._xray_structure.scattering_type_registry())
else:
self._results = ext.structure_factors_direct(
cos_sin_table,
self._miller_set.unit_cell(),
self._miller_set.space_group(),
self._miller_set.indices(),
self._xray_structure.scatterers(),
self._xray_structure.scattering_type_registry())
if (manager is not None):
manager.estimate_time_direct.register(
xray_structure.scatterers().size() * miller_set.indices().size(),
timer.elapsed())
global_counters.from_scatterers_direct.process(time_all.elapsed())
def exercise_user_plus_sys_time():
s = StringIO()
utils.user_plus_sys_time().show_elapsed(out=s, prefix="e: ")
s = s.getvalue()
assert s.startswith("e: ")
assert s.endswith(" s")
utils.user_plus_sys_time().show_delta(out=s, prefix="d: ")
s = s.getvalue()
assert s.startswith("d: ")
assert s.endswith(" s")
def __init__(self, manager,
xray_structure,
miller_set,
algorithm="fft"):
time_all = user_plus_sys_time()
managed_calculation_base.__init__(self,
manager, xray_structure, miller_set, algorithm="fft")
assert miller_set.d_min() > manager.d_min() * (1-1e-6)
manager.setup_fft() # before timing
time_sampling = user_plus_sys_time()
sampled_density = ext.sampled_model_density(
unit_cell=xray_structure.unit_cell(),
scatterers=xray_structure.scatterers(),
scattering_type_registry=xray_structure.scattering_type_registry(),
fft_n_real=manager.rfft().n_real(),
fft_m_real=manager.rfft().m_real(),
u_base=manager.u_base(),
wing_cutoff=manager.wing_cutoff(),
exp_table_one_over_step_size=manager.exp_table_one_over_step_size(),
force_complex=manager.force_complex(),
sampled_density_must_be_positive=
manager.sampled_density_must_be_positive(),
tolerance_positive_definite=manager.tolerance_positive_definite())
time_sampling = time_sampling.elapsed()
time_fft = user_plus_sys_time()
if (not sampled_density.anomalous_flag()):
sf_map = manager.rfft().forward(sampled_density.real_map())
collect_conj = True
else:
sf_map = manager.cfft().backward(sampled_density.complex_map())
collect_conj = False
time_fft = time_fft.elapsed()
time_from_map = user_plus_sys_time()
self._f_calc_data = maptbx.structure_factors.from_map(
space_group=miller_set.space_group(),
anomalous_flag=sampled_density.anomalous_flag(),
miller_indices=miller_set.indices(),
complex_map=sf_map,
conjugate_flag=collect_conj).data()
time_from_map = time_from_map.elapsed()
time_apply_u_extra = user_plus_sys_time()
sampled_density.eliminate_u_extra_and_normalize(
miller_set.indices(),
self._f_calc_data)
time_apply_u_extra = time_apply_u_extra.elapsed()
introspection.virtual_memory_info().update_max()
manager.estimate_time_fft.register(
n_scatterers=xray_structure.scatterers().size(),
n_miller_indices=miller_set.indices().size(),
time_sampling=time_sampling,
time_fft=time_fft,
time_from_or_to_map=time_from_map,
time_apply_u_extra=time_apply_u_extra)
global_counters.from_scatterers_fft.process(time_all.elapsed())
def __init__(self, manager,
xray_structure,
u_iso_refinable_params,
miller_set,
d_target_d_f_calc,
n_parameters):
time_all = time_apply_u_extra = user_plus_sys_time()
gradients_base.__init__(self,
manager, xray_structure, miller_set, algorithm="fft")
self._d_target_d_f_calc = d_target_d_f_calc
manager.setup_fft() # before timing
time_apply_u_extra = user_plus_sys_time()
self._results = ext.fast_gradients(
unit_cell=xray_structure.unit_cell(),
scatterers=xray_structure.scatterers(),
scattering_type_registry=xray_structure.scattering_type_registry(),
u_base=manager.u_base(),
wing_cutoff=manager.wing_cutoff(),
exp_table_one_over_step_size=manager.exp_table_one_over_step_size(),
tolerance_positive_definite=manager.tolerance_positive_definite())
coeff = self._gradient_map_coeff()
time_apply_u_extra = time_apply_u_extra.elapsed()
time_from_or_to_map = user_plus_sys_time()
coeff_map = self._gradient_map_coeff_to_map(coeff)
time_from_or_to_map = time_from_or_to_map.elapsed()
time_fft = user_plus_sys_time()
if (not coeff.anomalous_flag()):
gradient_map = manager.rfft().backward(coeff_map.complex_map())
else:
gradient_map = manager.cfft().backward(coeff_map.complex_map())
time_fft = time_fft.elapsed()
time_sampling = user_plus_sys_time()
self._results.sampling(
scatterers=xray_structure.scatterers(),
u_iso_refinable_params=u_iso_refinable_params,
scattering_type_registry=xray_structure.scattering_type_registry(),
site_symmetry_table=xray_structure.site_symmetry_table(),
ft_d_target_d_f_calc=gradient_map,
n_parameters=n_parameters,
sampled_density_must_be_positive=
manager.sampled_density_must_be_positive())
time_sampling = time_sampling.elapsed()
introspection.virtual_memory_info().update_max()
manager.estimate_time_fft.register(
n_scatterers=xray_structure.scatterers().size(),
n_miller_indices=miller_set.indices().size(),
time_sampling=time_sampling,
time_fft=time_fft,
time_from_or_to_map=time_from_or_to_map,
time_apply_u_extra=time_apply_u_extra)
self.d_target_d_site_frac_was_used = False
self.d_target_d_u_star_was_used = False
global_counters.gradients_fft.process(time_all.elapsed())
def __init__(self, xray_structure,
u_iso_refinable_params,
miller_set,
d_target_d_f_calc,
n_parameters,
manager=None,
cos_sin_table=False):
time_all = user_plus_sys_time()
gradients_base.__init__(self,
manager, xray_structure, miller_set, algorithm="direct")
self._d_target_d_f_calc = d_target_d_f_calc
timer = user_plus_sys_time()
if (manager is not None):
cos_sin_table = manager.cos_sin_table()
if (cos_sin_table == True):
cos_sin_table = default_cos_sin_table
elif (cos_sin_table == False):
cos_sin_table = None
if (cos_sin_table is None):
self._results = ext.structure_factors_gradients_direct(
self._miller_set.unit_cell(),
self._miller_set.space_group(),
self._miller_set.indices(),
self._xray_structure.scatterers(),
u_iso_refinable_params,
self._xray_structure.scattering_type_registry(),
self._xray_structure.site_symmetry_table(),
d_target_d_f_calc,
n_parameters)
else:
self._results = ext.structure_factors_gradients_direct(
cos_sin_table,
self._miller_set.unit_cell(),
self._miller_set.space_group(),
self._miller_set.indices(),
self._xray_structure.scatterers(),
u_iso_refinable_params,
self._xray_structure.scattering_type_registry(),
self._xray_structure.site_symmetry_table(),
d_target_d_f_calc,
n_parameters)
if (manager is not None):
manager.estimate_time_direct.register(
xray_structure.scatterers().size() * miller_set.indices().size(),
timer.elapsed())
self.d_target_d_site_cart_was_used = False
self.d_target_d_u_cart_was_used = False
global_counters.gradients_direct.process(time_all.elapsed())
def exercise_01(grid_step = 0.03, d_min = 1.0, wing_cutoff = 1.e-9):
xrs = random_structure.xray_structure(
space_group_info = sgtbx.space_group_info("P 1"),
elements = ["O","N","C","P","S","U","AU"]*1,
random_u_iso = True,
general_positions_only = False)
# avoid excessive_range_error_limit crash
bs = xrs.extract_u_iso_or_u_equiv()*adptbx.u_as_b(1)
sel = bs < 1
bs = bs.set_selected(sel, 1)
xrs.set_b_iso(values = bs)
#
p = xrs.unit_cell().parameters()
timer = user_plus_sys_time()
res = manager(nx = int(p[0]/grid_step),
ny = int(p[1]/grid_step),
nz = int(p[2]/grid_step),
scattering_type_registry = xrs.scattering_type_registry(),
unit_cell = xrs.unit_cell(),
scatterers = xrs.scatterers(),
wing_cutoff = wing_cutoff)
print "time: %10.4f" % (timer.elapsed())
f_calc_dir = xrs.structure_factors(
d_min = d_min,
algorithm = "direct").f_calc()
#
f_calc_den = f_calc_dir.structure_factors_from_map(map = res.density_array,
use_scale = True)
f1 = flex.abs(f_calc_dir.data())
f2 = flex.abs(f_calc_den.data())
r = flex.sum(flex.abs(f1-f2))/flex.sum(f2)
print "r-factor:", r
assert r < 1.e-4, r
def quick_test(file_name): from libtbx.utils import user_plus_sys_time t = user_plus_sys_time() s = reader(file_name) print "Time read:", t.delta() s.show_summary() print tuple(s.original_indices[:3]) print tuple(s.unique_indices[:3]) print tuple(s.batch_numbers[:3]) print tuple(s.centric_tags[:3]) print tuple(s.spindle_flags[:3]) print tuple(s.asymmetric_unit_indices[:3]) print tuple(s.i_obs[:3]) print tuple(s.sigmas[:3]) print tuple(s.original_indices[-3:]) print tuple(s.unique_indices[-3:]) print tuple(s.batch_numbers[-3:]) print tuple(s.centric_tags[-3:]) print tuple(s.spindle_flags[-3:]) print tuple(s.asymmetric_unit_indices[-3:]) print tuple(s.i_obs[-3:]) print tuple(s.sigmas[-3:]) m = s.as_miller_array(merge_equivalents=False).merge_equivalents() print "min redundancies:", flex.min(m.redundancies().data()) print "max redundancies:", flex.max(m.redundancies().data()) print "mean redundancies:", flex.mean(m.redundancies().data().as_double()) s.as_miller_arrays()[0].show_summary() print
def get_times(space_group_info, volume=100000, d_min=1.5):
crystal_symmetry = dummy_structure(space_group_info, volume, 0)
crystal_symmetry.show_summary()
structure_factors_from_scatterers = xray.structure_factors.from_scatterers(
crystal_symmetry=crystal_symmetry,
d_min=d_min)
for log_n_reflections in xrange(15,16):
miller_set = dummy_miller_set(crystal_symmetry, log_n_reflections)
for log_n_scatterers in xrange(1,10):
structure = dummy_structure(space_group_info,volume,2**log_n_scatterers)
timer = user_plus_sys_time()
f_calc = structure_factors_from_scatterers(
xray_structure=structure,
miller_set=miller_set)
print "%6d %6d %10.3f direct=%10.3f fft=%10.3f" % (
2**log_n_scatterers, 2**log_n_reflections, timer.delta(),
structure_factors_from_scatterers.estimate_time_direct(
2**log_n_scatterers * 2**log_n_reflections),
structure_factors_from_scatterers.estimate_time_fft(
2**log_n_scatterers, 2**log_n_reflections))
e = structure_factors_from_scatterers.estimate_time_fft
print "time_sampling:", e.time_sampling
print "time_fft:", e.time_fft
print "time_from_or_to_map:", e.time_from_or_to_map
print "time_apply_u_extra:", e.time_apply_u_extra
def run_smtbx_ls(mode, cod_id, i_obs, f_obs, xray_structure, params):
import smtbx.refinement
fo_sq = i_obs
assert fo_sq.sigmas() is not None
sel = (fo_sq.data() == 0) & (fo_sq.sigmas() == 0)
fo_sq = fo_sq.select(~sel)
fo_sq.select(fo_sq.sigmas() <= 0).show_array()
assert fo_sq.sigmas().all_gt(0)
if (1): # work around bug currently in smtbx weighting scheme implementation
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.data().size(), 1))
xobs = fo_sq.as_xray_observations()
tm = user_plus_sys_time()
rm = smtbx.refinement.model(
fo_sq=xobs,
xray_structure=xray_structure,
constraints=[],
restraints_manager=smtbx.refinement.restraints.manager(),
weighting_scheme=smtbx.refinement.least_squares.unit_weighting())
ls = rm.least_squares()
if (mode == "simple"):
for i_cycle in xrange(params.ls_simple_iterations):
ls.build_up()
try:
ls.solve_and_step_forward()
except RuntimeError, e:
if (str(e).find("cholesky.failure") <= 0): raise
print 'Aborting run_smtbx_ls("simple"): cholesky.failure: %s' \
% cod_id
break
for sc in xray_structure.scatterers():
if (sc.u_iso <= 0 or sc.u_iso > 1):
sc.u_iso = 0.05
show_cc_r1(params, "ls%02d" % (i_cycle+1), f_obs, xray_structure)
tm.show_elapsed(prefix="time smtbx_ls_simple_iterations: ")
def _call(self, func, msg):
timer = user_plus_sys_time()
self.ma.add(msg)
func()
t = timer.elapsed()
self.total_time += t
self.ma.add(" time (s): %s (total time: %s)" %
(("%8.3f" % t).strip(),
("%8.3f" % self.total_time).strip()))
def gradients_wrt_atomic_parameters(self,
selection=None,
site=False,
u_iso=False,
u_aniso=False,
occupancy=False,
tan_b_iso_max=None,
u_iso_refinable_params=None):
if (tan_b_iso_max is not None and tan_b_iso_max != 0):
raise RuntimeError(
"Not implemented:\n"
" See CVS revision 1.87, 2007/03/03 01:53:05\n"
" method: manager.gradient_wrt_atomic_parameters()")
global time_gradients_wrt_atomic_parameters
timer = user_plus_sys_time()
manager = self.manager
xray_structure = manager.xray_structure
if (selection is not None):
xray_structure = xray_structure.select(selection)
d_target_d_f_calc = self.d_target_d_f_calc_work()
result = None
if (u_aniso):
result = manager.structure_factor_gradients_w(
u_iso_refinable_params=None,
d_target_d_f_calc=d_target_d_f_calc.data(),
xray_structure=xray_structure,
n_parameters=0,
miller_set=d_target_d_f_calc,
algorithm=manager.sfg_params.algorithm).d_target_d_u_cart()
elif (u_iso):
result = manager.structure_factor_gradients_w(
u_iso_refinable_params=None,
d_target_d_f_calc=d_target_d_f_calc.data(),
xray_structure=xray_structure,
n_parameters=0,
miller_set=d_target_d_f_calc,
algorithm=manager.sfg_params.algorithm).d_target_d_u_iso()
elif (occupancy):
result = manager.structure_factor_gradients_w(
u_iso_refinable_params=None,
d_target_d_f_calc=d_target_d_f_calc.data(),
xray_structure=xray_structure,
n_parameters=0,
miller_set=d_target_d_f_calc,
algorithm=manager.sfg_params.algorithm).d_target_d_occupancy()
else:
result = manager.structure_factor_gradients_w(
u_iso_refinable_params=u_iso_refinable_params,
d_target_d_f_calc=d_target_d_f_calc.data(),
xray_structure=xray_structure,
n_parameters=xray_structure.n_parameters(),
miller_set=d_target_d_f_calc,
algorithm=manager.sfg_params.algorithm)
time_gradients_wrt_atomic_parameters += timer.elapsed()
return result
def gradients_wrt_atomic_parameters(self,
selection=None,
site=False,
u_iso=False,
u_aniso=False,
occupancy=False,
tan_b_iso_max=None,
u_iso_refinable_params=None):
if (tan_b_iso_max is not None and tan_b_iso_max != 0):
raise RuntimeError("Not implemented:\n"
" See CVS revision 1.87, 2007/03/03 01:53:05\n"
" method: manager.gradient_wrt_atomic_parameters()")
global time_gradients_wrt_atomic_parameters
timer = user_plus_sys_time()
manager = self.manager
xray_structure = manager.xray_structure
if (selection is not None):
xray_structure = xray_structure.select(selection)
d_target_d_f_calc = self.d_target_d_f_calc_work()
result = None
if (u_aniso):
result = manager.structure_factor_gradients_w(
u_iso_refinable_params=None,
d_target_d_f_calc=d_target_d_f_calc.data(),
xray_structure=xray_structure,
n_parameters=0,
miller_set=d_target_d_f_calc,
algorithm=manager.sfg_params.algorithm).d_target_d_u_cart()
elif(u_iso):
result = manager.structure_factor_gradients_w(
u_iso_refinable_params=None,
d_target_d_f_calc=d_target_d_f_calc.data(),
xray_structure=xray_structure,
n_parameters=0,
miller_set=d_target_d_f_calc,
algorithm=manager.sfg_params.algorithm).d_target_d_u_iso()
elif(occupancy):
result = manager.structure_factor_gradients_w(
u_iso_refinable_params=None,
d_target_d_f_calc=d_target_d_f_calc.data(),
xray_structure=xray_structure,
n_parameters=0,
miller_set=d_target_d_f_calc,
algorithm=manager.sfg_params.algorithm).d_target_d_occupancy()
else:
result = manager.structure_factor_gradients_w(
u_iso_refinable_params=u_iso_refinable_params,
d_target_d_f_calc=d_target_d_f_calc.data(),
xray_structure=xray_structure,
n_parameters=xray_structure.n_parameters(),
miller_set=d_target_d_f_calc,
algorithm=manager.sfg_params.algorithm)
time_gradients_wrt_atomic_parameters += timer.elapsed()
return result
def caller(self, func):
timer = user_plus_sys_time()
doc = inspect.getdoc(func)
#if(doc is not None): broadcast(m = doc, log = self.log)
result = func()
t = timer.elapsed()
self.total_time += t
fmt = " %s: %s" % (doc, str("%8.3f" % t).strip())
self.time_strings.append(fmt)
#self.log.flush()
return result
def run_smtbx_ls(mode, cod_id, i_obs, f_obs, xray_structure, params):
import smtbx.refinement
fo_sq = i_obs
assert fo_sq.sigmas() is not None
sel = (fo_sq.data() == 0) & (fo_sq.sigmas() == 0)
fo_sq = fo_sq.select(~sel)
fo_sq.select(fo_sq.sigmas() <= 0).show_array()
assert fo_sq.sigmas().all_gt(0)
if (1): # work around bug currently in smtbx weighting scheme implementation
fo_sq = fo_sq.customized_copy(sigmas=flex.double(fo_sq.data().size(), 1))
xobs = fo_sq.as_xray_observations()
tm = user_plus_sys_time()
rm = smtbx.refinement.model(
fo_sq=xobs,
xray_structure=xray_structure,
constraints=[],
restraints_manager=smtbx.refinement.restraints.manager(),
weighting_scheme=smtbx.refinement.least_squares.unit_weighting())
ls = rm.least_squares()
if (mode == "simple"):
for i_cycle in range(params.ls_simple_iterations):
ls.build_up()
try:
ls.solve_and_step_forward()
except RuntimeError as e:
if (str(e).find("cholesky.failure") <= 0): raise
print('Aborting run_smtbx_ls("simple"): cholesky.failure: %s' \
% cod_id)
break
for sc in xray_structure.scatterers():
if (sc.u_iso <= 0 or sc.u_iso > 1):
sc.u_iso = 0.05
show_cc_r1(params, "ls%02d" % (i_cycle+1), f_obs, xray_structure)
tm.show_elapsed(prefix="time smtbx_ls_simple_iterations: ")
elif (mode == "lm"):
from scitbx.lstbx import normal_eqns_solving
thresh = 1e-6
try:
cycles = normal_eqns_solving.levenberg_marquardt_iterations(
ls,
gradient_threshold=thresh,
step_threshold=thresh,
tau=1e-7)
except RuntimeError as e:
if (not str(e).startswith(
"cctbx::adptbx::debye_waller_factor_exp: arg_limit exceeded")):
raise
print('Aborting run_smtbx_ls("lm"):' \
' debye_waller_factor_exp failure: %s' % cod_id)
show_cc_r1(params, "smtbx_lm", f_obs, xray_structure)
tm.show_elapsed(prefix="time levenberg_marquardt_iterations: ")
else:
raise RuntimeError('Unknown run_smtbx_ls(mode="%s")' % mode)
def exclude_pairs(self):
# exclude all pairs with dist >= 4 * tolerance
# dist_allowed is invariant under refine_adjusted_shift:
# exclude all pairs with dist_allowed >= tolerance
# if 0 continuous shifts: dist_allowed == dist:
# exclude all pairs with dist >= tolerance
timer = user_plus_sys_time()
self.add_pair_ext.next_pivot(
self.match_symmetry.continuous_shift_flags, self.eucl_symop,
self.adjusted_shift, flex.int(self.singles1),
flex.int(self.singles2))
self.times.exclude_pairs += timer.delta()
def refine_adjusted_shift(self):
timer = user_plus_sys_time()
unit_cell = self.ref_model1.unit_cell()
sum_diff_cart = matrix.col([0., 0., 0.])
for diff in self.calculate_shortest_diffs():
diff_allowed = self.match_symmetry.filter_shift(diff, selector=1)
diff_cart = unit_cell.orthogonalize(diff_allowed)
sum_diff_cart += matrix.col(diff_cart)
mean_diff_cart = sum_diff_cart / len(self.pairs)
mean_diff_frac = matrix.col(unit_cell.fractionalize(mean_diff_cart))
self.adjusted_shift = matrix.col(self.adjusted_shift) + mean_diff_frac
self.times.refine_adjusted_shift += timer.delta()
def refine_adjusted_shift(self):
timer = user_plus_sys_time()
unit_cell = self.ref_model1.unit_cell()
sum_diff_cart = matrix.col([0.,0.,0.])
for diff in self.calculate_shortest_diffs():
diff_allowed = self.match_symmetry.filter_shift(diff, selector=1)
diff_cart = unit_cell.orthogonalize(diff_allowed)
sum_diff_cart += matrix.col(diff_cart)
mean_diff_cart = sum_diff_cart / len(self.pairs)
mean_diff_frac = matrix.col(unit_cell.fractionalize(mean_diff_cart))
self.adjusted_shift = matrix.col(self.adjusted_shift) + mean_diff_frac
self.times.refine_adjusted_shift += timer.delta()
def update_xray_structure_with_tls(xray_structure,
selections,
tlsos,
selections_1d = None,
correct_adp = True):
global time_update_xray_structure_with_tls
timer = user_plus_sys_time()
u_cart_from_tls_ = u_cart_from_tls(sites_cart = xray_structure.sites_cart(),
selections = selections,
tlsos = tlsos)
xray_structure.set_u_cart(u_cart=u_cart_from_tls_, selection = selections_1d)
if(correct_adp): xray_structure.tidy_us(u_min = 1.e-6)
time_update_xray_structure_with_tls += timer.elapsed()
def update_xray_structure_with_tls(xray_structure,
selections,
tlsos,
selections_1d=None,
correct_adp=True):
global time_update_xray_structure_with_tls
timer = user_plus_sys_time()
u_cart_from_tls_ = u_cart_from_tls(sites_cart=xray_structure.sites_cart(),
selections=selections,
tlsos=tlsos)
xray_structure.set_u_cart(u_cart=u_cart_from_tls_, selection=selections_1d)
if (correct_adp): xray_structure.tidy_us(u_min=1.e-6)
time_update_xray_structure_with_tls += timer.elapsed()
def exclude_pairs(self):
# exclude all pairs with dist >= 4 * tolerance
# dist_allowed is invariant under refine_adjusted_shift:
# exclude all pairs with dist_allowed >= tolerance
# if 0 continuous shifts: dist_allowed == dist:
# exclude all pairs with dist >= tolerance
timer = user_plus_sys_time()
self.add_pair_ext.next_pivot(
self.match_symmetry.continuous_shift_flags,
self.eucl_symop,
self.adjusted_shift,
flex.int(self.singles1),
flex.int(self.singles2))
self.times.exclude_pairs += timer.delta()
def add_pairs(self):
# XXX possible optimizations:
# if 0 continuous shifts:
# tabulate dist
# keep all < tolerance
# we do not need eliminate_weak_matches
timer = user_plus_sys_time()
while (len(self.singles1) and len(self.singles2)):
if (not self.add_pair_ext.next_pair(self.adjusted_shift,
flex.int(self.singles1),
flex.int(self.singles2))):
break
new_pair = (self.add_pair_ext.new_pair_1(),
self.add_pair_ext.new_pair_2())
self.pairs.append(new_pair)
self.singles1.remove(new_pair[0])
self.singles2.remove(new_pair[1])
self.refine_adjusted_shift()
self.times.add_pairs += timer.delta()
def update_xray_structure_with_nm(xray_structure,
adp_nmas,
selections,
xs,
n_modes,
selections_1d=None,
correct_adp=True,
zero_mode_flag=True):
global time_update_xray_structure_with_nm
timer = user_plus_sys_time()
total_size = xray_structure.sites_cart().size()
u_cart_from_nm_ = u_cart_from_nm(adp_nmas=adp_nmas,
selections=selections,
xs=xs,
n_modes=n_modes,
total_size=total_size,
zero_mode_flag=zero_mode_flag)
xray_structure.set_u_cart(u_cart=u_cart_from_nm_, selection=selections_1d)
if (correct_adp): xray_structure.tidy_us(u_min=1.e-6)
time_update_xray_structure_with_nm += timer.elapsed()
def add_pairs(self):
# XXX possible optimizations:
# if 0 continuous shifts:
# tabulate dist
# keep all < tolerance
# we do not need eliminate_weak_matches
timer = user_plus_sys_time()
while (len(self.singles1) and len(self.singles2)):
if (not self.add_pair_ext.next_pair(
self.adjusted_shift,
flex.int(self.singles1),
flex.int(self.singles2))):
break
new_pair = (self.add_pair_ext.new_pair_1(),
self.add_pair_ext.new_pair_2())
self.pairs.append(new_pair)
self.singles1.remove(new_pair[0])
self.singles2.remove(new_pair[1])
self.refine_adjusted_shift()
self.times.add_pairs += timer.delta()
def update_xray_structure_with_nm(xray_structure,
adp_nmas,
selections,
xs,
n_modes,
selections_1d = None,
correct_adp = True,
zero_mode_flag = True):
global time_update_xray_structure_with_nm
timer = user_plus_sys_time()
total_size = xray_structure.sites_cart().size()
u_cart_from_nm_ = u_cart_from_nm(adp_nmas = adp_nmas,
selections = selections,
xs = xs,
n_modes = n_modes,
total_size = total_size,
zero_mode_flag = zero_mode_flag)
xray_structure.set_u_cart(u_cart=u_cart_from_nm_, selection = selections_1d)
if(correct_adp): xray_structure.tidy_us(u_min = 1.e-6)
time_update_xray_structure_with_nm += timer.elapsed()
def eliminate_weak_pairs(self):
timer = user_plus_sys_time()
while 1:
weak_pair = 0
max_dist = 0
for pair in self.pairs[1:]:
dist = self.calculate_shortest_dist(pair)
if (dist > max_dist):
weak_pair = pair
max_dist = dist
if (weak_pair == 0): break
if (max_dist < self.tolerance):
dist = self.calculate_shortest_dist(self.pairs[0])
if (dist < self.tolerance):
break
assert len(self.pairs) > 1
self.pairs.remove(weak_pair)
self.singles1.append(weak_pair[0])
self.singles2.append(weak_pair[1])
self.refine_adjusted_shift()
self.times.eliminate_weak_pairs += timer.delta()
def eliminate_weak_pairs(self):
timer = user_plus_sys_time()
while 1:
weak_pair = 0
max_dist = 0
for pair in self.pairs[1:]:
dist = self.calculate_shortest_dist(pair)
if (dist > max_dist):
weak_pair = pair
max_dist = dist
if (weak_pair == 0): break
if (max_dist < self.tolerance):
dist = self.calculate_shortest_dist(self.pairs[0])
if (dist < self.tolerance):
break
assert len(self.pairs) > 1
self.pairs.remove(weak_pair)
self.singles1.append(weak_pair[0])
self.singles2.append(weak_pair[1])
self.refine_adjusted_shift()
self.times.eliminate_weak_pairs += timer.delta()
def run_smtbx_ls(mode, cod_id, i_obs, f_obs, xray_structure, params):
import smtbx.refinement
fo_sq = i_obs
assert fo_sq.sigmas() is not None
sel = (fo_sq.data() == 0) & (fo_sq.sigmas() == 0)
fo_sq = fo_sq.select(~sel)
fo_sq.select(fo_sq.sigmas() <= 0).show_array()
assert fo_sq.sigmas().all_gt(0)
if (
1
): # work around bug currently in smtbx weighting scheme implementation
fo_sq = fo_sq.customized_copy(
sigmas=flex.double(fo_sq.data().size(), 1))
xobs = fo_sq.as_xray_observations()
tm = user_plus_sys_time()
rm = smtbx.refinement.model(
fo_sq=xobs,
xray_structure=xray_structure,
constraints=[],
restraints_manager=smtbx.refinement.restraints.manager(),
weighting_scheme=smtbx.refinement.least_squares.unit_weighting())
ls = rm.least_squares()
if (mode == "simple"):
for i_cycle in xrange(params.ls_simple_iterations):
ls.build_up()
try:
ls.solve_and_step_forward()
except RuntimeError, e:
if (str(e).find("cholesky.failure") <= 0): raise
print 'Aborting run_smtbx_ls("simple"): cholesky.failure: %s' \
% cod_id
break
for sc in xray_structure.scatterers():
if (sc.u_iso <= 0 or sc.u_iso > 1):
sc.u_iso = 0.05
show_cc_r1(params, "ls%02d" % (i_cycle + 1), f_obs, xray_structure)
tm.show_elapsed(prefix="time smtbx_ls_simple_iterations: ")
def split_u(xray_structure, tls_selections, offset):
global time_split_u
timer = user_plus_sys_time()
uc = xray_structure.unit_cell()
u_iso = xray_structure.scatterers().extract_u_iso()
u_eq_1 = xray_structure.extract_u_iso_or_u_equiv()
for tls_selection in tls_selections:
u_iso_sel = u_iso.select(tls_selection)
u_iso_min = flex.min(u_iso_sel)
if (offset):
offset_ = adptbx.b_as_u(5.0)
else:
offset_ = 0.0
if u_iso_min >= offset_:
u_iso_min = u_iso_min - offset_
t = adptbx.u_iso_as_u_star(uc, u_iso_min)
for i_seq in tls_selection:
sc = xray_structure.scatterers()[i_seq]
assert sc.u_iso == u_iso[i_seq]
u_iso_new = sc.u_iso - u_iso_min
assert u_iso_new >= 0.0
sc.u_iso = u_iso_new
assert sc.flags.use_u_aniso()
assert sc.flags.use_u_iso()
if (sc.u_star == (-1.0, -1.0, -1.0, -1.0, -1.0, -1.0)):
sc.u_star = t
else:
x = flex.double(sc.u_star)
y = flex.double(t)
z = list(x + y)
sc.u_star = z
u_iso = xray_structure.scatterers().extract_u_iso().select(
xray_structure.use_u_iso())
assert (u_iso < 0.0).count(True) == 0
u_eq_2 = xray_structure.extract_u_iso_or_u_equiv()
assert approx_equal(u_eq_1, u_eq_2)
time_split_u += timer.elapsed()
def split_u(xray_structure, tls_selections, offset):
global time_split_u
timer = user_plus_sys_time()
uc = xray_structure.unit_cell()
u_iso = xray_structure.scatterers().extract_u_iso()
u_eq_1 = xray_structure.extract_u_iso_or_u_equiv()
for tls_selection in tls_selections:
u_iso_sel = u_iso.select(tls_selection)
u_iso_min = flex.min(u_iso_sel)
if(offset):
offset_ = adptbx.b_as_u(5.0)
else: offset_ = 0.0
if u_iso_min >= offset_:
u_iso_min = u_iso_min - offset_
t = adptbx.u_iso_as_u_star(uc, u_iso_min)
for i_seq in tls_selection:
sc = xray_structure.scatterers()[i_seq]
assert sc.u_iso == u_iso[i_seq]
u_iso_new = sc.u_iso - u_iso_min
assert u_iso_new >= 0.0
sc.u_iso = u_iso_new
assert sc.flags.use_u_aniso()
assert sc.flags.use_u_iso()
if(sc.u_star == (-1.0,-1.0,-1.0,-1.0,-1.0,-1.0)):
sc.u_star = t
else:
x = flex.double(sc.u_star)
y = flex.double(t)
z = list(x + y)
sc.u_star = z
u_iso = xray_structure.scatterers().extract_u_iso().select(
xray_structure.use_u_iso())
assert (u_iso < 0.0).count(True) == 0
u_eq_2 = xray_structure.extract_u_iso_or_u_equiv()
assert approx_equal(u_eq_1, u_eq_2)
time_split_u += timer.elapsed()
def split_u(u_eq, xray_structure, tls_selections, offset):
global time_split_u
timer = user_plus_sys_time()
uc = xray_structure.unit_cell()
for tls_selection in tls_selections:
if (offset):
offset_ = adptbx.b_as_u(5.0)
else:
offset_ = 0.0
for i_seq in tls_selection:
sc = xray_structure.scatterers()[i_seq]
u_nm = sc.u_star
u_nm_iso = adptbx.u_star_as_u_iso(uc, u_nm)
u_iso_new = u_eq[i_seq] - u_nm_iso
if u_iso_new < 0.0:
u_iso_new = 0.0
assert u_iso_new >= 0.0
sc.u_iso = u_iso_new
assert sc.flags.use_u_aniso()
assert sc.flags.use_u_iso()
u_iso = xray_structure.scatterers().extract_u_iso().select(
xray_structure.use_u_iso())
assert (u_iso < 0.0).count(True) == 0
time_split_u += timer.elapsed()
def run_and_time(*args, **kw):
timer = user_plus_sys_time()
try:
run(*args, **kw)
finally:
print("CPU time: %.2f seconds" % timer.elapsed())
def __init__(self, fmodels,
restraints_manager = None,
model = None,
is_neutron_scat_table = None,
target_weights = None,
tan_b_iso_max = None,
refine_xyz = False,
refine_adp = False,
lbfgs_termination_params = None,
use_fortran = False,
verbose = 0,
correct_special_position_tolerance = 1.0,
iso_restraints = None,
h_params = None,
qblib_params = None,
macro_cycle = None,
u_min = adptbx.b_as_u(-5.0),
u_max = adptbx.b_as_u(999.99),
collect_monitor = True,
log = None):
timer = user_plus_sys_time()
adopt_init_args(self, locals())
self.f=None
self.xray_structure = self.fmodels.fmodel_xray().xray_structure
self.fmodels.create_target_functors()
self.fmodels.prepare_target_functors_for_minimization()
if(self.refine_adp and fmodels.fmodel_neutron() is None):
self.xray_structure.tidy_us()
self.fmodels.update_xray_structure(
xray_structure = self.xray_structure,
update_f_calc = True)
self.weights = None
# QBLIB INSERT
self.qblib_params = qblib_params
if(self.qblib_params is not None and self.qblib_params.qblib):
self.macro = macro_cycle
self.qblib_cycle_count = 0
self.tmp_XYZ = None
self.XYZ_diff_curr=None
# QBLIB END
self.correct_special_position_tolerance = correct_special_position_tolerance
if(refine_xyz and target_weights is not None):
self.weights = target_weights.xyz_weights_result
elif(refine_adp and target_weights is not None):
self.weights = target_weights.adp_weights_result
else:
from phenix.refinement import weight_xray_chem
self.weights = weight_xray_chem.weights(wx = 1,
wx_scale = 1,
angle_x = None,
wn = 1,
wn_scale = 1,
angle_n = None,
w = 0,
wxn = 1)
if(self.collect_monitor):
self.monitor = monitor(
weights = self.weights,
fmodels = fmodels,
model = model,
iso_restraints = iso_restraints,
refine_xyz = refine_xyz,
refine_adp = refine_adp,
refine_occ = False)
if(self.collect_monitor): self.monitor.collect()
self.neutron_refinement = (self.fmodels.fmodel_n is not None)
self.x = flex.double(self.xray_structure.n_parameters(), 0)
self._scatterers_start = self.xray_structure.scatterers()
self.minimizer = scitbx.lbfgs.run(
target_evaluator = self,
termination_params = lbfgs_termination_params,
use_fortran = use_fortran,
exception_handling_params = scitbx.lbfgs.exception_handling_parameters(
ignore_line_search_failed_step_at_lower_bound = True))
self.apply_shifts()
del self._scatterers_start
self.compute_target(compute_gradients = False,u_iso_refinable_params = None)
if(self.refine_adp and self.fmodels.fmodel_neutron() is None):
self.xray_structure.tidy_us()
self.fmodels.update_xray_structure(
xray_structure = self.xray_structure,
update_f_calc = True)
if(self.collect_monitor):
self.monitor.collect(iter = self.minimizer.iter(),
nfun = self.minimizer.nfun())
self.fmodels.create_target_functors()
# QBLIB INSERT
if(self.qblib_params is not None and self.qblib_params.qblib):
print >>self.qblib_params.qblib_log,'{:-^80}'.format("")
print >>self.qblib_params.qblib_log
def timings(structure,
d_min,
fft_only=False,
wing_cutoff_reference=1.e-6,
wing_cutoff_others=1.e-3):
structure_ng = structure.deep_copy_scatterers()
structure_5g = structure.deep_copy_scatterers()
structure_4g = structure.deep_copy_scatterers()
structure_2g = structure.deep_copy_scatterers()
structure_1g = structure.deep_copy_scatterers()
structure_ng.scattering_type_registry(d_min=d_min, table="n_gaussian")
structure_5g.scattering_type_registry(table="wk1995")
structure_4g.scattering_type_registry(table="it1992")
custom_dict = {}
custom_dict.update(eltbx.xray_scattering.two_gaussian_agarwal_isaacs.table)
structure_2g.scattering_type_registry(custom_dict=custom_dict)
custom_dict.update(eltbx.xray_scattering.one_gaussian_agarwal_1978.table)
structure_1g.scattering_type_registry(custom_dict=custom_dict)
miller_set = miller.build_set(crystal_symmetry=structure,
d_min=d_min,
anomalous_flag=False)
miller_set.show_summary()
print("d_min:", d_min)
if (fft_only):
timer = user_plus_sys_time()
f_calc_reference = xray.structure_factors.from_scatterers(
miller_set=miller_set,
wing_cutoff=wing_cutoff_reference,
exp_table_one_over_step_size=0)(xray_structure=structure,
miller_set=miller_set,
algorithm="fft").f_calc().data()
print("fft exp function wing_cutoff=%3.1e: %.2f seconds" %
(wing_cutoff_reference, timer.elapsed()))
else:
timer = user_plus_sys_time()
f_calc_reference = xray.structure_factors_simple(
structure_5g.unit_cell(), structure_5g.space_group(),
miller_set.indices(), structure_5g.scatterers(),
structure_5g.scattering_type_registry()).f_calc()
print("direct simple: %.2f seconds" % timer.elapsed())
f_calc_reference = flex.abs(f_calc_reference)
print("wing_cutoff for following fft calculations: %3.1e" %
wing_cutoff_others)
for structure in (structure_ng, structure_5g, structure_4g, structure_2g,
structure_1g):
structure.scattering_type_registry().show_summary()
if (not fft_only):
for calc_type, cos_sin_flag in (("direct cos+sin function:",
False), ("direct cos+sin table:",
True)):
timer = user_plus_sys_time()
f_calc = miller_set.structure_factors_from_scatterers(
xray_structure=structure,
algorithm="direct",
cos_sin_table=cos_sin_flag).f_calc()
print(" %-24s %.2f seconds," % (calc_type, timer.elapsed()),
end=' ')
ls = xray.targets_least_squares_residual(
f_calc_reference, f_calc.data(), False, 1)
print("r-factor: %.6f" % ls.target())
for calc_type, exp_table_one_over_step_size in (("fft exp function:",
0), ("fft exp table:",
-100)):
timer = user_plus_sys_time()
f_calc = xray.structure_factors.from_scatterers(
miller_set=miller_set,
wing_cutoff=wing_cutoff_others,
exp_table_one_over_step_size=exp_table_one_over_step_size)(
xray_structure=structure,
miller_set=miller_set,
algorithm="fft").f_calc()
print(" %-24s %.2f seconds," % (calc_type, timer.elapsed()),
end=' ')
ls = xray.targets_least_squares_residual(f_calc_reference,
f_calc.data(), False, 1)
print("r-factor: %.6f" % ls.target())
print()
def exercise_hybrid_36():
hybrid_36.exercise(hy36enc=pdb.hy36encode, hy36dec=pdb.hy36decode)
for width, s in [(3, "AAA"), (6, "zzzzzz")]:
try:
pdb.hy36encode(width=width, value=0)
except RuntimeError, e:
assert str(e) == "unsupported width."
else:
raise Exception_expected
try:
pdb.hy36decode(width=width, s=s)
except RuntimeError, e:
assert str(e) == "unsupported width."
else:
raise Exception_expected
ups = user_plus_sys_time()
n_ok = pdb.hy36recode_width_4_all()
ups = ups.elapsed()
print "time hy36recode_width_4_all: %.2f s" \
" (%.3f micro s per encode-decode cycle)" % (ups, 1.e6*ups/max(1,n_ok))
assert n_ok == 999 + 10000 + 2 * 26 * 36**3
#
assert pdb.resseq_decode(s=1234) == 1234
assert pdb.resseq_decode(s="A123") == 11371
assert pdb.resseq_decode(s="1") == 1
pdb.resseq_encode(value=1) == " 1"
pdb.resseq_encode(value=11371) == "A123"
pdb.resseq_encode(value=1234) == "1234"
#
try:
pdb.resseq_decode(s="18A")
def caller(self, func, prefix):
timer = user_plus_sys_time()
func(prefix = prefix)
t = timer.elapsed()
self.total_time += t
self.time_strings.append(" %s: %s"%(prefix, str("%8.3f"%t).strip()))
def __init__(self,
fmodel,
model,
selections,
selections_1d,
n_modes,
number_of_macro_cycles,
max_number_of_iterations,
weight_nmre=1.0,
evecin="evec.dat",
evalin="eval.dat",
fsub_evecin="proj.dat",
nm_params_filename="nm_params.dat",
zero_mode_flag=True,
zero_mode_input_flag=False,
update_evec=False,
enmcalc_path="enmcalc_fix.csh",
start_xs_value=None,
zero_mode_corr=True,
fsub_ref=False,
fsub_n_modes=20,
run_finite_differences_test=False,
eps=1.e-6,
out=None,
macro_cycle=None,
verbose=True):
adopt_init_args(self, locals())
global time_nm_total
timer = user_plus_sys_time()
if (out is None): out = sys.stdout
prefix = "NM refinement:"
fmodel.info().show_targets(text=prefix + " start model", out=out)
fmodel.xray_structure.show_u_statistics(text=prefix + " start model",
out=out)
xrs = fmodel.xray_structure
pdb_hierarchy = model.pdb_hierarchy()
#pdb_hierarchy misssing a parameter, I am not sure whether it is all right.
# xrs.tidy_us(u_min = 1.e-6)
if (update_evec):
f_pdb = open("tmp.pdb", "w")
m = model.deep_copy()
m.write_pdb_file(out=f_pdb)
f_pdb.close()
try:
retcode = subprocess.call(enmcalc_path)
if retcode < 0:
print >> out, "Enmcalc was terminated by signal", -retcode
else:
print >> out, "Enmcalc completed", retcode
evalin = "eval.dat"
evecin = "evec.dat"
fsub_evecin = "proj.dat"
print >> out, "evalin was changed to", evalin
print >> out, "evecin was changed to", evecin
print >> out, "fsub_evecin was change to", fsub_evecin
except OSError as e:
print >> out, "Executing enmcalc failed:", e
if (fsub_ref):
self.total_evec = evecin
evecin = fsub_evecin
modes = generate_evec(selections=selections,
selections_1d=selections_1d,
xray_structure=xrs,
pdb_hierarchy=pdb_hierarchy,
filename=evecin,
n_modes=n_modes,
zero_mode_input_flag=zero_mode_input_flag,
zero_mode_flag=zero_mode_flag)
#selections will be modified during this step
# for nm_sel in selections:
# fmodel.xray_structure.convert_to_anisotropic(selection = nm_sel)
if (zero_mode_corr):
curr_zero_mode_flag = zero_mode_flag
zero_mode_flag = False
adp_nmas = []
for selection in selections:
modes1d = selected_modes_to_1D(modes=modes,
n_modes=n_modes,
selection=selection)
weights_selected = xrs.atomic_weights().select(selection)
print "The number of selected atoms is %d." % len(weights_selected)
adp_nma = init_nm_adp(modes=modes1d,
weights=weights_selected,
n_modes=n_modes,
zero_mode_flag=zero_mode_flag)
adp_nmas.append(adp_nma)
u_cart = xrs.scatterers().extract_u_cart(xrs.unit_cell())
if (fsub_ref):
fsub_ref_manager = self.fsub_total_refinement()
fsub_adp = xrs.scatterers().extract_u_cart(xrs.unit_cell())
else:
fsub_adp = None
if (start_xs_value is not None):
xs = nm_params_reader(filename=start_xs_value)
elif (macro_cycle == 1):
xs_initial = []
nmval = read_nmval_file(evalin=evalin,
n_modes=n_modes,
zero_mode_input_flag=zero_mode_input_flag,
zero_mode_flag=zero_mode_flag)
for adp_nma, selection in zip(adp_nmas, selections):
x = init_nm_para(nmval=nmval,
n_modes=n_modes,
zero_mode_flag=zero_mode_flag)
u_cart_selected = u_cart.select(selection)
uaniso_from_s_manager = uaniso_from_s(
x=x,
adp_nma=adp_nma,
n_modes=n_modes,
zero_mode_flag=zero_mode_flag)
u = uaniso_from_s_manager.u_cart()
x_scaled = scale_x(x=x,
uanisos=u_cart_selected,
adp_all=u,
n_modes=n_modes,
zero_mode_flag=zero_mode_flag)
xs_initial.append(x_scaled)
xs = nm_from_uanisos(xray_structure=xrs,
selections=selections,
modes=None,
xs_initial=xs_initial,
n_modes=n_modes,
adp_nmas=adp_nmas,
number_of_macro_cycles=2,
max_iterations=self.max_number_of_iterations,
zero_mode_flag=zero_mode_flag,
verbose=-1)
else:
if (os.path.isfile("fsub_params.dat") is True
and fsub_ref is False):
xs_initial = nm_params_reader(filename="fsub_params.dat")
else:
xs_initial = model.nm_groups.xs
xs = nm_from_uanisos(xray_structure=xrs,
selections=selections,
modes=None,
xs_initial=xs_initial,
n_modes=n_modes,
adp_nmas=adp_nmas,
number_of_macro_cycles=3,
max_iterations=self.max_number_of_iterations,
zero_mode_flag=zero_mode_flag,
verbose=-1)
if (verbose):
show_nm(xs=xs,
n_modes=n_modes,
text=prefix + " start parameters",
zero_mode_flag=zero_mode_flag,
out=out)
for macro_cycle in range(1, number_of_macro_cycles + 1):
print >> out
prefix = "NM refinement: after macrocycle " + str(macro_cycle)
minimized = nm_xray_target_minimizer(
fmodel=fmodel,
model=model,
xs_initial=xs,
adp_nmas=adp_nmas,
fsub_adp=fsub_adp,
selections=selections,
selections_1d=selections_1d,
max_iterations=max_number_of_iterations,
n_modes=n_modes,
weight_nmre=weight_nmre,
run_finite_differences_test=run_finite_differences_test,
zero_mode_flag=zero_mode_flag)
xrs = minimized.fmodel_copy.xray_structure
xrs.show_u_statistics(text=prefix, out=out)
if (verbose):
show_nm(xs=minimized.xs_result,
n_modes=n_modes,
text=prefix,
zero_mode_flag=zero_mode_flag,
out=out)
fmodel.update_xray_structure(xray_structure=xrs,
update_f_calc=True)
fmodel.info().show_targets(text=prefix, out=out)
if (xrs.is_positive_definite_u().count(False) > 0):
xrs.tidy_us(u_min=1.e-6)
xrs.show_u_statistics(text=prefix +
": after making positive definite",
out=out)
fmodel.update_xray_structure(xray_structure=xrs,
update_f_calc=True)
fmodel.info().show_targets(text=prefix +
": after making positive definite",
out=out)
xs = make_nm_compatible_with_u_positive_definite(
xray_structure=xray_structure,
adp_nmas=adp_nmas,
selections=selections,
max_iterations=100,
number_of_u_nonpositive_definite=0,
eps=eps,
number_of_macro_cycles_for_nm_from_uanisos=2,
n_modes=n_modes,
xs=minimize.xs_result,
selections_1d=selections_1d,
zero_mode_flag=zero_mode_flag,
out=out)
else:
xs = minimized.xs_result
if (verbose):
show_nm(xs=xs,
n_modes=n_modes,
text="NM refinement: final values",
zero_mode_flag=zero_mode_flag,
out=out)
self.xs = xs
filename = nm_params_filename
nm_params_writer(xs=minimized.xs_result,
n_modes=n_modes,
zero_mode_flag=zero_mode_flag,
filename=filename)
if (os.path.isfile("fsub_params.dat") is False or fsub_ref is True):
model.nm_groups.xs = self.xs
self.fmodel = fmodel
time_nm_total += timer.elapsed()
def tls_from_u_cart(xray_structure,
tlsos_initial,
tls_selections,
number_of_macro_cycles = 100,
max_iterations = 100):
global time_tls_from_u_cart
timer = user_plus_sys_time()
uc = xray_structure.unit_cell()
xray_structure.tidy_us(u_min = 1.e-6)
ueq = xray_structure.extract_u_iso_or_u_equiv()
assert (ueq < 0.0).count(True) == 0
u_cart = xray_structure.scatterers().extract_u_cart(uc)
for tls_selection in tls_selections:
u_cart_selected = u_cart.select(tls_selection)
assert adptbx.is_positive_definite(u_cart_selected,1.e-6).count(False)==0
xray_structure.tidy_us(u_min = 1.e-6)
t = []
l = []
s = []
lim_l = 10.0
for tls_selection, tlso in zip(tls_selections, tlsos_initial):
t.append( tlso.t )
#if(abs(tlso.t[0]) < eps and abs(tlso.t[1]) < eps and abs(tlso.t[2]) < eps and
# abs(tlso.t[3]) < eps and abs(tlso.t[4]) < eps and abs(tlso.t[5]) < eps):
# t.append( t_from_u_cart(u_cart.select(tls_selection), 1.e-6) )
#else:
# t.append( tlso.t )
#l.append( [0,0,0,0,0,0] )
if abs(tlso.l[0])>lim_l: l1 = tlso.l[0]/5.
else: l1 = tlso.l[0]
if abs(tlso.l[1])>lim_l: l2 = tlso.l[1]/5.
else: l2 = tlso.l[1]
if abs(tlso.l[2])>lim_l: l3 = tlso.l[2]/5.
else: l3 = tlso.l[2]
if abs(tlso.l[3])>lim_l: l4 = tlso.l[3]/5.
else: l4 = tlso.l[3]
if abs(tlso.l[4])>lim_l: l5 = tlso.l[4]/5.
else: l5 = tlso.l[4]
if abs(tlso.l[5])>lim_l: l6 = tlso.l[5]/5.
else: l6 = tlso.l[5]
l.append( [l1,l2,l3,l4,l5,l6] )
s.append( tlso.s )
tlsos = generate_tlsos(selections = tls_selections,
xray_structure = xray_structure,
T = t, L = l, S = s)
#for rt,rl,rs in [[0,1,1],[1,0,0]]*3:
for rt,rl,rs in [[0,1,1],[1,0,0],[1,1,1]]:
tlsos_ = tls_from_uanisos(xray_structure = xray_structure,
selections = tls_selections,
tlsos_initial = tlsos,
number_of_macro_cycles = number_of_macro_cycles,
max_iterations = max_iterations,
refine_T = rt,
refine_L = rl,
refine_S = rs,
verbose = -1,
out = None)
tlsos = tlsos_
t = []
l = []
s = []
for tlso in tlsos:
t.append( tlso.t )
if abs(tlso.l[0])>lim_l: l1 = lim_l/5.
else: l1 = tlso.l[0]
if abs(tlso.l[1])>lim_l: l2 = lim_l/5.
else: l2 = tlso.l[1]
if abs(tlso.l[2])>lim_l: l3 = lim_l/5.
else: l3 = tlso.l[2]
if abs(tlso.l[3])>lim_l: l4 = lim_l/5.
else: l4 = tlso.l[3]
if abs(tlso.l[4])>lim_l: l5 = lim_l/5.
else: l5 = tlso.l[4]
if abs(tlso.l[5])>lim_l: l6 = lim_l/5.
else: l6 = tlso.l[5]
l.append( [l1,l2,l3,l4,l5,l6] )
s.append( tlso.s )
tlsos = generate_tlsos(selections = tls_selections,
xray_structure = xray_structure,
T = t, L = l, S = s)
time_tls_from_u_cart += timer.elapsed()
return tlsos
def tls_from_u_cart(xray_structure,
tlsos_initial,
tls_selections,
number_of_macro_cycles=100,
max_iterations=100):
global time_tls_from_u_cart
timer = user_plus_sys_time()
uc = xray_structure.unit_cell()
xray_structure.tidy_us(u_min=1.e-6)
ueq = xray_structure.extract_u_iso_or_u_equiv()
assert (ueq < 0.0).count(True) == 0
u_cart = xray_structure.scatterers().extract_u_cart(uc)
for tls_selection in tls_selections:
u_cart_selected = u_cart.select(tls_selection)
assert adptbx.is_positive_definite(u_cart_selected,
1.e-6).count(False) == 0
xray_structure.tidy_us(u_min=1.e-6)
t = []
l = []
s = []
lim_l = 10.0
for tls_selection, tlso in zip(tls_selections, tlsos_initial):
t.append(tlso.t)
#if(abs(tlso.t[0]) < eps and abs(tlso.t[1]) < eps and abs(tlso.t[2]) < eps and
# abs(tlso.t[3]) < eps and abs(tlso.t[4]) < eps and abs(tlso.t[5]) < eps):
# t.append( t_from_u_cart(u_cart.select(tls_selection), 1.e-6) )
#else:
# t.append( tlso.t )
#l.append( [0,0,0,0,0,0] )
if abs(tlso.l[0]) > lim_l: l1 = tlso.l[0] / 5.
else: l1 = tlso.l[0]
if abs(tlso.l[1]) > lim_l: l2 = tlso.l[1] / 5.
else: l2 = tlso.l[1]
if abs(tlso.l[2]) > lim_l: l3 = tlso.l[2] / 5.
else: l3 = tlso.l[2]
if abs(tlso.l[3]) > lim_l: l4 = tlso.l[3] / 5.
else: l4 = tlso.l[3]
if abs(tlso.l[4]) > lim_l: l5 = tlso.l[4] / 5.
else: l5 = tlso.l[4]
if abs(tlso.l[5]) > lim_l: l6 = tlso.l[5] / 5.
else: l6 = tlso.l[5]
l.append([l1, l2, l3, l4, l5, l6])
s.append(tlso.s)
tlsos = generate_tlsos(selections=tls_selections,
xray_structure=xray_structure,
T=t,
L=l,
S=s)
#for rt,rl,rs in [[0,1,1],[1,0,0]]*3:
for rt, rl, rs in [[0, 1, 1], [1, 0, 0], [1, 1, 1]]:
tlsos_ = tls_from_uanisos(
xray_structure=xray_structure,
selections=tls_selections,
tlsos_initial=tlsos,
number_of_macro_cycles=number_of_macro_cycles,
max_iterations=max_iterations,
refine_T=rt,
refine_L=rl,
refine_S=rs,
verbose=-1,
out=None)
tlsos = tlsos_
t = []
l = []
s = []
for tlso in tlsos:
t.append(tlso.t)
if abs(tlso.l[0]) > lim_l: l1 = lim_l / 5.
else: l1 = tlso.l[0]
if abs(tlso.l[1]) > lim_l: l2 = lim_l / 5.
else: l2 = tlso.l[1]
if abs(tlso.l[2]) > lim_l: l3 = lim_l / 5.
else: l3 = tlso.l[2]
if abs(tlso.l[3]) > lim_l: l4 = lim_l / 5.
else: l4 = tlso.l[3]
if abs(tlso.l[4]) > lim_l: l5 = lim_l / 5.
else: l5 = tlso.l[4]
if abs(tlso.l[5]) > lim_l: l6 = lim_l / 5.
else: l6 = tlso.l[5]
l.append([l1, l2, l3, l4, l5, l6])
s.append(tlso.s)
tlsos = generate_tlsos(selections=tls_selections,
xray_structure=xray_structure,
T=t,
L=l,
S=s)
time_tls_from_u_cart += timer.elapsed()
return tlsos
def __init__(self,
fmodel,
model,
selections,
selections_1d,
n_modes,
number_of_macro_cycles,
max_number_of_iterations,
weight_nmre = 1.0,
evecin = "eigenvectors.dat",
evalin = "eigenvalues.dat",
nm_params_filename = "nm_params.dat",
zero_mode_flag = True,
zero_mode_input_flag = False,
start_xs_value = None,
run_finite_differences_test = False,
eps = 1.e-6,
out = None,
macro_cycle = None,
verbose = True):
global time_nm_total
timer = user_plus_sys_time()
if(out is None): out = sys.stdout
prefix = "NM refinement:"
fmodel.info().show_targets(text = prefix + " start model", out = out)
fmodel.xray_structure.show_u_statistics(text = prefix+" start model",
out = out)
xrs = fmodel.xray_structure
pdb_hierarchy = model.pdb_hierarchy()
#pdb_hierarchy misssing a parameter, I am not sure whether it is all right.
xrs.tidy_us(u_min = 1.e-6)
modes = generate_evec(selections = selections,
selections_1d = selections_1d,
xray_structure = xrs,
pdb_hierarchy = pdb_hierarchy,
filename = evecin,
n_modes = n_modes,
zero_mode_input_flag = zero_mode_input_flag,
zero_mode_flag = zero_mode_flag)
#selections will be modified during this step
adp_nmas = []
for selection in selections:
modes1d = selected_modes_to_1D(modes = modes, n_modes = n_modes, selection = selection)
weights_selected = xrs.atomic_weights().select(selection)
print "The number of selected atoms is %d." % len(weights_selected)
adp_nma = init_nm_adp(modes = modes1d,
weights = weights_selected,
n_modes = n_modes,
zero_mode_flag = zero_mode_flag)
adp_nmas.append(adp_nma)
u_cart = xrs.scatterers().extract_u_cart(xrs.unit_cell())
if(start_xs_value is not None):
xs = nm_params_reader(filename = start_xs_value)
elif(macro_cycle == 1):
xs_initial = []
nmval = read_nmval_file(evalin = evalin,
n_modes = n_modes,
zero_mode_input_flag = zero_mode_input_flag,
zero_mode_flag = zero_mode_flag)
for adp_nma, selection in zip(adp_nmas, selections):
x = init_nm_para(nmval = nmval,
n_modes = n_modes,
zero_mode_flag = zero_mode_flag)
u_cart_selected = u_cart.select(selection)
uaniso_from_s_manager = uaniso_from_s(x = x,
adp_nma = adp_nma,
n_modes = n_modes,
zero_mode_flag = zero_mode_flag)
u = uaniso_from_s_manager.u_cart()
x_scaled = scale_x(x = x,
uanisos = u_cart_selected,
adp_all = u,
n_modes = n_modes,
zero_mode_flag = zero_mode_flag)
xs_initial.append(x_scaled)
xs = nm_from_uanisos(xray_structure = xrs,
selections = selections,
modes = modes,
xs_initial = xs_initial,
n_modes = n_modes,
number_of_macro_cycles = 1,
zero_mode_flag = zero_mode_flag,
verbose = verbose)
else:
xs = model.nm_groups.xs
if (verbose) :
show_nm(xs = xs,
n_modes = n_modes,
text = prefix + " start parameters",
zero_mode_flag = zero_mode_flag,
out = out)
for macro_cycle in range(1, number_of_macro_cycles+1):
print >> out
prefix = "NM refinement: after macrocycle "+str(macro_cycle)
minimized = nm_xray_target_minimizer(
fmodel = fmodel,
model = model,
xs_initial = xs,
adp_nmas = adp_nmas,
selections = selections,
selections_1d = selections_1d,
max_iterations = max_number_of_iterations,
n_modes = n_modes,
weight_nmre = weight_nmre,
run_finite_differences_test = run_finite_differences_test,
zero_mode_flag = zero_mode_flag)
xrs = minimized.fmodel_copy.xray_structure
xrs.show_u_statistics(text = prefix, out = out)
if(verbose):
show_nm(xs = minimized.xs_result,
n_modes = n_modes,
text = prefix,
zero_mode_flag = zero_mode_flag,
out = out)
fmodel.update_xray_structure(xray_structure = xrs,
update_f_calc = True)
fmodel.info().show_targets(text = prefix, out = out)
if(xrs.is_positive_definite_u().count(False) > 0):
xrs.tidy_us(u_min = 1.e-6)
xrs.show_u_statistics(
text = prefix+": after making positive definite",
out = out)
fmodel.update_xray_structure(xray_structure = xrs,
update_f_calc = True)
fmodel.info().show_targets(text = prefix+": after making positive definite",
out = out)
xs = make_nm_compatible_with_u_positive_definite(
xray_structure = xray_structure,
adp_nmas = adp_nmas,
selections = selections,
max_iterations = 100,
number_of_u_nonpositive_definite = 0,
eps = eps,
number_of_macro_cycles_for_nm_from_uanisos = 10,
n_modes = n_modes,
xs = minimize.xs_result,
selections_1d = selections_1d,
zero_mode_flag = zero_mode_flag,
out = out)
else: xs = minimized.xs_result
if(verbose):
show_nm(xs = xs,
n_modes = n_modes,
text = "NM refinement: final values",
zero_mode_flag = zero_mode_flag,
out = out)
self.xs = xs
filename = nm_params_filename
nm_params_writer(xs = minimized.xs_result,
n_modes = n_modes,
zero_mode_flag = zero_mode_flag,
filename = filename)
model.nm_groups.xs = xs
self.fmodel = fmodel
time_nm_total += timer.elapsed()
def caller(self, func, prefix):
timer = user_plus_sys_time()
func(prefix = prefix)
t = timer.elapsed()
self.total_time += t
self.time_strings.append(" %s: %s"%(prefix, str("%8.3f"%t).strip()))
def __init__(self,
fmodel,
model,
selections,
selections_1d,
refine_T,
refine_L,
refine_S,
number_of_macro_cycles,
max_number_of_iterations,
start_tls_value=None,
run_finite_differences_test=False,
eps=1.e-6,
out=None,
macro_cycle=None,
verbose=True):
global time_tls_total
timer = user_plus_sys_time()
if (out is None): out = sys.stdout
prefix = "TLS refinement:"
fmodel.info().show_targets(text=prefix + " start model", out=out)
fmodel.xray_structure.show_u_statistics(text=prefix + " start model",
out=out)
xrs = fmodel.xray_structure
xrs.tidy_us(u_min=1.e-6)
if (start_tls_value is not None):
try:
crash_or_not = abs(start_tls_value + 0)
tlsos = generate_tlsos(value=start_tls_value,
selections=selections,
xray_structure=xrs)
except Exception:
tlsos = start_tls_value
else:
tlsos = tls_from_u_cart(xray_structure=xrs,
tlsos_initial=model.tls_groups.tlsos,
tls_selections=selections,
number_of_macro_cycles=100,
max_iterations=100)
if (verbose):
show_tls(tlsos=tlsos, text=prefix + " start parameters", out=out)
for macro_cycle in range(1, number_of_macro_cycles + 1):
print >> out
prefix = "TLS refinement: after macrocycle " + str(macro_cycle)
minimized = tls_xray_target_minimizer(
fmodel=fmodel,
tlsos_initial=tlsos,
refine_T=refine_T,
refine_L=refine_L,
refine_S=refine_S,
selections=selections,
selections_1d=selections_1d,
max_iterations=max_number_of_iterations,
run_finite_differences_test=run_finite_differences_test)
xrs = minimized.fmodel_copy.xray_structure
xrs.show_u_statistics(text=prefix, out=out)
if (verbose):
show_tls(tlsos=minimized.tlsos_result, text=prefix, out=out)
fmodel.update_xray_structure(xray_structure=xrs,
update_f_calc=True)
fmodel.info().show_targets(text=prefix, out=out)
if (xrs.is_positive_definite_u().count(False) > 0):
xrs.tidy_us(u_min=1.e-6)
xrs.show_u_statistics(text=prefix +
": after making positive definite",
out=out)
fmodel.update_xray_structure(xray_structure=xrs,
update_f_calc=True)
fmodel.info().show_targets(text=prefix +
": after making positive definite",
out=out)
tlsos = make_tlso_compatible_with_u_positive_definite(
tlsos=minimized.tlsos_result,
xray_structure=xrs.deep_copy_scatterers(),
selections=selections,
max_iterations=10,
number_of_u_nonpositive_definite=0,
eps=eps,
refine_T=refine_T,
refine_L=refine_L,
refine_S=refine_S,
out=out,
number_of_macro_cycles_for_tls_from_uanisos=10)
else:
tlsos = minimized.tlsos_result
if (verbose):
show_tls(tlsos=tlsos, text="TLS refinement: final values", out=out)
self.tlsos = tlsos
model.tls_groups.tlsos = tlsos
self.fmodel = fmodel
time_tls_total += timer.elapsed()
def __init__(self,
fmodel,
selections=None,
params=None,
r_initial=None,
t_initial=None,
bss=None,
log=None,
monitors=None):
global time_rigid_body_total
self.params = params
save_original_target_name = fmodel.target_name
save_bss_anisotropic_scaling = None
if (bss is not None):
save_bss_anisotropic_scaling = bss.anisotropic_scaling
timer_rigid_body_total = user_plus_sys_time()
save_r_work = fmodel.r_work()
save_r_free = fmodel.r_free()
save_xray_structure = fmodel.xray_structure.deep_copy_scatterers()
if (log is None): log = sys.stdout
if (selections is None):
selections = []
selections.append(
flex.bool(fmodel.xray_structure.scatterers().size(),
True).iselection())
else:
assert len(selections) > 0
fmodel.xray_structure.scatterers().flags_set_grads(state=False)
fmodel.xray_structure.scatterers().flags_set_grad_site(
iselection=flex.bool(fmodel.xray_structure.scatterers().size(),
True).iselection())
self.total_rotation = []
self.total_translation = []
for item in selections:
self.total_rotation.append(flex.double(3, 0))
self.total_translation.append(flex.double(3, 0))
if (r_initial is None):
r_initial = []
for item in selections:
r_initial.append(flex.double(3, 0))
if (t_initial is None):
t_initial = []
for item in selections:
t_initial.append(flex.double(3, 0))
fmodel_copy = fmodel.deep_copy()
if (fmodel_copy.mask_params is not None):
fmodel_copy.mask_params.verbose = -1
d_mins, target_names = split_resolution_range(
d_spacings=fmodel_copy.f_obs_work().d_spacings().data(),
n_bodies=len(selections),
target=params.target,
target_auto_switch_resolution=params.target_auto_switch_resolution,
n_ref_first=params.min_number_of_reflections,
multi_body_factor_n_ref_first=params.multi_body_factor,
d_low=params.max_low_high_res_limit,
d_high=params.high_resolution,
number_of_zones=params.number_of_zones,
zone_exponent=params.zone_exponent,
log=log)
print(file=log)
if (fmodel.target_name != target_names[0]):
fmodel.update(target_name=target_names[0])
self.show(fmodel=fmodel,
r_mat=self.total_rotation,
t_vec=self.total_translation,
header="Start",
out=log)
if (params.number_of_zones == 1 or monitors is None):
monitors_call_back_handler = None
else:
monitors_call_back_handler = monitors.call_back_handler
if (monitors_call_back_handler is not None):
monitors_call_back_handler(monitor=None,
model=None,
fmodel=fmodel,
method="rigid_body")
for res, target_name in zip(d_mins, target_names):
xrs = fmodel_copy.xray_structure.deep_copy_scatterers()
fmodel_copy = fmodel.resolution_filter(d_min=res)
if (fmodel_copy.target_name != target_name):
fmodel_copy.update(target_name=target_name)
d_max_min = fmodel_copy.f_obs_work().d_max_min()
line = "Refinement at resolution: "+\
str("%7.2f"%d_max_min[0]).strip() + " - " \
+ str("%6.2f"%d_max_min[1]).strip() \
+ " target=" + fmodel_copy.target_name
print_statistics.make_sub_header(line, out=log)
fmodel_copy.update_xray_structure(xray_structure=xrs,
update_f_calc=True)
rworks = flex.double()
if (len(d_mins) == 1):
n_rigid_body_minimizer_cycles = 1
else:
n_rigid_body_minimizer_cycles = min(int(res), 4)
for i_macro_cycle in range(n_rigid_body_minimizer_cycles):
if (bss is not None and params.bulk_solvent_and_scale):
if (fmodel_copy.f_obs().d_min() > 3.0):
bss.anisotropic_scaling = False
fast = True
if (bss.mode == "slow"): fast = False
fmodel_copy.update_all_scales(update_f_part1=False,
params=bss,
fast=fast,
log=log,
remove_outliers=False,
optimize_mask=False,
refine_hd_scattering=False)
if (fmodel_copy.f_obs().d_min() > 3.0):
assert save_bss_anisotropic_scaling is not None
bss.anisotropic_scaling = save_bss_anisotropic_scaling
bss.minimization_b_cart = save_bss_anisotropic_scaling
minimized = rigid_body_minimizer(
fmodel=fmodel_copy,
selections=selections,
r_initial=r_initial,
t_initial=t_initial,
refine_r=params.refine_rotation,
refine_t=params.refine_translation,
max_iterations=params.max_iterations,
euler_angle_convention=params.euler_angle_convention,
lbfgs_maxfev=params.
lbfgs_line_search_max_function_evaluations)
rotation_matrices = []
translation_vectors = []
for i in range(len(selections)):
self.total_rotation[i] += flex.double(minimized.r_min[i])
self.total_translation[i] += flex.double(
minimized.t_min[i])
rot_obj = scitbx.rigid_body.euler(
phi=minimized.r_min[i][0],
psi=minimized.r_min[i][1],
the=minimized.r_min[i][2],
convention=params.euler_angle_convention)
rotation_matrices.append(rot_obj.rot_mat())
translation_vectors.append(minimized.t_min[i])
new_xrs = apply_transformation(
xray_structure=minimized.fmodel.xray_structure,
rotation_matrices=rotation_matrices,
translation_vectors=translation_vectors,
selections=selections)
fmodel_copy.update_xray_structure(xray_structure=new_xrs,
update_f_calc=True,
update_f_mask=True)
rwork = minimized.fmodel.r_work()
rfree = minimized.fmodel.r_free()
assert approx_equal(rwork, fmodel_copy.r_work())
fmodel.update_xray_structure(
xray_structure=fmodel_copy.xray_structure,
update_f_calc=True,
update_f_mask=True)
if (bss is not None and params.bulk_solvent_and_scale):
fast = True
if (bss.mode == "slow"): fast = False
fmodel_copy.update_all_scales(update_f_part1=False,
params=bss,
fast=fast,
log=log,
remove_outliers=False,
optimize_mask=False,
refine_hd_scattering=False)
self.show(fmodel=fmodel,
r_mat=self.total_rotation,
t_vec=self.total_translation,
header="Rigid body refinement",
out=log)
if (monitors_call_back_handler is not None):
monitors_call_back_handler(monitor=None,
model=None,
fmodel=fmodel,
method="rigid_body")
if (bss is not None and params.bulk_solvent_and_scale):
fast = True
if (bss.mode == "slow"): fast = False
fmodel_copy.update_all_scales(update_f_part1=False,
params=bss,
fast=fast,
log=log,
remove_outliers=False,
optimize_mask=False,
refine_hd_scattering=False)
print(file=log)
self.show(fmodel=fmodel,
r_mat=self.total_rotation,
t_vec=self.total_translation,
header="Rigid body end",
out=log)
print(file=log)
self.evaluate_after_end(fmodel, save_r_work, save_r_free,
save_xray_structure, log)
self.fmodel = fmodel
self.fmodel.update(target_name=save_original_target_name)
time_rigid_body_total += timer_rigid_body_total.elapsed()
import cPickle
import pickle
import sys, os
def exercise_hybrid_36():
hybrid_36.exercise(hy36enc=pdb.hy36encode, hy36dec=pdb.hy36decode)
for width,s in [(3,"AAA"), (6,"zzzzzz")]:
try: pdb.hy36encode(width=width, value=0)
except RuntimeError, e:
assert str(e) == "unsupported width."
else: raise Exception_expected
try: pdb.hy36decode(width=width, s=s)
except RuntimeError, e:
assert str(e) == "unsupported width."
else: raise Exception_expected
ups = user_plus_sys_time()
n_ok = pdb.hy36recode_width_4_all()
ups = ups.elapsed()
print "time hy36recode_width_4_all: %.2f s" \
" (%.3f micro s per encode-decode cycle)" % (ups, 1.e6*ups/max(1,n_ok))
assert n_ok == 999+10000+2*26*36**3
#
assert pdb.resseq_decode(s=1234) == 1234
assert pdb.resseq_decode(s="A123") == 11371
assert pdb.resseq_decode(s="1") == 1
pdb.resseq_encode(value=1) == " 1"
pdb.resseq_encode(value=11371) == "A123"
pdb.resseq_encode(value=1234) == "1234"
#
try: pdb.resseq_decode(s="18A")
except ValueError, e:
def run(args, log):
timer = user_plus_sys_time()
format_usage_message(log=log)
if (len(args) == 0): return
parsed = master_params()
inputs = mmtbx.utils.process_command_line_args(args=args,
master_params=parsed,
log=log)
params = inputs.params.extract()
broadcast(m="Input parameters", log=log)
inputs.params.show(prefix=" ")
###
xray_structure = None
if (len(inputs.pdb_file_names) > 0):
broadcast(m="Input model", log=log)
assert len(inputs.pdb_file_names) == 1
print >> log, " file name:", inputs.pdb_file_names[0]
xray_structure = iotbx.pdb.input(
file_name=inputs.pdb_file_names[0]).xray_structure_simple()
assert xray_structure is not None
xray_structure.show_summary(prefix=" ", f=log)
mmtbx.utils.setup_scattering_dictionaries(
scattering_table=params.scattering_table,
xray_structure=xray_structure,
d_min=0.25)
xray_structure.scattering_type_registry().show(prefix=" ", out=log)
###
broadcast(m="Input reflection data", log=log)
reff = inputs.reflection_file_names
if (len(reff) > 1):
raise Sorry("One reflection file should be provided.")
elif (len(reff) == 0):
if (params.hkl_file_name is None):
raise Sorry("No reflection file provided.")
else:
reff = [params.hkl_file_name]
map_coeffs = reflection_file_utils.extract_miller_array_from_file(
file_name=reff[0], label=params.label, type="complex", log=log)
assert map_coeffs is not None
map_coeffs.show_comprehensive_summary(prefix=" ", f=log)
###
broadcast(m="MEM calculations begin", log=log)
f_000 = params.f_000
solvent_fraction = params.solvent_fraction
if (f_000 is None):
f_000_obj = mmtbx.utils.f_000(
xray_structure=xray_structure,
unit_cell_volume=map_coeffs.unit_cell().volume(),
solvent_fraction=params.solvent_fraction,
mean_solvent_density=params.mean_solvent_density)
f_000 = f_000_obj.f_000
solvent_fraction = f_000_obj.solvent_fraction
print >> log, "F(0,0,0): %12.6f" % f_000
if (solvent_fraction is not None):
print >> log, "solvent_fraction: %6.4f" % solvent_fraction
result = mem.run(f=map_coeffs,
f_000=f_000,
lam=params.lam,
lambda_increment_factor=params.lambda_increment_factor,
resolution_factor=params.resolution_factor,
verbose=True,
start_map="min_shifted",
max_iterations=params.max_iterations,
use_modification=True,
beta=params.beta,
convergence_at_r_factor=params.convergence_at_r_factor,
xray_structure=xray_structure,
convergence_r_threshold=params.convergence_r_threshold,
log=log)
###
broadcast(m="Output MEM map coefficients", log=log)
ind = max(0, reff[0].rfind("."))
ofn = params.output_file_name
if (ofn is None):
ofn = reff[0] + "_mem.mtz" if ind == 0 else reff[0][:ind] + "_mem.mtz"
print >> log, " Output file name:", ofn
result.write_mtz_file(file_name=ofn,
column_root_label=params.column_root_label,
d_min=params.output_high_resolution)
broadcast(m="All done", log=log)
return os.path.abspath(ofn)
def run(args):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_call = ["iotbx.python", __file__]
command_line = (iotbx_option_parser(
usage=" ".join(command_call) +
" [options] directory|file...").enable_chunk(
easy_all=True).enable_multiprocessing()).process(args=args,
min_nargs=1)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return
co = command_line.options
#
print "TIME BEGIN cod_refine:", date_and_time()
print
#
master_phil = get_master_phil()
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in command_line.args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
print
params = work_phil.extract()
#
qi_dict = {}
all_pickles = []
for arg in remaining_args:
if (op.isdir(arg)):
for node in sorted(os.listdir(arg)):
if (node.endswith(".pickle")):
all_pickles.append(op.join(arg, node))
elif (node.startswith("qi_") and len(node) == 10):
qi = open(op.join(arg, node)).read().splitlines()
if (len(qi) == 1):
cod_id = node[3:]
quick_info = eval(qi[0])
assert cod_id not in qi_dict
qi_dict[cod_id] = quick_info
elif (op.isfile(arg)):
all_pickles.append(arg)
else:
raise RuntimeError("Not a file or directory: %s" % arg)
print "Number of pickle files:", len(all_pickles)
print "Number of quick_infos:", len(qi_dict)
sort_choice = params.sorting_of_pickle_files
if (len(qi_dict) != 0 and sort_choice is not None):
print "Sorting pickle files by n_atoms * n_refl:", sort_choice
assert sort_choice in ["down", "up"]
def sort_pickle_files():
if (sort_choice == "down"): i_sign = -1
else: i_sign = 1
buffer = []
for i, path in enumerate(all_pickles):
cod_id = op.basename(path).split(".", 1)[0]
qi = qi_dict.get(cod_id)
if (qi is None): nn = 2**31
else: nn = qi[0] * qi[1] * qi[2]
buffer.append((nn, i_sign * i, path))
buffer.sort()
if (i_sign < 0):
buffer.reverse()
result = []
for elem in buffer:
result.append(elem[-1])
return result
all_pickles = sort_pickle_files()
print
#
rss = params.random_subset.size
if (rss is not None and rss > 0):
seed = params.random_subset.seed
print "Selecting subset of %d pickle files using random seed %d" % (
rss, seed)
mt = flex.mersenne_twister(seed=seed)
perm = mt.random_permutation(size=len(all_pickles))[:rss]
flags = flex.bool(len(all_pickles), False).set_selected(perm, True)
all_pickles = flex.select(all_pickles, permutation=flags.iselection())
print
#
from libtbx.path import makedirs_race
if (params.wdir_root is not None):
makedirs_race(path=params.wdir_root)
if (params.pickle_refined_dir is not None):
makedirs_race(path=params.pickle_refined_dir)
#
n_caught = 0
for i_pickle, pickle_file_name in enumerate(all_pickles):
if (i_pickle % command_line.chunk.n != command_line.chunk.i): continue
tm = user_plus_sys_time()
try:
process(params, pickle_file_name)
except KeyboardInterrupt:
print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt"
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
return
except Exception:
sys.stdout.flush()
print >> sys.stderr, "CAUGHT EXCEPTION: %s" % pickle_file_name
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
n_caught += 1
else:
print "done_with: %s (%.2f seconds)" % (pickle_file_name,
tm.elapsed())
print
sys.stdout.flush()
print
print "Number of exceptions caught:", n_caught
#
show_times()
print
print "TIME END cod_refine:", date_and_time()
def __init__(self,
fmodel,
model,
selections,
selections_1d,
refine_T,
refine_L,
refine_S,
number_of_macro_cycles,
max_number_of_iterations,
start_tls_value = None,
run_finite_differences_test = False,
eps = 1.e-6,
out = None,
macro_cycle = None,
verbose = True):
global time_tls_total
timer = user_plus_sys_time()
if(out is None): out = sys.stdout
prefix = "TLS refinement:"
fmodel.info().show_targets(text = prefix+" start model", out = out)
fmodel.xray_structure.show_u_statistics(text = prefix+" start model",
out = out)
xrs = fmodel.xray_structure
xrs.tidy_us(u_min = 1.e-6)
if(start_tls_value is not None):
try:
crash_or_not = abs(start_tls_value + 0)
tlsos = generate_tlsos(value = start_tls_value,
selections = selections,
xray_structure = xrs)
except Exception:
tlsos = start_tls_value
else:
tlsos = tls_from_u_cart(xray_structure = xrs,
tlsos_initial = model.tls_groups.tlsos,
tls_selections = selections,
number_of_macro_cycles = 100,
max_iterations = 100)
if (verbose) :
show_tls(tlsos = tlsos, text = prefix+" start parameters",out = out)
for macro_cycle in range(1, number_of_macro_cycles+1):
print >> out
prefix = "TLS refinement: after macrocycle "+str(macro_cycle)
minimized = tls_xray_target_minimizer(
fmodel = fmodel,
tlsos_initial = tlsos,
refine_T = refine_T,
refine_L = refine_L,
refine_S = refine_S,
selections = selections,
selections_1d = selections_1d,
max_iterations = max_number_of_iterations,
run_finite_differences_test = run_finite_differences_test)
xrs = minimized.fmodel_copy.xray_structure
xrs.show_u_statistics(text = prefix, out = out)
if(verbose):
show_tls(tlsos = minimized.tlsos_result, text = prefix, out = out)
fmodel.update_xray_structure(xray_structure = xrs,
update_f_calc = True)
fmodel.info().show_targets(text = prefix, out = out)
if(xrs.is_positive_definite_u().count(False) > 0):
xrs.tidy_us(u_min = 1.e-6)
xrs.show_u_statistics(
text = prefix+": after making positive definite",
out = out)
fmodel.update_xray_structure(xray_structure = xrs,
update_f_calc = True)
fmodel.info().show_targets(text=prefix+": after making positive definite",
out = out)
tlsos = make_tlso_compatible_with_u_positive_definite(
tlsos = minimized.tlsos_result,
xray_structure = xrs.deep_copy_scatterers(),
selections = selections,
max_iterations = 10,
number_of_u_nonpositive_definite = 0,
eps = eps,
refine_T = refine_T,
refine_L = refine_L,
refine_S = refine_S,
out = out,
number_of_macro_cycles_for_tls_from_uanisos = 10)
else: tlsos = minimized.tlsos_result
if (verbose) :
show_tls(tlsos = tlsos,
text = "TLS refinement: final values", out = out)
self.tlsos = tlsos
model.tls_groups.tlsos = tlsos
self.fmodel = fmodel
time_tls_total += timer.elapsed()
def __init__(self,
fmodels,
model,
all_params,
group_adp_selections=None,
group_adp_selections_h=None,
group_adp_params=group_adp_master_params.extract(),
tls_selections=None,
nm_selections=None,
tls_params=tls_master_params.extract(),
nm_params=nm_master_params.extract(),
individual_adp_params=individual_adp_master_params.extract(),
adp_restraints_params=adp_restraints_master_params.extract(),
refine_adp_individual=None,
refine_adp_group=None,
refine_tls=None,
refine_nm=None,
tan_b_iso_max=None,
restraints_manager=None,
target_weights=None,
macro_cycle=None,
log=None,
h_params=None,
nproc=None):
global time_adp_refinement_py
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
timer = user_plus_sys_time()
if (log is None): log = sys.stdout
tan_u_iso = False
param = 0
if (tan_b_iso_max > 0.0):
tan_u_iso = True
param = int(tan_b_iso_max)
if (macro_cycle == 1):
offset = True
else:
offset = False
if (refine_tls):
print_statistics.make_sub_header(text="TLS refinement", out=log)
tls_sel_st = flex.size_t()
for ts in tls_selections:
tls_sel_st.extend(ts)
tls_sel_bool = flex.bool(scatterers.size(),
flex.size_t(tls_sel_st))
### totally ad hoc fix
tmp_site_t = flex.size_t()
for gs in group_adp_selections:
for gs_ in gs:
tmp_site_t.append(gs_)
###
if (macro_cycle == 1 or tmp_site_t.size() != scatterers.size()):
gbr_selections = []
for s in tls_selections:
gbr_selections.append(s)
else:
gbr_selections = []
for gs in group_adp_selections:
gbr_selection = flex.size_t()
for gs_ in gs:
if (tls_sel_bool[gs_]):
gbr_selection.append(gs_)
if (gbr_selection.size() > 0):
gbr_selections.append(gbr_selection)
gbr_selections_one_arr = flex.size_t()
for gbs in gbr_selections:
gbr_selections_one_arr.extend(gbs)
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
for gbr_selection in gbr_selections_one_arr:
scatterers[gbr_selection].flags.set_use_u_iso(True)
#group_b_manager = mmtbx.refinement.group.manager(
# fmodel = fmodels.fmodel_xray(),
# selections = gbr_selections,
# convergence_test = group_adp_params.convergence_test,
# max_number_of_iterations = 50,
# number_of_macro_cycles = 1,
# refine_adp = True,
# log = log)
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
for tls_selection_ in tls_selections:
for tls_selection__ in tls_selection_:
scatterers[tls_selection__].flags.set_use_u_aniso(True)
model.show_groups(tls=True, out=log)
current_target_name = fmodels.fmodel_xray().target_name
#fmodels.fmodel_xray().update(target_name = "ls_wunit_k1")
tools.split_u(fmodels.fmodel_xray().xray_structure, tls_selections,
offset)
self.tls_refinement_manager = tools.tls_refinement(
fmodel=fmodels.fmodel_xray(),
model=model,
selections=tls_selections,
selections_1d=tls_sel_st,
refine_T=tls_params.refine_T,
refine_L=tls_params.refine_L,
refine_S=tls_params.refine_S,
number_of_macro_cycles=tls_params.number_of_macro_cycles,
max_number_of_iterations=tls_params.max_number_of_iterations,
start_tls_value=tls_params.start_tls_value,
run_finite_differences_test=tls_params.
run_finite_differences_test,
eps=tls_params.eps,
out=log,
macro_cycle=macro_cycle,
verbose=tls_params.verbose)
fmodels.fmodel_xray().update(target_name=current_target_name)
fmodels.update_xray_structure(
xray_structure=self.tls_refinement_manager.fmodel.
xray_structure,
update_f_calc=True)
model.xray_structure = fmodels.fmodel_xray().xray_structure
if (refine_nm):
print_statistics.make_sub_header(text="NM refinement", out=log)
nm_sel_st = flex.size_t()
for ns in nm_selections:
nm_sel_st.extend(ns)
nm_sel_bool = flex.bool(scatterers.size(), flex.size_t(nm_sel_st))
### totally ad hoc fix
tmp_site_t = flex.size_t()
for gs in group_adp_selections:
# print gs.size()
for gs_ in gs:
tmp_site_t.append(gs_)
###
if (macro_cycle == 1 or tmp_site_t.size() != scatterers.size()):
gbr_selections = []
# print "gbr selections are based on nm selections"
for s in nm_selections:
gbr_selections.append(s)
else:
gbr_selections = []
for gs in group_adp_selections:
gbr_selection = flex.size_t()
for gs_ in gs:
if (nm_sel_bool[gs_]):
gbr_selection.append(gs_)
if (gbr_selection.size() > 0):
gbr_selections.append(gbr_selection)
#print "group length ", len(group_adp_selections)
gbr_selections_one_arr = flex.size_t()
for gbs in gbr_selections:
gbr_selections_one_arr.extend(gbs)
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
for gbr_selection in gbr_selections_one_arr:
scatterers[gbr_selection].flags.set_use_u_iso(True)
if nm_params.group_b_refine:
group_b_manager = mmtbx.refinement.group.manager(
fmodel=fmodels.fmodel_xray(),
selections=gbr_selections,
convergence_test=group_adp_params.convergence_test,
max_number_of_iterations=50,
number_of_macro_cycles=1,
refine_adp=True,
log=log)
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
for nm_selection_ in nm_selections:
for nm_selection__ in nm_selection_:
scatterers[nm_selection__].flags.set_use_u_aniso(True)
model.show_groups(nm=True, out=log)
current_target_name = fmodels.fmodel_xray().target_name
if (macro_cycle <= nm_params.stabilize_cycle):
tools.split_u(fmodels.fmodel_xray().xray_structure,
gbr_selections, offset)
elif (macro_cycle <= nm_params.stabilize_cycle):
u_eq = model.xray_structure_initial.extract_u_iso_or_u_equiv()
nmtools.split_u(u_eq,
fmodels.fmodel_xray().xray_structure,
nm_selections, offset)
else:
u_eq = model.xray_structure.extract_u_iso_or_u_equiv()
nmtools.split_u(u_eq,
fmodels.fmodel_xray().xray_structure,
nm_selections, offset)
if not nm_params.update_target_function:
fmodels.fmodel_xray().update(target_name="ls_wunit_k1")
update_eigenvec = False
if nm_params.update_evec and macro_cycle % nm_params.update_evec_per_cycle == 0:
update_eigenvec = True
self.nm_refinement_manager = nmtools.nm_refinement(
fmodel=fmodels.fmodel_xray(),
model=model,
selections=nm_selections,
selections_1d=nm_sel_st,
n_modes=nm_params.n_modes,
number_of_macro_cycles=nm_params.number_of_macro_cycles,
max_number_of_iterations=nm_params.max_number_of_iterations,
weight_nmre=nm_params.weight_nmre,
evecin=nm_params.evecin,
evalin=nm_params.evalin,
fsub_evecin=nm_params.fsub_evecin,
nm_params_filename=nm_params.nm_params_filename,
zero_mode_flag=nm_params.zero_mode_flag,
zero_mode_input_flag=nm_params.zero_mode_input_flag,
update_evec=update_eigenvec,
enmcalc_path=nm_params.enmcalc_path,
start_xs_value=nm_params.start_xs_value,
zero_mode_corr=nm_params.zero_mode_corr,
fsub_ref=nm_params.fsub_ref,
fsub_n_modes=nm_params.fsub_n_modes,
run_finite_differences_test=nm_params.
run_finite_differences_test,
eps=nm_params.eps,
out=log,
macro_cycle=macro_cycle,
verbose=nm_params.verbose)
fmodels.fmodel_xray().update(target_name=current_target_name)
fmodels.update_xray_structure(
xray_structure=self.nm_refinement_manager.fmodel.
xray_structure,
update_f_calc=True)
model.xray_structure = fmodels.fmodel_xray().xray_structure
# from mmtbx.command_line import find_tls_groups
# tls_params = find_tls_groups.master_phil.fetch().extract()
# tls_params.nproc = 1
# tls_selections_str = find_tls_groups.find_tls(
# params = tls_params,
# pdb_inp = None,
# pdb_hierarchy = model.pdb_hierarchy(),
# xray_structure = model.xray_structure,
# return_as_list = True,
# out = log)
# tls_selections = utils.get_atom_selections(
# all_chain_proxies = model.all_chain_proxies,
# selection_strings = tls_selections_str,
# xray_structure = model.xray_structure)
if (refine_adp_individual):
refine_adp(model=model,
fmodels=fmodels,
target_weights=target_weights,
individual_adp_params=individual_adp_params,
adp_restraints_params=adp_restraints_params,
h_params=h_params,
log=log,
all_params=all_params,
nproc=nproc)
if (refine_adp_group):
print_statistics.make_sub_header(
text="group isotropic ADP refinement", out=log)
group_b_manager = mmtbx.refinement.group.manager(
fmodel=fmodels.fmodel_xray(),
selections=group_adp_selections,
convergence_test=group_adp_params.convergence_test,
max_number_of_iterations=group_adp_params.
max_number_of_iterations,
number_of_macro_cycles=group_adp_params.number_of_macro_cycles,
run_finite_differences_test=group_adp_params.
run_finite_differences_test,
refine_adp=True,
log=log)
time_adp_refinement_py += timer.elapsed()
def __init__(
self,
fmodels,
model,
all_params,
group_adp_selections = None,
group_adp_selections_h = None,
group_adp_params = None,
tls_selections = None,
tls_params = tls_master_params.extract(),
individual_adp_params = individual_adp_master_params.extract(),
adp_restraints_params = adp_restraints_master_params.extract(),
refine_adp_individual = None,
refine_adp_group = None,
refine_tls = None,
tan_b_iso_max = None,
restraints_manager = None,
target_weights = None,
macro_cycle = None,
log = None,
h_params = None,
nproc = None):
global time_adp_refinement_py
if(group_adp_params is None):
group_adp_params = group_adp_master_params.extract()
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
timer = user_plus_sys_time()
if(log is None): log = sys.stdout
tan_u_iso = False
param = 0
if(tan_b_iso_max > 0.0):
tan_u_iso = True
param = int(tan_b_iso_max)
if(macro_cycle == 1):
offset = True
else:
offset = False
if(refine_tls):
print_statistics.make_sub_header(text = "TLS refinement",
out = log)
tls_sel_st = flex.size_t()
for ts in tls_selections:
tls_sel_st.extend(ts)
tls_sel_bool = flex.bool(scatterers.size(), flex.size_t(tls_sel_st))
### totally ad hoc fix
tmp_site_t = flex.size_t()
for gs in group_adp_selections:
for gs_ in gs:
tmp_site_t.append(gs_)
###
if(macro_cycle == 1 or tmp_site_t.size() != scatterers.size()):
gbr_selections = []
for s in tls_selections:
gbr_selections.append(s)
else:
gbr_selections = []
for gs in group_adp_selections:
gbr_selection = flex.size_t()
for gs_ in gs:
if(tls_sel_bool[gs_]):
gbr_selection.append(gs_)
if(gbr_selection.size() > 0):
gbr_selections.append(gbr_selection)
gbr_selections_one_arr = flex.size_t()
for gbs in gbr_selections:
gbr_selections_one_arr.extend(gbs)
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
for gbr_selection in gbr_selections_one_arr:
scatterers[gbr_selection].flags.set_use_u_iso(True)
group_b_manager = mmtbx.refinement.group.manager(
fmodel = fmodels.fmodel_xray(),
selections = gbr_selections,
convergence_test = group_adp_params.convergence_test,
max_number_of_iterations = 50,
number_of_macro_cycles = 1,
refine_adp = True,
use_restraints = False, #XXX do not use in TLS refinement for now
log = log)
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
for tls_selection_ in tls_selections:
for tls_selection__ in tls_selection_:
scatterers[tls_selection__].flags.set_use_u_aniso(True)
model.show_groups(tls = True, out = log)
current_target_name = fmodels.fmodel_xray().target_name
fmodels.fmodel_xray().update(target_name = "ls_wunit_k1")
tools.split_u(fmodels.fmodel_xray().xray_structure, tls_selections, offset)
self.tls_refinement_manager = tools.tls_refinement(
fmodel = fmodels.fmodel_xray(),
model = model,
selections = tls_selections,
selections_1d = tls_sel_st,
refine_T = tls_params.refine_T,
refine_L = tls_params.refine_L,
refine_S = tls_params.refine_S,
number_of_macro_cycles = tls_params.number_of_macro_cycles,
max_number_of_iterations = tls_params.max_number_of_iterations,
start_tls_value = tls_params.start_tls_value,
run_finite_differences_test = tls_params.run_finite_differences_test,
eps = tls_params.eps,
out = log,
macro_cycle = macro_cycle,
verbose = tls_params.verbose)
fmodels.fmodel_xray().update(target_name = current_target_name)
fmodels.update_xray_structure(
xray_structure = self.tls_refinement_manager.fmodel.xray_structure,
update_f_calc = True)
model.set_xray_structure(fmodels.fmodel_xray().xray_structure)
if(refine_adp_individual):
refine_adp(
model = model,
fmodels = fmodels,
target_weights = target_weights,
individual_adp_params = individual_adp_params,
adp_restraints_params = adp_restraints_params,
h_params = h_params,
log = log,
all_params = all_params,
nproc = nproc)
if(refine_adp_group):
print_statistics.make_sub_header(
text= "group isotropic ADP refinement", out = log)
group_b_manager = mmtbx.refinement.group.manager(
fmodel = fmodels.fmodel_xray(),
selections = group_adp_selections,
convergence_test = group_adp_params.convergence_test,
max_number_of_iterations = group_adp_params.max_number_of_iterations,
number_of_macro_cycles = group_adp_params.number_of_macro_cycles,
run_finite_differences_test = group_adp_params.run_finite_differences_test,
use_restraints = group_adp_params.use_restraints,
restraints_weight = group_adp_params.restraints_weight,
refine_adp = True,
log = log)
time_adp_refinement_py += timer.elapsed()
def run_and_time(*args, **kw):
timer = user_plus_sys_time()
try:
run(*args, **kw)
finally:
print "CPU time: %.2f seconds" % timer.elapsed()
def run(args):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_call = ["iotbx.python", __file__]
command_line = (iotbx_option_parser(
usage=" ".join(command_call) + " [options] directory|file...")
.enable_chunk(easy_all=True)
.enable_multiprocessing()
).process(args=args, min_nargs=1)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return
co = command_line.options
#
print "TIME BEGIN pdb_dev:", date_and_time()
print
libtbx.utils.host_and_user().show()
print
sys.stdout.flush()
#
from cctbx.omz import cod_refine
master_phil = cod_refine.get_master_phil(
max_atoms=None,
f_calc_options_algorithm="direct *fft",
bulk_solvent_correction=True)
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in command_line.args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
print
params = work_phil.extract()
#
mtz_pdb_pairs = []
arg_iter = iter(remaining_args)
pdb_v3_mirror_dir = os.environ.get("PDB_MIRROR_PDB")
assert pdb_v3_mirror_dir is None or op.isdir(pdb_v3_mirror_dir)
cci_pdbmtz_path = os.environ.get("CCI_PDBMTZ")
assert cci_pdbmtz_path is None or op.isdir(cci_pdbmtz_path)
for arg in arg_iter:
def get_next(expected_exts):
def raise_bad_file(what, fn=None):
msg = "%s file name (%s expected)" % (what, " or ".join(expected_exts))
if (fn is None):
msg += "."
else:
msg += ": " + show_string(fn)
raise RuntimeError(msg)
try:
arg = arg_iter.next()
except StopIteration:
raise_bad_file("Missing")
if (not arg.endswith(tuple(expected_exts))):
raise_bad_file("Unexpected", arg)
return arg
if (op.isfile(arg) and arg.endswith((".mtz", ".pdb", ".ent"))):
if (arg.endswith(".mtz")):
fn_mtz = arg
fn_pdb = get_next([".pdb", ".ent"])
else:
fn_pdb = arg
fn_mtz = get_next([".mtz"])
else:
fn_mtz = arg+".mtz"
def raise_mtz_but_no_pdb():
raise RuntimeError(
"MTZ file found but no PDB file: %s" % show_string(fn_mtz))
if (op.isfile(fn_mtz)):
for ext in [".pdb", ".ent"]:
fn_pdb = arg+ext
if (op.isfile(fn_pdb)):
break
else:
raise_mtz_but_no_pdb()
else:
fn_mtz = op.join(cci_pdbmtz_path, arg+".mtz")
if (not op.isfile(fn_mtz)):
raise RuntimeError(
"MTZ file not found: %s" % show_string(fn_mtz))
fn_pdb = op.join(pdb_v3_mirror_dir, arg[1:3], "pdb"+arg+".ent.gz")
if (not op.isfile(fn_pdb)):
raise_mtz_but_no_pdb()
mtz_pdb_pairs.append((fn_mtz, fn_pdb))
#
n_caught = 0
for i_pair,mtz_pdb_pair in enumerate(mtz_pdb_pairs):
if (i_pair % command_line.chunk.n != command_line.chunk.i): continue
tm = user_plus_sys_time()
try:
process(params, mtz_pdb_pair)
except KeyboardInterrupt:
print >> sys.stderr, "CAUGHT EXCEPTION: KeyboardInterrupt"
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
return
except Exception:
sys.stdout.flush()
print >> sys.stderr, "CAUGHT EXCEPTION: %s" % ", ".join(mtz_pdb_pair)
traceback.print_exc()
print >> sys.stderr
sys.stderr.flush()
n_caught += 1
else:
print "done_with: %s, %s (%.2f seconds)" % (
mtz_pdb_pair + (tm.elapsed(),))
print
sys.stdout.flush()
print
print "Number of exceptions caught:", n_caught
#
show_times()
print
print "TIME END pdb_dev:", date_and_time()
sys.stdout.flush()
def __init__(self,
fmodel,
selections=None,
max_number_of_iterations=50,
number_of_macro_cycles=5,
use_restraints=False,
restraints_weight=None,
convergence_test=True,
convergence_delta=0.00001,
run_finite_differences_test=False,
refine_adp=False,
refine_occ=False,
log=None,
occupancy_max=None,
occupancy_min=None):
global time_group_py
#
tmp = flex.size_t()
for s in selections:
tmp.extend(s)
selections_as_bool = flex.bool(
fmodel.xray_structure.scatterers().size(), tmp)
#
if (log is None): log = sys.stdout
timer = user_plus_sys_time()
self.show(rw=fmodel.r_work(),
rf=fmodel.r_free(),
tw=fmodel.target_w(),
mc=0,
it=0,
refine_adp=refine_adp,
refine_occ=refine_occ,
weight=restraints_weight,
out=log)
assert [refine_adp, refine_occ].count(True) == 1
if (selections is None):
selections = []
selections.append(
flex.bool(fmodel.xray_structure.scatterers().size(), True))
else:
assert len(selections) > 0
par_initial = []
selections_ = []
for sel in selections:
if (refine_adp): par_initial.append(adptbx.b_as_u(0.0))
if (refine_occ): par_initial.append(0.0)
if (str(type(sel).__name__) == "bool"):
selections_.append(sel.iselection())
else:
selections_.append(sel)
selections = selections_
scatterers = fmodel.xray_structure.scatterers()
scatterers.flags_set_grads(state=False)
# XXX very inefficient: same code is in driver.py file. fix asap. Pavel.
save_use_u_iso = fmodel.xray_structure.use_u_iso()
save_use_u_aniso = fmodel.xray_structure.use_u_aniso()
for sel in selections:
if (refine_adp):
for s in sel:
sc = scatterers[s]
if (not sc.flags.use_u_iso()):
sc.flags.set_use_u_iso(True)
if (sc.u_iso == -1): sc.u_iso = 0
scatterers.flags_set_grad_u_iso(iselection=sel)
if (refine_occ):
scatterers.flags_set_grad_occupancy(iselection=sel)
restraints_manager = None
if (use_restraints):
restraints_manager = sphere_similarity_restraints(
xray_structure=fmodel.xray_structure,
selection=selections_as_bool,
refine_adp=refine_adp,
refine_occ=refine_occ)
fmodel_copy = fmodel.deep_copy()
rworks = flex.double()
sc_start = fmodel.xray_structure.scatterers().deep_copy()
minimized = None
self.tested = 0
for macro_cycle in range(1, number_of_macro_cycles + 1, 1):
if (minimized is not None): par_initial = minimized.par_min
minimized = group_minimizer(
fmodel=fmodel_copy,
sc_start=sc_start,
selections=selections,
par_initial=par_initial,
refine_adp=refine_adp,
refine_occ=refine_occ,
max_number_of_iterations=max_number_of_iterations,
run_finite_differences_test=run_finite_differences_test,
restraints_manager=restraints_manager,
restraints_weight=restraints_weight)
if (minimized is not None):
par_initial = minimized.par_min
self.tested += minimized.tested
apply_transformation(xray_structure=fmodel.xray_structure,
par=par_initial,
sc_start=sc_start,
selections=selections,
refine_adp=refine_adp,
refine_occ=refine_occ)
fmodel_copy.update_xray_structure(
xray_structure=fmodel.xray_structure, update_f_calc=True)
rwork = minimized.fmodel.r_work()
rfree = minimized.fmodel.r_free()
assert approx_equal(rwork, fmodel_copy.r_work())
self.show(rw=rwork,
rf=rfree,
tw=minimized.fmodel.target_w(),
mc=macro_cycle,
it=minimized.counter,
refine_adp=refine_adp,
refine_occ=refine_occ,
weight=minimized.weight,
out=log)
if (convergence_test):
rworks.append(rwork)
if (rworks.size() > 1):
size = rworks.size() - 1
if (abs(rworks[size] - rworks[size - 1]) <
convergence_delta):
break
fmodel_copy.xray_structure.tidy_us()
fmodel.update_xray_structure(xray_structure=fmodel_copy.xray_structure,
update_f_calc=True)
if (refine_occ):
i_selection = flex.size_t()
for sel in selections:
i_selection.extend(sel)
fmodel.xray_structure.adjust_occupancy(occ_max=occupancy_max,
occ_min=occupancy_min,
selection=i_selection)
self.fmodel = fmodel
time_group_py += timer.elapsed()
def run(args):
from iotbx.option_parser import option_parser as iotbx_option_parser
import libtbx.utils
show_times = libtbx.utils.show_times(time_start="now")
command_call = ["iotbx.python", __file__]
command_line = (iotbx_option_parser(
usage=" ".join(command_call) +
" [options] directory|file...").enable_chunk(
easy_all=True).enable_multiprocessing()).process(args=args,
min_nargs=1)
if (command_line.run_multiprocessing_chunks_if_applicable(
command_call=command_call)):
show_times()
return
co = command_line.options
#
print("TIME BEGIN pdb_dev:", date_and_time())
print()
libtbx.utils.host_and_user().show()
print()
sys.stdout.flush()
#
from cctbx.omz import cod_refine
master_phil = cod_refine.get_master_phil(
max_atoms=None,
f_calc_options_algorithm="direct *fft",
bulk_solvent_correction=True)
argument_interpreter = master_phil.command_line_argument_interpreter()
phil_objects = []
remaining_args = []
for arg in command_line.args:
if (arg.find("=") >= 0):
phil_objects.append(argument_interpreter.process(arg=arg))
else:
remaining_args.append(arg)
work_phil = master_phil.fetch(sources=phil_objects)
work_phil.show()
print()
params = work_phil.extract()
#
mtz_pdb_pairs = []
arg_iter = iter(remaining_args)
pdb_v3_mirror_dir = os.environ.get("PDB_MIRROR_PDB")
assert pdb_v3_mirror_dir is None or op.isdir(pdb_v3_mirror_dir)
cci_pdbmtz_path = os.environ.get("CCI_PDBMTZ")
assert cci_pdbmtz_path is None or op.isdir(cci_pdbmtz_path)
for arg in arg_iter:
def get_next(expected_exts):
def raise_bad_file(what, fn=None):
msg = "%s file name (%s expected)" % (
what, " or ".join(expected_exts))
if (fn is None):
msg += "."
else:
msg += ": " + show_string(fn)
raise RuntimeError(msg)
try:
arg = next(arg_iter)
except StopIteration:
raise_bad_file("Missing")
if (not arg.endswith(tuple(expected_exts))):
raise_bad_file("Unexpected", arg)
return arg
if (op.isfile(arg) and arg.endswith((".mtz", ".pdb", ".ent"))):
if (arg.endswith(".mtz")):
fn_mtz = arg
fn_pdb = get_next([".pdb", ".ent"])
else:
fn_pdb = arg
fn_mtz = get_next([".mtz"])
else:
fn_mtz = arg + ".mtz"
def raise_mtz_but_no_pdb():
raise RuntimeError("MTZ file found but no PDB file: %s" %
show_string(fn_mtz))
if (op.isfile(fn_mtz)):
for ext in [".pdb", ".ent"]:
fn_pdb = arg + ext
if (op.isfile(fn_pdb)):
break
else:
raise_mtz_but_no_pdb()
else:
fn_mtz = op.join(cci_pdbmtz_path, arg + ".mtz")
if (not op.isfile(fn_mtz)):
raise RuntimeError("MTZ file not found: %s" %
show_string(fn_mtz))
fn_pdb = op.join(pdb_v3_mirror_dir, arg[1:3],
"pdb" + arg + ".ent.gz")
if (not op.isfile(fn_pdb)):
raise_mtz_but_no_pdb()
mtz_pdb_pairs.append((fn_mtz, fn_pdb))
#
n_caught = 0
for i_pair, mtz_pdb_pair in enumerate(mtz_pdb_pairs):
if (i_pair % command_line.chunk.n != command_line.chunk.i): continue
tm = user_plus_sys_time()
try:
process(params, mtz_pdb_pair)
except KeyboardInterrupt:
print("CAUGHT EXCEPTION: KeyboardInterrupt", file=sys.stderr)
traceback.print_exc()
print(file=sys.stderr)
sys.stderr.flush()
return
except Exception:
sys.stdout.flush()
print("CAUGHT EXCEPTION: %s" % ", ".join(mtz_pdb_pair),
file=sys.stderr)
traceback.print_exc()
print(file=sys.stderr)
sys.stderr.flush()
n_caught += 1
else:
print("done_with: %s, %s (%.2f seconds)" % (mtz_pdb_pair +
(tm.elapsed(), )))
print()
sys.stdout.flush()
print()
print("Number of exceptions caught:", n_caught)
#
show_times()
print()
print("TIME END pdb_dev:", date_and_time())
sys.stdout.flush()
def __init__(
self,
fmodel,
selections = None,
max_number_of_iterations = 50,
number_of_macro_cycles = 5,
use_restraints = False,
restraints_weight = None,
convergence_test = True,
convergence_delta = 0.00001,
run_finite_differences_test = False,
refine_adp = False,
refine_occ = False,
log = None,
occupancy_max = None,
occupancy_min = None):
global time_group_py
#
tmp = flex.size_t()
for s in selections: tmp.extend(s)
selections_as_bool = flex.bool(fmodel.xray_structure.scatterers().size(),
tmp)
#
timer = user_plus_sys_time()
self.show(
rw = fmodel.r_work(),
rf = fmodel.r_free(),
tw = fmodel.target_w(),
mc = 0,
it = 0,
refine_adp = refine_adp,
refine_occ = refine_occ,
weight = restraints_weight,
out = log)
if(log is None): log = sys.stdout
assert [refine_adp, refine_occ].count(True) == 1
if(selections is None):
selections = []
selections.append(
flex.bool(fmodel.xray_structure.scatterers().size(), True))
else: assert len(selections) > 0
par_initial = []
selections_ = []
for sel in selections:
if(refine_adp): par_initial.append(adptbx.b_as_u(0.0))
if(refine_occ): par_initial.append(0.0)
if(str(type(sel).__name__) == "bool"):
selections_.append(sel.iselection())
else:
selections_.append(sel)
selections = selections_
scatterers = fmodel.xray_structure.scatterers()
scatterers.flags_set_grads(state=False)
# XXX very inefficient: same code is in driver.py file. fix asap. Pavel.
save_use_u_iso = fmodel.xray_structure.use_u_iso()
save_use_u_aniso = fmodel.xray_structure.use_u_aniso()
for sel in selections:
if(refine_adp):
for s in sel:
sc = scatterers[s]
if(not sc.flags.use_u_iso()):
sc.flags.set_use_u_iso(True)
if(sc.u_iso == -1): sc.u_iso = 0
scatterers.flags_set_grad_u_iso(iselection = sel)
if(refine_occ):
scatterers.flags_set_grad_occupancy(iselection = sel)
restraints_manager = None
if(use_restraints):
restraints_manager = sphere_similarity_restraints(
xray_structure = fmodel.xray_structure,
selection = selections_as_bool,
refine_adp = refine_adp,
refine_occ = refine_occ)
fmodel_copy = fmodel.deep_copy()
rworks = flex.double()
sc_start = fmodel.xray_structure.scatterers().deep_copy()
minimized = None
self.tested = 0
for macro_cycle in xrange(1,number_of_macro_cycles+1,1):
if(minimized is not None): par_initial = minimized.par_min
minimized = group_minimizer(
fmodel = fmodel_copy,
sc_start = sc_start,
selections = selections,
par_initial = par_initial,
refine_adp = refine_adp,
refine_occ = refine_occ,
max_number_of_iterations = max_number_of_iterations,
run_finite_differences_test = run_finite_differences_test,
restraints_manager = restraints_manager,
restraints_weight = restraints_weight)
if(minimized is not None):
par_initial = minimized.par_min
self.tested += minimized.tested
apply_transformation(
xray_structure = fmodel.xray_structure,
par = par_initial,
sc_start = sc_start,
selections = selections,
refine_adp = refine_adp,
refine_occ = refine_occ)
fmodel_copy.update_xray_structure(
xray_structure = fmodel.xray_structure,
update_f_calc = True)
rwork = minimized.fmodel.r_work()
rfree = minimized.fmodel.r_free()
assert approx_equal(rwork, fmodel_copy.r_work())
self.show(
rw = rwork,
rf = rfree,
tw = minimized.fmodel.target_w(),
mc = macro_cycle,
it = minimized.counter,
refine_adp = refine_adp,
refine_occ = refine_occ,
weight = minimized.weight,
out = log)
if(convergence_test):
rworks.append(rwork)
if(rworks.size() > 1):
size = rworks.size() - 1
if(abs(rworks[size]-rworks[size-1])<convergence_delta):
break
fmodel_copy.xray_structure.tidy_us()
fmodel.update_xray_structure(
xray_structure = fmodel_copy.xray_structure, update_f_calc = True)
if(refine_occ):
i_selection = flex.size_t()
for sel in selections:
i_selection.extend(sel)
fmodel.xray_structure.adjust_occupancy(
occ_max = occupancy_max,
occ_min = occupancy_min,
selection = i_selection)
self.fmodel = fmodel
time_group_py += timer.elapsed()
def __init__(self, fmodel,
selections = None,
params = None,
r_initial = None,
t_initial = None,
bss = None,
log = None,
monitors = None):
global time_rigid_body_total
self.params = params
save_original_target_name = fmodel.target_name
save_bss_anisotropic_scaling = None
if(bss is not None):
save_bss_anisotropic_scaling = bss.anisotropic_scaling
timer_rigid_body_total = user_plus_sys_time()
save_r_work = fmodel.r_work()
save_r_free = fmodel.r_free()
save_xray_structure = fmodel.xray_structure.deep_copy_scatterers()
if(log is None): log = sys.stdout
if(selections is None):
selections = []
selections.append(flex.bool(
fmodel.xray_structure.scatterers().size(), True).iselection())
else: assert len(selections) > 0
fmodel.xray_structure.scatterers().flags_set_grads(state=False)
fmodel.xray_structure.scatterers().flags_set_grad_site(
iselection = flex.bool(fmodel.xray_structure.scatterers().size(), True
).iselection())
self.total_rotation = []
self.total_translation = []
for item in selections:
self.total_rotation.append(flex.double(3,0))
self.total_translation.append(flex.double(3,0))
if(r_initial is None):
r_initial = []
for item in selections:
r_initial.append(flex.double(3,0))
if(t_initial is None):
t_initial = []
for item in selections:
t_initial.append(flex.double(3,0))
fmodel_copy = fmodel.deep_copy()
if(fmodel_copy.mask_params is not None):
fmodel_copy.mask_params.verbose = -1
d_mins, target_names = split_resolution_range(
d_spacings = fmodel_copy.f_obs_work().d_spacings().data(),
n_bodies = len(selections),
target = params.target,
target_auto_switch_resolution = params.target_auto_switch_resolution,
n_ref_first = params.min_number_of_reflections,
multi_body_factor_n_ref_first = params.multi_body_factor,
d_low = params.max_low_high_res_limit,
d_high = params.high_resolution,
number_of_zones = params.number_of_zones,
zone_exponent = params.zone_exponent,
log = log)
print >> log
if (fmodel.target_name != target_names[0]):
fmodel.update(target_name=target_names[0])
self.show(fmodel = fmodel,
r_mat = self.total_rotation,
t_vec = self.total_translation,
header = "Start",
out = log)
if (params.number_of_zones == 1 or monitors is None):
monitors_call_back_handler = None
else:
monitors_call_back_handler = monitors.call_back_handler
if (monitors_call_back_handler is not None):
monitors_call_back_handler(
monitor=None, model=None, fmodel=fmodel, method="rigid_body")
for res,target_name in zip(d_mins, target_names):
xrs = fmodel_copy.xray_structure.deep_copy_scatterers()
fmodel_copy = fmodel.resolution_filter(d_min = res)
if (fmodel_copy.target_name != target_name):
fmodel_copy.update(target_name=target_name)
d_max_min = fmodel_copy.f_obs_work().d_max_min()
line = "Refinement at resolution: "+\
str("%7.2f"%d_max_min[0]).strip() + " - " \
+ str("%6.2f"%d_max_min[1]).strip() \
+ " target=" + fmodel_copy.target_name
print_statistics.make_sub_header(line, out = log)
fmodel_copy.update_xray_structure(xray_structure = xrs,
update_f_calc = True)
rworks = flex.double()
if(len(d_mins) == 1):
n_rigid_body_minimizer_cycles = 1
else:
n_rigid_body_minimizer_cycles = min(int(res),4)
for i_macro_cycle in xrange(n_rigid_body_minimizer_cycles):
if(bss is not None and params.bulk_solvent_and_scale):
if(fmodel_copy.f_obs().d_min() > 3.0):
bss.anisotropic_scaling=False
fast=True
if(bss.mode=="slow"): fast=False
fmodel_copy.update_all_scales(
update_f_part1 = False,
params = bss,
fast = fast,
log = log,
remove_outliers = False,
optimize_mask = False,
refine_hd_scattering = False)
if(fmodel_copy.f_obs().d_min() > 3.0):
assert save_bss_anisotropic_scaling is not None
bss.anisotropic_scaling = save_bss_anisotropic_scaling
bss.minimization_b_cart = save_bss_anisotropic_scaling
minimized = rigid_body_minimizer(
fmodel = fmodel_copy,
selections = selections,
r_initial = r_initial,
t_initial = t_initial,
refine_r = params.refine_rotation,
refine_t = params.refine_translation,
max_iterations = params.max_iterations,
euler_angle_convention = params.euler_angle_convention,
lbfgs_maxfev = params.lbfgs_line_search_max_function_evaluations)
rotation_matrices = []
translation_vectors = []
for i in xrange(len(selections)):
self.total_rotation[i] += flex.double(minimized.r_min[i])
self.total_translation[i] += flex.double(minimized.t_min[i])
rot_obj = scitbx.rigid_body.euler(
phi = minimized.r_min[i][0],
psi = minimized.r_min[i][1],
the = minimized.r_min[i][2],
convention = params.euler_angle_convention)
rotation_matrices.append(rot_obj.rot_mat())
translation_vectors.append(minimized.t_min[i])
new_xrs = apply_transformation(
xray_structure = minimized.fmodel.xray_structure,
rotation_matrices = rotation_matrices,
translation_vectors = translation_vectors,
selections = selections)
fmodel_copy.update_xray_structure(xray_structure = new_xrs,
update_f_calc = True,
update_f_mask = True)
rwork = minimized.fmodel.r_work()
rfree = minimized.fmodel.r_free()
assert approx_equal(rwork, fmodel_copy.r_work())
fmodel.update_xray_structure(
xray_structure = fmodel_copy.xray_structure,
update_f_calc = True,
update_f_mask = True)
if(bss is not None and params.bulk_solvent_and_scale):
fast=True
if(bss.mode=="slow"): fast=False
fmodel_copy.update_all_scales(
update_f_part1 = False,
params = bss,
fast = fast,
log = log,
remove_outliers = False,
optimize_mask = False,
refine_hd_scattering = False)
self.show(fmodel = fmodel,
r_mat = self.total_rotation,
t_vec = self.total_translation,
header = "Rigid body refinement",
out = log)
if (monitors_call_back_handler is not None):
monitors_call_back_handler(
monitor=None, model=None, fmodel=fmodel, method="rigid_body")
if(bss is not None and params.bulk_solvent_and_scale):
fast=True
if(bss.mode=="slow"): fast=False
fmodel_copy.update_all_scales(
update_f_part1 = False,
params = bss,
fast = fast,
log = log,
remove_outliers = False,
optimize_mask = False,
refine_hd_scattering = False)
print >> log
self.show(fmodel = fmodel,
r_mat = self.total_rotation,
t_vec = self.total_translation,
header = "Rigid body end",
out = log)
print >> log
self.evaluate_after_end(fmodel, save_r_work, save_r_free,
save_xray_structure, log)
self.fmodel = fmodel
self.fmodel.update(target_name = save_original_target_name)
time_rigid_body_total += timer_rigid_body_total.elapsed()
def timings(structure, d_min, fft_only=False,
wing_cutoff_reference=1.e-6,
wing_cutoff_others=1.e-3):
structure_ng = structure.deep_copy_scatterers()
structure_5g = structure.deep_copy_scatterers()
structure_4g = structure.deep_copy_scatterers()
structure_2g = structure.deep_copy_scatterers()
structure_1g = structure.deep_copy_scatterers()
structure_ng.scattering_type_registry(d_min=d_min, table="n_gaussian")
structure_5g.scattering_type_registry(table="wk1995")
structure_4g.scattering_type_registry(table="it1992")
custom_dict = {}
custom_dict.update(eltbx.xray_scattering.two_gaussian_agarwal_isaacs.table)
structure_2g.scattering_type_registry(custom_dict=custom_dict)
custom_dict.update(eltbx.xray_scattering.one_gaussian_agarwal_1978.table)
structure_1g.scattering_type_registry(custom_dict=custom_dict)
miller_set = miller.build_set(
crystal_symmetry=structure,
d_min=d_min,
anomalous_flag=False)
miller_set.show_summary()
print "d_min:", d_min
if (fft_only):
timer = user_plus_sys_time()
f_calc_reference = xray.structure_factors.from_scatterers(
miller_set=miller_set,
wing_cutoff=wing_cutoff_reference,
exp_table_one_over_step_size=0)(
xray_structure=structure,
miller_set=miller_set,
algorithm="fft").f_calc().data()
print "fft exp function wing_cutoff=%3.1e: %.2f seconds" % (
wing_cutoff_reference, timer.elapsed())
else:
timer = user_plus_sys_time()
f_calc_reference = xray.structure_factors_simple(
structure_5g.unit_cell(),
structure_5g.space_group(),
miller_set.indices(),
structure_5g.scatterers(),
structure_5g.scattering_type_registry()).f_calc()
print "direct simple: %.2f seconds" % timer.elapsed()
f_calc_reference = flex.abs(f_calc_reference)
print "wing_cutoff for following fft calculations: %3.1e"%wing_cutoff_others
for structure in (structure_ng, structure_5g, structure_4g,
structure_2g, structure_1g):
structure.scattering_type_registry().show_summary()
if (not fft_only):
for calc_type,cos_sin_flag in (("direct cos+sin function:",False),
("direct cos+sin table:",True)):
timer = user_plus_sys_time()
f_calc = miller_set.structure_factors_from_scatterers(
xray_structure=structure,
algorithm="direct",
cos_sin_table=cos_sin_flag).f_calc()
print " %-24s %.2f seconds," % (calc_type, timer.elapsed()),
ls = xray.targets_least_squares_residual(
f_calc_reference, f_calc.data(), False, 1)
print "r-factor: %.6f" % ls.target()
for calc_type,exp_table_one_over_step_size in (("fft exp function:",0),
("fft exp table:",-100)):
timer = user_plus_sys_time()
f_calc = xray.structure_factors.from_scatterers(
miller_set=miller_set,
wing_cutoff=wing_cutoff_others,
exp_table_one_over_step_size=exp_table_one_over_step_size)(
xray_structure=structure,
miller_set=miller_set,
algorithm="fft").f_calc()
print " %-24s %.2f seconds," % (calc_type, timer.elapsed()),
ls = xray.targets_least_squares_residual(
f_calc_reference, f_calc.data(), False, 1)
print "r-factor: %.6f" % ls.target()
print
class launcher (runtime_utils.target_with_save_result) :
def run (self) :
os.mkdir(self.output_dir)
os.chdir(self.output_dir)
return run(args=self.args, log=sys.stdout,
use_directory_prefix=False).output_file_name
def validate_params (params) :
if (params.file_name is None) :
raise Sorry("Please specify a model file to minimize.")
if (params.restraints_directory is not None) :
if (not os.path.isdir(params.restraints_directory)) :
raise Sorry("The path '%s' does not exist or is not a directory." %
params.restraints_directory)
return True
def finish_job (result) :
output_files = []
if (result is not None) :
output_files.append((result, "Minimized model"))
return output_files, []
if(__name__ == "__main__"):
timer = user_plus_sys_time()
log = sys.stdout
o = run(sys.argv[1:], log=log)
tt = timer.elapsed()
print >> o.log, "Overall runtime: %-8.3f" % tt
assert abs(tt-o.total_time) < 0.1 # guard against unaccounted times
def __init__(self, fmodels,
restraints_manager = None,
model = None,
is_neutron_scat_table = None,
target_weights = None,
refine_xyz = False,
refine_adp = False,
lbfgs_termination_params = None,
use_fortran = False,
verbose = 0,
correct_special_position_tolerance = 1.0,
iso_restraints = None,
h_params = None,
qblib_params = None,
macro_cycle = None,
u_min = adptbx.b_as_u(-5.0),
u_max = adptbx.b_as_u(999.99),
collect_monitor = True,
log = None):
timer = user_plus_sys_time()
adopt_init_args(self, locals())
self.f=None
self.xray_structure = self.fmodels.fmodel_xray().xray_structure
self.fmodels.create_target_functors()
self.fmodels.prepare_target_functors_for_minimization()
if(self.refine_adp and fmodels.fmodel_neutron() is None):
self.xray_structure.tidy_us()
self.fmodels.update_xray_structure(
xray_structure = self.xray_structure,
update_f_calc = True)
self.weights = None
# QBLIB INSERT
self.qblib_params = qblib_params
if(self.qblib_params is not None and self.qblib_params.qblib):
self.macro = macro_cycle
self.qblib_cycle_count = 0
self.tmp_XYZ = None
self.XYZ_diff_curr=None
# QBLIB END
self.correct_special_position_tolerance = correct_special_position_tolerance
if(refine_xyz and target_weights is not None):
self.weights = target_weights.xyz_weights_result
elif(refine_adp and target_weights is not None):
self.weights = target_weights.adp_weights_result
else:
from phenix.refinement import weight_xray_chem
self.weights = weight_xray_chem.weights(wx = 1,
wx_scale = 1,
angle_x = None,
wn = 1,
wn_scale = 1,
angle_n = None,
w = 0,
wxn = 1)
if(self.collect_monitor):
self.monitor = monitor(
weights = self.weights,
fmodels = fmodels,
model = model,
iso_restraints = iso_restraints,
refine_xyz = refine_xyz,
refine_adp = refine_adp,
refine_occ = False)
if(self.collect_monitor): self.monitor.collect()
self.neutron_refinement = (self.fmodels.fmodel_n is not None)
self.x = flex.double(self.xray_structure.n_parameters(), 0)
self.riding_h_manager = self.model.riding_h_manager
self._scatterers_start = self.xray_structure.scatterers()
#lbfgs_core_params = scitbx.lbfgs.core_parameters(
# stpmin = 1.e-9,
# stpmax = adptbx.b_as_u(10))
self.minimizer = scitbx.lbfgs.run(
target_evaluator = self,
termination_params = lbfgs_termination_params,
use_fortran = use_fortran,
# core_params = lbfgs_core_params,
# gradient_only=True,
exception_handling_params = scitbx.lbfgs.exception_handling_parameters(
ignore_line_search_failed_step_at_lower_bound = True))
self.apply_shifts()
del self._scatterers_start
self.compute_target(compute_gradients = False,u_iso_refinable_params = None)
if(self.refine_adp and self.fmodels.fmodel_neutron() is None):
self.xray_structure.tidy_us()
self.fmodels.update_xray_structure(
xray_structure = self.xray_structure,
update_f_calc = True)
if(self.collect_monitor):
self.monitor.collect(iter = self.minimizer.iter(),
nfun = self.minimizer.nfun())
self.fmodels.create_target_functors()
# QBLIB INSERT
if(self.qblib_params is not None and self.qblib_params.qblib):
print('{:-^80}'.format(""), file=self.qblib_params.qblib_log)
print(file=self.qblib_params.qblib_log)
filename = run(args=self.args,
log=sys.stdout,
use_directory_prefix=False).result_model_fname
return os.path.join(self.output_dir, filename)
def validate_params(params):
if (params.file_name is None):
raise Sorry("Please specify a model file to minimize.")
if (params.restraints_directory is not None):
if (not os.path.isdir(params.restraints_directory)):
raise Sorry("The path '%s' does not exist or is not a directory." %
params.restraints_directory)
return True
def finish_job(result):
output_files = []
if (result is not None):
output_files.append((result, "Minimized model"))
return output_files, []
if (__name__ == "__main__"):
timer = user_plus_sys_time()
log = sys.stdout
o = run(sys.argv[1:], log=log)
tt = timer.elapsed()
print("Overall runtime: %-8.3f" % tt, file=o.log)
assert abs(tt - o.total_time) < 0.1 # guard against unaccounted times
def __init__(
self,
fmodels,
model,
all_params,
group_adp_selections = None,
group_adp_selections_h = None,
group_adp_params = group_adp_master_params.extract(),
tls_selections = None,
nm_selections = None,
tls_params = tls_master_params.extract(),
nm_params = nm_master_params.extract(),
individual_adp_params = individual_adp_master_params.extract(),
adp_restraints_params = adp_restraints_master_params.extract(),
refine_adp_individual = None,
refine_adp_group = None,
refine_tls = None,
refine_nm = None,
tan_b_iso_max = None,
restraints_manager = None,
target_weights = None,
macro_cycle = None,
log = None,
h_params = None,
nproc = None):
global time_adp_refinement_py
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
timer = user_plus_sys_time()
if(log is None): log = sys.stdout
tan_u_iso = False
param = 0
if(tan_b_iso_max > 0.0):
tan_u_iso = True
param = int(tan_b_iso_max)
if(macro_cycle == 1):
offset = True
else:
offset = False
if(refine_tls):
print_statistics.make_sub_header(text = "TLS refinement",
out = log)
tls_sel_st = flex.size_t()
for ts in tls_selections:
tls_sel_st.extend(ts)
tls_sel_bool = flex.bool(scatterers.size(), flex.size_t(tls_sel_st))
### totally ad hoc fix
tmp_site_t = flex.size_t()
for gs in group_adp_selections:
for gs_ in gs:
tmp_site_t.append(gs_)
###
if(macro_cycle == 1 or tmp_site_t.size() != scatterers.size()):
gbr_selections = []
for s in tls_selections:
gbr_selections.append(s)
else:
gbr_selections = []
for gs in group_adp_selections:
gbr_selection = flex.size_t()
for gs_ in gs:
if(tls_sel_bool[gs_]):
gbr_selection.append(gs_)
if(gbr_selection.size() > 0):
gbr_selections.append(gbr_selection)
gbr_selections_one_arr = flex.size_t()
for gbs in gbr_selections:
gbr_selections_one_arr.extend(gbs)
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
for gbr_selection in gbr_selections_one_arr:
scatterers[gbr_selection].flags.set_use_u_iso(True)
group_b_manager = mmtbx.refinement.group.manager(
fmodel = fmodels.fmodel_xray(),
selections = gbr_selections,
convergence_test = group_adp_params.convergence_test,
max_number_of_iterations = 50,
number_of_macro_cycles = 1,
refine_adp = True,
log = log)
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
for tls_selection_ in tls_selections:
for tls_selection__ in tls_selection_:
scatterers[tls_selection__].flags.set_use_u_aniso(True)
model.show_groups(tls = True, out = log)
current_target_name = fmodels.fmodel_xray().target_name
fmodels.fmodel_xray().update(target_name = "ls_wunit_k1")
tools.split_u(fmodels.fmodel_xray().xray_structure, tls_selections, offset)
self.tls_refinement_manager = tools.tls_refinement(
fmodel = fmodels.fmodel_xray(),
model = model,
selections = tls_selections,
selections_1d = tls_sel_st,
refine_T = tls_params.refine_T,
refine_L = tls_params.refine_L,
refine_S = tls_params.refine_S,
number_of_macro_cycles = tls_params.number_of_macro_cycles,
max_number_of_iterations = tls_params.max_number_of_iterations,
start_tls_value = tls_params.start_tls_value,
run_finite_differences_test = tls_params.run_finite_differences_test,
eps = tls_params.eps,
out = log,
macro_cycle = macro_cycle,
verbose = tls_params.verbose)
fmodels.fmodel_xray().update(target_name = current_target_name)
fmodels.update_xray_structure(
xray_structure = self.tls_refinement_manager.fmodel.xray_structure,
update_f_calc = True)
model.xray_structure = fmodels.fmodel_xray().xray_structure
if(refine_nm):
print_statistics.make_sub_header(text = "NM refinement",
out = log)
nm_sel_st = flex.size_t()
for ns in nm_selections:
nm_sel_st.extend(ns)
scatterers = fmodels.fmodel_xray().xray_structure.scatterers()
for nm_selection_ in nm_selections:
for nm_selection__ in nm_selection_:
scatterers[nm_selection__].flags.set_use_u_aniso(True)
model.show_groups(nm = True, out = log)
current_target_name = fmodels.fmodel_xray().target_name
# fmodels.fmodel_xray().update(target_name = "ls_wunit_k1")
# tools.split_u(fmodels.fmodel_xray().xray_structure, tls_selections, offset)
self.nm_refinement_manager = nmtools.nm_refinement(
fmodel = fmodels.fmodel_xray(),
model = model,
selections = nm_selections,
selections_1d = nm_sel_st,
n_modes = nm_params.n_modes,
number_of_macro_cycles = nm_params.number_of_macro_cycles,
max_number_of_iterations = nm_params.max_number_of_iterations,
weight_nmre = nm_params.weight_nmre,
evecin = nm_params.evecin,
evalin = nm_params.evalin,
nm_params_filename = nm_params.nm_params_filename,
zero_mode_flag = nm_params.zero_mode_flag,
zero_mode_input_flag = nm_params.zero_mode_input_flag,
start_xs_value = nm_params.start_xs_value,
run_finite_differences_test = nm_params.run_finite_differences_test,
eps = nm_params.eps,
out = log,
macro_cycle = macro_cycle,
verbose = nm_params.verbose)
fmodels.fmodel_xray().update(target_name = current_target_name)
fmodels.update_xray_structure(
xray_structure = self.nm_refinement_manager.fmodel.xray_structure,
update_f_calc = True)
model.xray_structure = fmodels.fmodel_xray().xray_structure
if(refine_adp_individual):
refine_adp(
model = model,
fmodels = fmodels,
target_weights = target_weights,
individual_adp_params = individual_adp_params,
adp_restraints_params = adp_restraints_params,
h_params = h_params,
log = log,
all_params = all_params,
nproc = nproc)
if(refine_adp_group):
print_statistics.make_sub_header(text= "group isotropic ADP refinement",
out = log)
group_b_manager = mmtbx.refinement.group.manager(
fmodel = fmodels.fmodel_xray(),
selections = group_adp_selections,
convergence_test = group_adp_params.convergence_test,
max_number_of_iterations = group_adp_params.max_number_of_iterations,
number_of_macro_cycles = group_adp_params.number_of_macro_cycles,
run_finite_differences_test = group_adp_params.run_finite_differences_test,
refine_adp = True,
log = log)
time_adp_refinement_py += timer.elapsed()
