def find_anomalous_scatterer_groups(
pdb_atoms,
xray_structure,
group_same_element=True, # XXX should this be True by default?
out=None):
"""
Automatic setup of anomalously scattering atoms, defined here as anything
with atomic number 15 (P) or greater. Not yet accessible from phenix.refine.
"""
from cctbx.eltbx import sasaki
from cctbx import xray
if (out is None): out = sys.stdout
element_i_seqs = {}
groups = []
if (out is None): out = null_out()
hd_selection = xray_structure.hd_selection()
for i_seq, scatterer in enumerate(xray_structure.scatterers()):
if (hd_selection[i_seq]):
continue
element = scatterer.element_symbol().strip()
try:
atomic_number = sasaki.table(element).atomic_number()
except RuntimeError as e:
print("Error for %s" % pdb_atoms[i_seq].id_str(), file=out)
print(" " + str(e), file=out)
continue
if (atomic_number >= 15):
if (group_same_element):
if (not element in element_i_seqs):
element_i_seqs[element] = flex.size_t()
element_i_seqs[element].append(i_seq)
else:
print(" creating anomalous group for %s" % \
pdb_atoms[i_seq].id_str(), file=out)
asg = xray.anomalous_scatterer_group(
iselection=flex.size_t([i_seq]),
f_prime=0,
f_double_prime=0,
refine=["f_prime", "f_double_prime"],
selection_string=get_single_atom_selection_string(
pdb_atoms[i_seq]))
groups.append(asg)
if (group_same_element):
for elem in sorted(element_i_seqs.keys()):
iselection = element_i_seqs[elem]
print(" creating anomalous group for element %s with %d atoms" % \
(elem, len(iselection)), file=out)
asg = xray.anomalous_scatterer_group(
iselection=iselection,
f_prime=0,
f_double_prime=0,
refine=["f_prime", "f_double_prime"],
selection_string="element %s" % elem)
groups.append(asg)
return groups
def run_call_back(flags, space_group_info):
d_min = 2.0
structure = random_structure.xray_structure(
space_group_info=space_group_info,
elements=["N", "C", "O", "S"]*3 + ["Fe"]*2,
volume_per_atom=100)
if (not space_group_info.group().is_centric()):
fp_fdp_targets = [(-1,2), (-2,6)]
else:
fp_fdp_targets = [(-1,0), (-2,0)]
anomalous_scatterer_groups = [
xray.anomalous_scatterer_group(
iselection=flex.size_t(),
f_prime=fp,
f_double_prime=fdp,
refine=["f_prime", "f_double_prime"]) for fp,fdp in fp_fdp_targets]
for i_seq,scatterer in enumerate(structure.scatterers()):
if (scatterer.scattering_type == "S"):
anomalous_scatterer_groups[0].iselection.append(i_seq)
if (scatterer.scattering_type == "Fe"):
anomalous_scatterer_groups[1].iselection.append(i_seq)
for group in anomalous_scatterer_groups:
group.copy_to_scatterers_in_place(scatterers=structure.scatterers())
if (flags.Verbose):
structure.show_summary().show_scatterers()
f_obs = abs(structure.structure_factors(
d_min=2.0, anomalous_flag=True).f_calc())
if (flags.Verbose):
f_obs.show_comprehensive_summary()
#
for group in anomalous_scatterer_groups:
group.f_prime = 0
group.f_double_prime = 0
group.copy_to_scatterers_in_place(scatterers=structure.scatterers())
sfg_params = mmtbx.f_model.sf_and_grads_accuracy_master_params.extract()
sfg_params.algorithm = "direct"
fmodel = mmtbx.f_model.manager(
xray_structure=structure,
f_obs=f_obs,
r_free_flags=f_obs.generate_r_free_flags(),
sf_and_grads_accuracy_params = sfg_params,
target_name="ls")
#
n_cycles = [0]
def call_back(minimizer):
n_cycles[0] += 1
return True
minimized = mmtbx.refinement.anomalous_scatterer_groups.minimizer(
fmodel=fmodel,
groups=anomalous_scatterer_groups,
call_back_after_minimizer_cycle=call_back,
number_of_finite_difference_tests=3)
assert n_cycles == [3]
#
for group,(fp,fdp) in zip(anomalous_scatterer_groups, fp_fdp_targets):
# Large eps because the minimization doesn't reliably converge.
# We don't want to exercise the minimizer here, the important
# test is the finite difference test embedded in the minimizer.
assert approx_equal(group.f_prime, fp, eps=1)
assert approx_equal(group.f_double_prime, fdp, eps=1)
def find_anomalous_scatterer_groups (
pdb_atoms,
xray_structure,
group_same_element=True, # XXX should this be True by default?
out=None) :
"""
Automatic setup of anomalously scattering atoms, defined here as anything
with atomic number 15 (P) or greater. Not yet accessible from phenix.refine.
"""
from cctbx.eltbx import sasaki
from cctbx import xray
if (out is None) : out = sys.stdout
element_i_seqs = {}
groups = []
if (out is None) : out = null_out()
hd_selection = xray_structure.hd_selection()
for i_seq, scatterer in enumerate(xray_structure.scatterers()) :
if (hd_selection[i_seq]) :
continue
element = scatterer.element_symbol().strip()
try :
atomic_number = sasaki.table(element).atomic_number()
except RuntimeError, e :
print >> out, "Error for %s" % pdb_atoms[i_seq].id_str()
print >> out, " " + str(e)
continue
if (atomic_number >= 15) :
if (group_same_element) :
if (not element in element_i_seqs) :
element_i_seqs[element] = flex.size_t()
element_i_seqs[element].append(i_seq)
else :
print >> out, " creating anomalous group for %s" % \
pdb_atoms[i_seq].id_str()
asg = xray.anomalous_scatterer_group(
iselection=flex.size_t([i_seq]),
f_prime=0,
f_double_prime=0,
refine=["f_prime","f_double_prime"],
selection_string=get_single_atom_selection_string(pdb_atoms[i_seq]))
groups.append(asg)
def find_and_build_ions (
manager,
fmodels,
model,
wavelength,
params,
nproc=1,
elements=Auto,
out=None,
run_ordered_solvent=False,
occupancy_strategy_enabled=False,
group_anomalous_strategy_enabled=False,
use_svm=None) :
"""
Analyzes the water molecules in a structure and re-labels them as ions if
they scatter and bind environments that we expect of that ion.
Parameters
----------
manager : mmtbx.ions.identity.manager
fmodels : mmtbx.fmodels
model : mmtbx.model.manager
wavelength : float
params : libtbx.phil.scope_extract
nproc : int, optional
elements : list of str, optional
out : file, optional
run_ordered_solvent : bool, optional
occupancy_strategy_enabled : bool, optional
group_anomalous_strategy_enabled : bool, optional
use_svm : bool, optional
See Also
--------
mmtbx.ions.identify.manager.analyze_waters
"""
import mmtbx.refinement.minimization
from mmtbx.refinement.anomalous_scatterer_groups import \
get_single_atom_selection_string
from mmtbx.refinement import anomalous_scatterer_groups
import mmtbx.ions.identify
import mmtbx.ions.svm
from cctbx.eltbx import sasaki
from cctbx import crystal
from cctbx import adptbx
from cctbx import xray
from scitbx.array_family import flex
import scitbx.lbfgs
if (use_svm is None) :
use_svm = getattr(params, "use_svm", False)
assert (1.0 >= params.initial_occupancy >= 0)
fmodel = fmodels.fmodel_xray()
anomalous_flag = fmodel.f_obs().anomalous_flag()
if (out is None) : out = sys.stdout
model.xray_structure = fmodel.xray_structure
model.xray_structure.tidy_us()
pdb_hierarchy = model.pdb_hierarchy(sync_with_xray_structure=True)
pdb_atoms = pdb_hierarchy.atoms()
pdb_atoms.reset_i_seq()
# FIXME why does B for anisotropic waters end up negative?
u_iso = model.xray_structure.extract_u_iso_or_u_equiv()
for i_seq, atom in enumerate(pdb_atoms) :
labels = atom.fetch_labels()
if (labels.resname == "HOH") and (atom.b < 0) :
assert (u_iso[i_seq] >= 0)
atom.b = adptbx.u_as_b(u_iso[i_seq])
if (manager is None) :
manager_class = None
if (use_svm) :
manager_class = mmtbx.ions.svm.manager
if params.svm.svm_name == "merged_high_res" :
params.find_anomalous_substructure = False
params.use_phaser = False
manager = mmtbx.ions.identify.create_manager(
pdb_hierarchy=pdb_hierarchy,
geometry_restraints_manager=model.restraints_manager.geometry,
fmodel=fmodel,
wavelength=wavelength,
params=params,
nproc=nproc,
verbose=params.debug,
log=out,
manager_class=manager_class)
else :
grm = model.restraints_manager.geometry
connectivity = grm.shell_sym_tables[0].full_simple_connectivity()
manager.update_structure(
pdb_hierarchy=pdb_hierarchy,
xray_structure=fmodel.xray_structure,
connectivity=connectivity,
log=out)
manager.update_maps()
model.update_anomalous_groups(out=out)
make_sub_header("Analyzing water molecules", out=out)
manager.show_current_scattering_statistics(out=out)
anomalous_groups = []
# XXX somehow comma-separation of phil strings fields doesn't work
if (isinstance(elements, list)) and (len(elements) == 1) :
elements = elements[0].split(",")
water_ion_candidates = manager.analyze_waters(
out=out,
candidates=elements)
modified_iselection = flex.size_t()
default_b_iso = manager.get_initial_b_iso()
# Build in the identified ions
for_building = []
if (use_svm) :
for result in water_ion_candidates :
for_building.append((result.i_seq, result.final_choice))
else :
for i_seq, final_choices, two_fofc in water_ion_candidates :
if (len(final_choices) == 1) :
for_building.append((i_seq, final_choices[0]))
skipped = []
if (len(for_building) > 0) :
make_sub_header("Adding %d ions to model" % len(for_building), out)
for k, (i_seq, final_choice) in enumerate(for_building) :
atom = manager.pdb_atoms[i_seq]
skip = False
for other_i_seq, other_ion in for_building[:k] :
if (other_i_seq in skipped) : continue
if (((other_ion.charge > 0) and (final_choice.charge > 0)) or
((other_ion.charge < 0) and (final_choice.charge < 0))) :
other_atom = manager.pdb_atoms[other_i_seq]
dxyz = atom.distance(other_atom)
if (dxyz < params.max_distance_between_like_charges) :
print >> out, \
" %s (%s%+d) is only %.3fA from %s (%s%+d), skipping for now" %\
(atom.id_str(), final_choice.element, final_choice.charge, dxyz,
other_atom.id_str(), other_ion.element, other_ion.charge)
skipped.append(i_seq)
skip = True
break
if (skip) : continue
print >> out, " %s becomes %s%+d" % \
(atom.id_str(), final_choice.element, final_choice.charge)
refine_adp = params.refine_ion_adp
if (refine_adp == "Auto") :
if (fmodel.f_obs().d_min() <= 1.5) :
refine_adp = "anisotropic"
elif (fmodel.f_obs().d_min() < 2.5) :
atomic_number = sasaki.table(final_choice.element).atomic_number()
if (atomic_number >= 19) :
refine_adp = "anisotropic"
# Modify the atom object - this is clumsy but they will be grouped into
# a single chain at the end of refinement
initial_b_iso = params.initial_b_iso
if (initial_b_iso is Auto) :
initial_b_iso = manager.guess_b_iso_real(i_seq)
element = final_choice.element
if (element == "IOD") : # FIXME
element = "I"
modified_atom = model.convert_atom(
i_seq=i_seq,
scattering_type=final_choice.scattering_type(),
atom_name=element,
element=element,
charge=final_choice.charge,
residue_name=final_choice.element,
initial_occupancy=params.initial_occupancy,
initial_b_iso=initial_b_iso,
chain_id=params.ion_chain_id,
segid="ION",
refine_adp=refine_adp,
refine_occupancies=False) #params.refine_ion_occupancies)
if (params.refine_anomalous) and (anomalous_flag) :
scatterer = model.xray_structure.scatterers()[i_seq]
if (wavelength is not None) :
fp_fdp_info = sasaki.table(final_choice.element).at_angstrom(
wavelength)
scatterer.fp = fp_fdp_info.fp()
scatterer.fdp = fp_fdp_info.fdp()
print >> out, " setting f'=%g, f''=%g" % (scatterer.fp,
scatterer.fdp)
group = xray.anomalous_scatterer_group(
iselection=flex.size_t([i_seq]),
f_prime=scatterer.fp,
f_double_prime=scatterer.fdp,
refine=["f_prime","f_double_prime"],
selection_string=get_single_atom_selection_string(modified_atom),
update_from_selection=True)
anomalous_groups.append(group)
modified_iselection.append(i_seq)
if (len(modified_iselection) > 0) :
scatterers = model.xray_structure.scatterers()
# FIXME not sure this is actually working as desired...
site_symmetry_table = model.xray_structure.site_symmetry_table()
for i_seq in site_symmetry_table.special_position_indices() :
scatterers[i_seq].site = crystal.correct_special_position(
crystal_symmetry=model.xray_structure,
special_op=site_symmetry_table.get(i_seq).special_op(),
site_frac=scatterers[i_seq].site,
site_label=scatterers[i_seq].label,
tolerance=1.0)
model.xray_structure.replace_scatterers(scatterers=scatterers)
def show_r_factors () :
return "r_work=%6.4f r_free=%6.4f" % (fmodel.r_work(), fmodel.r_free())
fmodel.update_xray_structure(
xray_structure=model.xray_structure,
update_f_calc=True,
update_f_mask=True)
n_anom = len(anomalous_groups)
refine_anomalous = anomalous_flag and params.refine_anomalous and n_anom>0
refine_occupancies = ((params.refine_ion_occupancies or refine_anomalous)
and ((not occupancy_strategy_enabled) or
(model.refinement_flags.s_occupancies is None) or
(len(model.refinement_flags.s_occupancies) == 0)))
if (refine_anomalous) :
if ((model.anomalous_scatterer_groups is not None) and
(group_anomalous_strategy_enabled)) :
model.anomalous_scatterer_groups.extend(anomalous_groups)
refine_anomalous = False
if (refine_occupancies) or (refine_anomalous) :
print >> out, ""
print >> out, " occupancy refinement (new ions only): start %s" % \
show_r_factors()
fmodel.xray_structure.scatterers().flags_set_grads(state = False)
fmodel.xray_structure.scatterers().flags_set_grad_occupancy(
iselection = modified_iselection)
lbfgs_termination_params = scitbx.lbfgs.termination_parameters(
max_iterations = 25)
minimized = mmtbx.refinement.minimization.lbfgs(
restraints_manager = None,
fmodels = fmodels,
model = model,
is_neutron_scat_table = False,
lbfgs_termination_params = lbfgs_termination_params)
fmodel.xray_structure.adjust_occupancy(
occ_max = 1.0,
occ_min = 0,
selection = modified_iselection)
zero_occ = []
for i_seq in modified_iselection :
occ = fmodel.xray_structure.scatterers()[i_seq].occupancy
if (occ == 0) :
zero_occ.append(i_seq)
fmodel.update_xray_structure(
update_f_calc=True,
update_f_mask=True)
print >> out, " final %s" % \
show_r_factors()
if (len(zero_occ) > 0) :
print >> out, " WARNING: occupancy dropped to zero for %d atoms:"
atoms = model.pdb_hierarchy().atoms()
for i_seq in zero_occ :
print >> out, " %s" % atoms[i_seq].id_str(suppress_segid=True)
print >> out, ""
if (refine_anomalous) :
assert fmodel.f_obs().anomalous_flag()
print >> out, " anomalous refinement (new ions only): start %s" % \
show_r_factors()
fmodel.update(target_name="ls")
anomalous_scatterer_groups.minimizer(
fmodel=fmodel,
groups=anomalous_groups)
fmodel.update(target_name="ml")
print >> out, " final %s" % \
show_r_factors()
print >> out, ""
return manager
def refine_anomalous_substructure(fmodel,
pdb_hierarchy,
wavelength=None,
map_type="anom_residual",
exclude_waters=False,
exclude_non_water_light_elements=True,
n_cycles_max=None,
map_sigma_min=3.0,
refine=("f_prime", "f_double_prime"),
reset_water_u_iso=True,
use_all_anomalous=True,
verbose=True,
out=sys.stdout):
"""
Crude mimic of Phaser's substructure completion, with two essential
differences: only the existing real scatterers in the input model will be
used (with the assumption that the model is already more or less complete),
and the anomalous refinement will be performed in Phenix, yielding both
f-prime and f-double-prime. The refined f-prime provides us with an
orthogonal estimate of the number of electrons missing from an incorrectly
labeled scatterer.
:param wavelength: X-ray wavelenth in Angstroms
:param exclude_waters: Don't refine anomalous scattering for water oxygens
:param exclude_non_water_light_elements: Don't refine anomalous scattering
for light atoms other than water (CHNO).
:param n_cycles_max: Maximum number of refinement cycles
:param map_sigma_min: Sigma cutoff for identify anomalous scatterers
:param reset_water_u_iso: Reset B-factors for water atoms prior to f'
refinement
:param use_all_anomalous: include any scatterers which are already modeled
as anomalous in the refinement
"""
from cctbx import xray
assert (fmodel.f_obs().anomalous_flag())
assert (map_type in ["llg", "anom_residual"])
make_sub_header("Iterative anomalous substructure refinement", out=out)
fmodel.update(target_name="ls")
pdb_atoms = pdb_hierarchy.atoms()
non_water_non_hd_selection = pdb_hierarchy.atom_selection_cache(
).selection("(not element H and not element D and not resname HOH)")
sites_frac = fmodel.xray_structure.sites_frac()
scatterers = fmodel.xray_structure.scatterers()
u_iso_mean = flex.mean(
fmodel.xray_structure.extract_u_iso_or_u_equiv().select(
non_water_non_hd_selection))
anomalous_iselection = flex.size_t()
anomalous_groups = []
t_start = time.time()
n_cycle = 0
while ((n_cycles_max is None) or (n_cycle < n_cycles_max)):
n_cycle += 1
n_new_groups = 0
t_start_cycle = time.time()
print >> out, "Cycle %d" % n_cycle
anom_map = fmodel.map_coefficients(map_type=map_type).fft_map(
resolution_factor=0.25).apply_sigma_scaling().real_map_unpadded()
map_min = abs(flex.min(anom_map.as_1d()))
map_max = flex.max(anom_map.as_1d())
print >> out, " map range: -%.2f sigma to %.2f sigma" % (map_min,
map_max)
reset_u_iso_selection = flex.size_t()
for i_seq, atom in enumerate(pdb_atoms):
resname = atom.parent().resname
elem = atom.element.strip()
if ((i_seq in anomalous_iselection)
or ((exclude_waters) and (resname == "HOH")) or
((elem in ["H", "D", "N", "C", "O"]) and
(resname != "HOH") and exclude_non_water_light_elements)):
continue
scatterer = scatterers[i_seq]
site_frac = sites_frac[i_seq]
anom_map_value = anom_map.tricubic_interpolation(site_frac)
if ((anom_map_value >= map_sigma_min)
or ((scatterer.fdp != 0) and use_all_anomalous)):
if (verbose):
if (n_new_groups == 0):
print >> out, ""
print >> out, " new anomalous scatterers:"
print >> out, " %-34s map height: %6.2f sigma" % (
atom.id_str(), anom_map_value)
anomalous_iselection.append(i_seq)
selection_string = get_single_atom_selection_string(atom)
group = xray.anomalous_scatterer_group(
iselection=flex.size_t([i_seq]),
f_prime=0,
f_double_prime=0,
refine=list(refine),
selection_string=selection_string)
anomalous_groups.append(group)
n_new_groups += 1
if (resname == "HOH") and (reset_water_u_iso):
water_u_iso = scatterer.u_iso
if (water_u_iso < u_iso_mean):
reset_u_iso_selection.append(i_seq)
if (n_new_groups == 0):
print >> out, ""
print >> out, "No new groups - anomalous scatterer search terminated."
break
elif (not verbose):
print >> out, " %d new groups" % n_new_groups
for i_seq in anomalous_iselection:
sc = scatterers[i_seq]
sc.fp = 0
sc.fdp = 0
if (verbose):
print >> out, ""
print >> out, "Anomalous refinement:"
fmodel.info().show_targets(text="before minimization", out=out)
print >> out, ""
u_iso = fmodel.xray_structure.extract_u_iso_or_u_equiv()
u_iso.set_selected(reset_u_iso_selection, u_iso_mean)
fmodel.xray_structure.set_u_iso(values=u_iso)
fmodel.update_xray_structure(update_f_calc=True)
minimizer(fmodel=fmodel, groups=anomalous_groups)
if (verbose):
fmodel.info().show_targets(text="after minimization", out=out)
print >> out, ""
print >> out, " Refined sites:"
for i_seq, group in zip(anomalous_iselection, anomalous_groups):
print >> out, " %-34s f' = %6.3f f'' = %6.3f" % (
pdb_atoms[i_seq].id_str(), group.f_prime,
group.f_double_prime)
t_end_cycle = time.time()
print >> out, ""
if (verbose):
print >> out, " time for this cycle: %.1fs" % (t_end_cycle -
t_start_cycle)
fmodel.update(target_name="ml")
print >> out, "%d anomalous scatterer groups refined" % len(
anomalous_groups)
t_end = time.time()
print >> out, "overall time: %.1fs" % (t_end - t_start)
return anomalous_groups
iselection=flex.size_t([i_seq]),
f_prime=0,
f_double_prime=0,
refine=["f_prime", "f_double_prime"],
selection_string=get_single_atom_selection_string(
pdb_atoms[i_seq]))
groups.append(asg)
if (group_same_element):
for elem in sorted(element_i_seqs.keys()):
iselection = element_i_seqs[elem]
print >> out, \
" creating anomalous group for element %s with %d atoms" % \
(elem, len(iselection))
asg = xray.anomalous_scatterer_group(
iselection=iselection,
f_prime=0,
f_double_prime=0,
refine=["f_prime", "f_double_prime"],
selection_string="element %s" % elem)
groups.append(asg)
return groups
def refine_anomalous_substructure(fmodel,
pdb_hierarchy,
wavelength=None,
map_type="anom_residual",
exclude_waters=False,
exclude_non_water_light_elements=True,
n_cycles_max=None,
map_sigma_min=3.0,
refine=("f_prime", "f_double_prime"),
def find_and_build_ions(manager,
fmodels,
model,
wavelength,
params,
nproc=1,
elements=Auto,
out=None,
run_ordered_solvent=False,
occupancy_strategy_enabled=False,
group_anomalous_strategy_enabled=False,
use_svm=None):
"""
Analyzes the water molecules in a structure and re-labels them as ions if
they scatter and bind environments that we expect of that ion.
Parameters
----------
manager : mmtbx.ions.identity.manager
fmodels : mmtbx.fmodels
model : mmtbx.model.manager
wavelength : float
params : libtbx.phil.scope_extract
nproc : int, optional
elements : list of str, optional
out : file, optional
run_ordered_solvent : bool, optional
occupancy_strategy_enabled : bool, optional
group_anomalous_strategy_enabled : bool, optional
use_svm : bool, optional
See Also
--------
mmtbx.ions.identify.manager.analyze_waters
"""
import mmtbx.refinement.minimization
from mmtbx.refinement.anomalous_scatterer_groups import \
get_single_atom_selection_string
from mmtbx.refinement import anomalous_scatterer_groups
import mmtbx.ions.identify
import mmtbx.ions.svm
from cctbx.eltbx import sasaki
from cctbx import crystal
from cctbx import adptbx
from cctbx import xray
from scitbx.array_family import flex
import scitbx.lbfgs
if (use_svm is None):
use_svm = getattr(params, "use_svm", False)
assert (1.0 >= params.initial_occupancy >= 0)
fmodel = fmodels.fmodel_xray()
anomalous_flag = fmodel.f_obs().anomalous_flag()
if (out is None): out = sys.stdout
model.set_xray_structure(fmodel.xray_structure)
model.get_xray_structure().tidy_us()
pdb_hierarchy = model.get_hierarchy()
pdb_atoms = pdb_hierarchy.atoms()
pdb_atoms.reset_i_seq()
# FIXME why does B for anisotropic waters end up negative?
u_iso = model.get_xray_structure().extract_u_iso_or_u_equiv()
for i_seq, atom in enumerate(pdb_atoms):
labels = atom.fetch_labels()
if (labels.resname == "HOH") and (atom.b < 0):
assert (u_iso[i_seq] >= 0)
atom.b = adptbx.u_as_b(u_iso[i_seq])
if (manager is None):
manager_class = None
if (use_svm):
manager_class = mmtbx.ions.svm.manager
if params.svm.svm_name == "merged_high_res":
params.find_anomalous_substructure = False
params.use_phaser = False
manager = mmtbx.ions.identify.create_manager(
pdb_hierarchy=pdb_hierarchy,
geometry_restraints_manager=model.restraints_manager.geometry,
fmodel=fmodel,
wavelength=wavelength,
params=params,
nproc=nproc,
verbose=params.debug,
log=out,
manager_class=manager_class)
else:
grm = model.get_restraints_manager().geometry
connectivity = grm.shell_sym_tables[0].full_simple_connectivity()
manager.update_structure(pdb_hierarchy=pdb_hierarchy,
xray_structure=fmodel.xray_structure,
connectivity=connectivity,
log=out)
manager.update_maps()
model.update_anomalous_groups(out=out)
make_sub_header("Analyzing water molecules", out=out)
manager.show_current_scattering_statistics(out=out)
anomalous_groups = []
# XXX somehow comma-separation of phil strings fields doesn't work
if (isinstance(elements, list)) and (len(elements) == 1):
elements = elements[0].split(",")
water_ion_candidates = manager.analyze_waters(out=out, candidates=elements)
modified_iselection = flex.size_t()
default_b_iso = manager.get_initial_b_iso()
# Build in the identified ions
for_building = []
if (use_svm):
for result in water_ion_candidates:
for_building.append((result.i_seq, result.final_choice))
else:
for i_seq, final_choices, two_fofc in water_ion_candidates:
if (len(final_choices) == 1):
for_building.append((i_seq, final_choices[0]))
skipped = []
if (len(for_building) > 0):
make_sub_header("Adding %d ions to model" % len(for_building), out)
for k, (i_seq, final_choice) in enumerate(for_building):
atom = manager.pdb_atoms[i_seq]
skip = False
for other_i_seq, other_ion in for_building[:k]:
if (other_i_seq in skipped): continue
if (((other_ion.charge > 0) and (final_choice.charge > 0)) or
((other_ion.charge < 0) and (final_choice.charge < 0))):
other_atom = manager.pdb_atoms[other_i_seq]
dxyz = atom.distance(other_atom)
if (dxyz < params.max_distance_between_like_charges):
print(" %s (%s%+d) is only %.3fA from %s (%s%+d), skipping for now" %\
(atom.id_str(), final_choice.element, final_choice.charge, dxyz,
other_atom.id_str(), other_ion.element, other_ion.charge), file=out)
skipped.append(i_seq)
skip = True
break
if (skip): continue
print(" %s becomes %s%+d" % \
(atom.id_str(), final_choice.element, final_choice.charge), file=out)
refine_adp = params.refine_ion_adp
if (refine_adp == "Auto"):
if (fmodel.f_obs().d_min() <= 1.5):
refine_adp = "anisotropic"
elif (fmodel.f_obs().d_min() < 2.5):
atomic_number = sasaki.table(
final_choice.element).atomic_number()
if (atomic_number >= 19):
refine_adp = "anisotropic"
# Modify the atom object - this is clumsy but they will be grouped into
# a single chain at the end of refinement
initial_b_iso = params.initial_b_iso
if (initial_b_iso is Auto):
initial_b_iso = manager.guess_b_iso_real(i_seq)
element = final_choice.element
if (element == "IOD"): # FIXME
element = "I"
modified_atom = model.convert_atom(
i_seq=i_seq,
scattering_type=final_choice.scattering_type(),
atom_name=element,
element=element,
charge=final_choice.charge,
residue_name=final_choice.element,
initial_occupancy=params.initial_occupancy,
initial_b_iso=initial_b_iso,
chain_id=params.ion_chain_id,
segid="ION",
refine_adp=refine_adp,
refine_occupancies=False) #params.refine_ion_occupancies)
if (params.refine_anomalous) and (anomalous_flag):
scatterer = model.get_xray_structure().scatterers()[i_seq]
if (wavelength is not None):
fp_fdp_info = sasaki.table(
final_choice.element).at_angstrom(wavelength)
scatterer.fp = fp_fdp_info.fp()
scatterer.fdp = fp_fdp_info.fdp()
print(" setting f'=%g, f''=%g" %
(scatterer.fp, scatterer.fdp),
file=out)
group = xray.anomalous_scatterer_group(
iselection=flex.size_t([i_seq]),
f_prime=scatterer.fp,
f_double_prime=scatterer.fdp,
refine=["f_prime", "f_double_prime"],
selection_string=get_single_atom_selection_string(
modified_atom),
update_from_selection=True)
anomalous_groups.append(group)
modified_iselection.append(i_seq)
if (len(modified_iselection) > 0):
scatterers = model.get_xray_structure().scatterers()
# FIXME not sure this is actually working as desired...
site_symmetry_table = model.get_xray_structure().site_symmetry_table()
for i_seq in site_symmetry_table.special_position_indices():
scatterers[i_seq].site = crystal.correct_special_position(
crystal_symmetry=model.get_xray_structure(),
special_op=site_symmetry_table.get(i_seq).special_op(),
site_frac=scatterers[i_seq].site,
site_label=scatterers[i_seq].label,
tolerance=1.0)
model.get_xray_structure().replace_scatterers(scatterers=scatterers)
model.set_xray_structure(model.get_xray_structure())
def show_r_factors():
return "r_work=%6.4f r_free=%6.4f" % (fmodel.r_work(),
fmodel.r_free())
fmodel.update_xray_structure(xray_structure=model.get_xray_structure(),
update_f_calc=True,
update_f_mask=True)
n_anom = len(anomalous_groups)
refine_anomalous = anomalous_flag and params.refine_anomalous and n_anom > 0
refine_occupancies = (
(params.refine_ion_occupancies or refine_anomalous)
and ((not occupancy_strategy_enabled) or
(model.refinement_flags.s_occupancies is None) or
(len(model.refinement_flags.s_occupancies) == 0)))
if (refine_anomalous):
if (model.have_anomalous_scatterer_groups()
and (group_anomalous_strategy_enabled)):
model.set_anomalous_scatterer_groups(
model.get_anomalous_scatterer_groups() + anomalous_groups)
refine_anomalous = False
if (refine_occupancies) or (refine_anomalous):
print("", file=out)
print(" occupancy refinement (new ions only): start %s" % \
show_r_factors(), file=out)
fmodel.xray_structure.scatterers().flags_set_grads(state=False)
fmodel.xray_structure.scatterers().flags_set_grad_occupancy(
iselection=modified_iselection)
lbfgs_termination_params = scitbx.lbfgs.termination_parameters(
max_iterations=25)
minimized = mmtbx.refinement.minimization.lbfgs(
restraints_manager=None,
fmodels=fmodels,
model=model,
is_neutron_scat_table=False,
lbfgs_termination_params=lbfgs_termination_params)
fmodel.xray_structure.adjust_occupancy(
occ_max=1.0, occ_min=0, selection=modified_iselection)
zero_occ = []
for i_seq in modified_iselection:
occ = fmodel.xray_structure.scatterers()[i_seq].occupancy
if (occ == 0):
zero_occ.append(i_seq)
fmodel.update_xray_structure(update_f_calc=True,
update_f_mask=True)
print(" final %s" % \
show_r_factors(), file=out)
if (len(zero_occ) > 0):
print(" WARNING: occupancy dropped to zero for %d atoms:",
file=out)
atoms = model.get_atoms()
for i_seq in zero_occ:
print(" %s" % atoms[i_seq].id_str(suppress_segid=True),
file=out)
print("", file=out)
if (refine_anomalous):
assert fmodel.f_obs().anomalous_flag()
print(" anomalous refinement (new ions only): start %s" % \
show_r_factors(), file=out)
fmodel.update(target_name="ls")
anomalous_scatterer_groups.minimizer(fmodel=fmodel,
groups=anomalous_groups)
fmodel.update(target_name="ml")
print(" final %s" % \
show_r_factors(), file=out)
print("", file=out)
return manager
def refine_anomalous_substructure (
fmodel,
pdb_hierarchy,
wavelength=None,
map_type="anom_residual",
exclude_waters=False,
exclude_non_water_light_elements=True,
n_cycles_max=None,
map_sigma_min=3.0,
refine=("f_prime","f_double_prime"),
reset_water_u_iso=True,
use_all_anomalous=True,
verbose=True,
out=sys.stdout) :
"""
Crude mimic of Phaser's substructure completion, with two essential
differences: only the existing real scatterers in the input model will be
used (with the assumption that the model is already more or less complete),
and the anomalous refinement will be performed in Phenix, yielding both
f-prime and f-double-prime. The refined f-prime provides us with an
orthogonal estimate of the number of electrons missing from an incorrectly
labeled scatterer.
:param wavelength: X-ray wavelenth in Angstroms
:param exclude_waters: Don't refine anomalous scattering for water oxygens
:param exclude_non_water_light_elements: Don't refine anomalous scattering
for light atoms other than water (CHNO).
:param n_cycles_max: Maximum number of refinement cycles
:param map_sigma_min: Sigma cutoff for identify anomalous scatterers
:param reset_water_u_iso: Reset B-factors for water atoms prior to f'
refinement
:param use_all_anomalous: include any scatterers which are already modeled
as anomalous in the refinement
"""
from cctbx import xray
assert (fmodel.f_obs().anomalous_flag())
assert (map_type in ["llg", "anom_residual"])
make_sub_header("Iterative anomalous substructure refinement", out=out)
fmodel.update(target_name="ls")
pdb_atoms = pdb_hierarchy.atoms()
non_water_non_hd_selection = pdb_hierarchy.atom_selection_cache().selection(
"(not element H and not element D and not resname HOH)")
sites_frac = fmodel.xray_structure.sites_frac()
scatterers = fmodel.xray_structure.scatterers()
u_iso_mean = flex.mean(
fmodel.xray_structure.extract_u_iso_or_u_equiv().select(
non_water_non_hd_selection))
anomalous_iselection = flex.size_t()
anomalous_groups = []
t_start = time.time()
n_cycle = 0
while ((n_cycles_max is None) or (n_cycle < n_cycles_max)) :
n_cycle += 1
n_new_groups = 0
t_start_cycle = time.time()
print >> out, "Cycle %d" % n_cycle
anom_map = fmodel.map_coefficients(map_type=map_type).fft_map(
resolution_factor=0.25).apply_sigma_scaling().real_map_unpadded()
map_min = abs(flex.min(anom_map.as_1d()))
map_max = flex.max(anom_map.as_1d())
print >> out, " map range: -%.2f sigma to %.2f sigma" % (map_min, map_max)
reset_u_iso_selection = flex.size_t()
for i_seq, atom in enumerate(pdb_atoms) :
resname = atom.parent().resname
elem = atom.element.strip()
if ((i_seq in anomalous_iselection) or
((exclude_waters) and (resname == "HOH")) or
((elem in ["H","D","N","C","O"]) and (resname != "HOH") and
exclude_non_water_light_elements)) :
continue
scatterer = scatterers[i_seq]
site_frac = sites_frac[i_seq]
anom_map_value = anom_map.tricubic_interpolation(site_frac)
if ((anom_map_value >= map_sigma_min) or
((scatterer.fdp != 0) and use_all_anomalous)) :
if (verbose) :
if (n_new_groups == 0) :
print >> out, ""
print >> out, " new anomalous scatterers:"
print >> out, " %-34s map height: %6.2f sigma" % (atom.id_str(),
anom_map_value)
anomalous_iselection.append(i_seq)
selection_string = get_single_atom_selection_string(atom)
group = xray.anomalous_scatterer_group(
iselection=flex.size_t([i_seq]),
f_prime=0,
f_double_prime=0,
refine=list(refine),
selection_string=selection_string)
anomalous_groups.append(group)
n_new_groups += 1
if (resname == "HOH") and (reset_water_u_iso) :
water_u_iso = scatterer.u_iso
if (water_u_iso < u_iso_mean) :
reset_u_iso_selection.append(i_seq)
if (n_new_groups == 0) :
print >> out, ""
print >> out, "No new groups - anomalous scatterer search terminated."
break
elif (not verbose) :
print >> out, " %d new groups" % n_new_groups
for i_seq in anomalous_iselection :
sc = scatterers[i_seq]
sc.fp = 0
sc.fdp = 0
if (verbose) :
print >> out, ""
print >> out, "Anomalous refinement:"
fmodel.info().show_targets(text="before minimization", out=out)
print >> out, ""
u_iso = fmodel.xray_structure.extract_u_iso_or_u_equiv()
u_iso.set_selected(reset_u_iso_selection, u_iso_mean)
fmodel.xray_structure.set_u_iso(values=u_iso)
fmodel.update_xray_structure(update_f_calc=True)
minimizer(fmodel=fmodel, groups=anomalous_groups)
if (verbose) :
fmodel.info().show_targets(text="after minimization", out=out)
print >> out, ""
print >> out, " Refined sites:"
for i_seq, group in zip(anomalous_iselection, anomalous_groups) :
print >> out, " %-34s f' = %6.3f f'' = %6.3f" % (
pdb_atoms[i_seq].id_str(), group.f_prime, group.f_double_prime)
t_end_cycle = time.time()
print >> out, ""
if (verbose) :
print >> out, " time for this cycle: %.1fs" % (t_end_cycle-t_start_cycle)
fmodel.update(target_name="ml")
print >> out, "%d anomalous scatterer groups refined" % len(anomalous_groups)
t_end = time.time()
print >> out, "overall time: %.1fs" % (t_end - t_start)
return anomalous_groups
asg = xray.anomalous_scatterer_group(
iselection=flex.size_t([i_seq]),
f_prime=0,
f_double_prime=0,
refine=["f_prime","f_double_prime"],
selection_string=get_single_atom_selection_string(pdb_atoms[i_seq]))
groups.append(asg)
if (group_same_element) :
for elem in sorted(element_i_seqs.keys()) :
iselection = element_i_seqs[elem]
print >> out, \
" creating anomalous group for element %s with %d atoms" % \
(elem, len(iselection))
asg = xray.anomalous_scatterer_group(
iselection=iselection,
f_prime=0,
f_double_prime=0,
refine=["f_prime","f_double_prime"],
selection_string="element %s" % elem)
groups.append(asg)
return groups
def refine_anomalous_substructure (
fmodel,
pdb_hierarchy,
wavelength=None,
map_type="anom_residual",
exclude_waters=False,
exclude_non_water_light_elements=True,
n_cycles_max=None,
map_sigma_min=3.0,
refine=("f_prime","f_double_prime"),
