def compute_functional_and_gradients(O):
if (O.number_of_function_evaluations == 0):
O.number_of_function_evaluations += 1
return O.f_start, O.g_start
O.number_of_function_evaluations += 1
O.sites_cart_residue = flex.vec3_double(O.x)
rs_f = maptbx.real_space_target_simple(
unit_cell=O.unit_cell,
density_map=O.density_map,
sites_cart=O.sites_cart_residue,
selection=flex.bool(O.sites_cart_residue.size(),True))
O.real_space_target = rs_f
rs_g = maptbx.real_space_gradients_simple(
unit_cell=O.unit_cell,
density_map=O.density_map,
sites_cart=O.sites_cart_residue,
delta=O.real_space_gradients_delta,
selection=flex.bool(O.sites_cart_residue.size(),True))
O.rs_f = rs_f
rs_f *= -O.real_space_target_weight
rs_g *= -O.real_space_target_weight
if (O.geometry_restraints_manager is None):
f = rs_f
g = rs_g
else:
O.sites_cart_all.set_selected(O.residue_i_seqs, O.sites_cart_residue)
gr_e = O.geometry_restraints_manager.energies_sites(
sites_cart=O.sites_cart_all, compute_gradients=True)
f = rs_f + gr_e.target
g = rs_g + gr_e.gradients.select(indices=O.residue_i_seqs)
return f, g.as_double()
def _select(self, x, selection):
try: lx = len(x)
except Exception: lx = x.size()
if(lx == 0): return x
if(self.is_bool(x)):
x = x.select(selection)
elif(self.is_size_t(x[0])):
x_new = []
for i_seq, item in enumerate(x):
val = flex.bool(selection.size(), item).select(selection).iselection()
if(val.size() > 0): x_new.append(val)
x = x_new
elif(self.is_size_t(x[0][0])):
x_new = []
for item1 in x:
tmp = []
for item2 in item1:
if(self.is_size_t(item2)):
v=flex.bool(selection.size(),item2).select(selection).iselection()
if(v.size() > 0): tmp.append(v)
else: raise RuntimeError("Bad selection array type.")
if(len(tmp) > 0): x_new.append(tmp)
x = x_new
else: raise RuntimeError("Bad selection array type.")
return x
def add_new_solvent(self):
b_solv = self.params.b_iso
new_scatterers = flex.xray_scatterer(
self.sites.size(),
xray.scatterer(occupancy = self.params.occupancy,
b = b_solv,
scattering_type = self.params.scattering_type,
label = 'HOH'))
new_scatterers.set_sites(self.sites)
solvent_xray_structure = xray.structure(
special_position_settings = self.model.xray_structure,
scatterers = new_scatterers)
xrs_sol = self.model.xray_structure.select(self.model.solvent_selection())
xrs_mac = self.model.xray_structure.select(~self.model.solvent_selection())
xrs_sol = xrs_sol.concatenate(other = solvent_xray_structure)
sol_sel = flex.bool(xrs_mac.scatterers().size(), False)
sol_sel.extend( flex.bool(xrs_sol.scatterers().size(), True) )
self.model.add_solvent(
solvent_xray_structure = solvent_xray_structure,
residue_name = self.params.output_residue_name,
atom_name = self.params.output_atom_name,
chain_id = self.params.output_chain_id,
refine_occupancies = self.params.refine_occupancies,
refine_adp = self.params.new_solvent)
self.fmodel.update_xray_structure(
xray_structure = self.model.xray_structure,
update_f_calc = False)
def test_minimum_multiplicity_selection():
# some groups of indices with increasing multiplicity
hkl = flex.miller_index(
[(0,0,1),
(0,0,2), (0,0,2),
(0,0,3), (0,0,3), (0,0,3),
(0,0,4), (0,0,4), (0,0,4), (0,0,4),
(0,0,5), (0,0,5), (0,0,5), (0,0,5), (0,0,5)])
# test the various possible selections with multiplicity from 1 to 6
sel = minimum_multiplicity_selection(hkl, 1)
assert sel.count(True) == len(hkl)
sel = minimum_multiplicity_selection(hkl, 2)
assert (sel == flex.bool([False,] + [True] * 14)).all_eq(True)
sel = minimum_multiplicity_selection(hkl, 3)
assert (sel == flex.bool([False] * 3 + [True] * 12)).all_eq(True)
sel = minimum_multiplicity_selection(hkl, 4)
assert (sel == flex.bool([False] * 6 + [True] * 9)).all_eq(True)
sel = minimum_multiplicity_selection(hkl, 5)
assert (sel == flex.bool([False] * 10 + [True] * 5)).all_eq(True)
sel = minimum_multiplicity_selection(hkl, 6)
assert sel.count(False) == len(hkl)
print "OK"
def exercise_expand():
sg = sgtbx.space_group("P 41 (1,-1,0)")
h = flex.miller_index(((3,1,-2), (1,-2,0)))
assert tuple(sg.is_centric(h)) == (0, 1)
p1 = miller.expand_to_p1_iselection(
space_group=sg, anomalous_flag=False, indices=h, build_iselection=False)
p1_i0 = ((-3,-1,2), (-1, 3,2),(3,1,2),(1,-3,2),(1,-2, 0),(2,1,0))
assert tuple(p1.indices) == p1_i0
assert p1.iselection.size() == 0
p1 = miller.expand_to_p1_iselection(
space_group=sg, anomalous_flag=True, indices=h, build_iselection=False)
assert tuple(p1.indices) \
== ((3,1,-2), (1,-3,-2), (-3,-1,-2), (-1,3,-2),
(1,-2,0), (-2,-1,0), (-1,2,0), (2,1,0))
p1 = miller.expand_to_p1_iselection(
space_group=sg, anomalous_flag=False, indices=h, build_iselection=True)
assert tuple(p1.indices) == p1_i0
assert tuple(p1.iselection) == (0,0,0,0,1,1)
a = flex.double((1,2))
p = flex.double((10,90))
p1 = miller.expand_to_p1_phases(
space_group=sg, anomalous_flag=False, indices=h, data=p, deg=True)
assert approx_equal(tuple(p1.data), (-10,110,110,-10, 90,30))
p1 = miller.expand_to_p1_phases(
space_group=sg, anomalous_flag=True, indices=h, data=p, deg=True)
assert approx_equal(tuple(p1.data), (10,-110,-110,10, 90,-30,-90,30))
p = flex.double([x * math.pi/180 for x in p])
v = [x * math.pi/180 for x in p1.data]
p1 = miller.expand_to_p1_phases(
space_group=sg, anomalous_flag=True, indices=h, data=p, deg=False)
assert approx_equal(tuple(p1.data), v)
f = flex.polar(a, p)
p1 = miller.expand_to_p1_complex(
space_group=sg, anomalous_flag=True, indices=h, data=f)
assert approx_equal(tuple(flex.abs(p1.data)), (1,1,1,1,2,2,2,2))
assert approx_equal(tuple(flex.arg(p1.data)), v)
hl = flex.hendrickson_lattman([(1,2,3,4), (5,6,7,8)])
p1 = miller.expand_to_p1_hendrickson_lattman(
space_group=sg, anomalous_flag=True, indices=h, data=hl)
assert approx_equal(p1.data, [
[1,2,3,4],
[1.232051,-1.866025,-4.964102,0.5980762],
[1.232051,-1.866025,-4.964102,0.5980762],
[1,2,3,4],
[5,6,7,8],
[2.696152,-7.330127,-10.4282,2.062178],
[-5,-6,7,8],
[7.696152,-1.330127,3.428203,-10.06218]])
b = flex.bool([True,False])
p1 = miller.expand_to_p1_iselection(
space_group=sg, anomalous_flag=True, indices=h, build_iselection=True)
assert b.select(p1.iselection).all_eq(
flex.bool([True, True, True, True, False, False, False, False]))
i = flex.int([13,17])
p1 = miller.expand_to_p1_iselection(
space_group=sg, anomalous_flag=True, indices=h, build_iselection=True)
assert i.select(p1.iselection).all_eq(flex.int([13,13,13,13,17,17,17,17]))
#
assert approx_equal(miller.statistical_mean(sg, False, h, a), 4/3.)
assert approx_equal(miller.statistical_mean(sg, True, h, a), 3/2.)
def exclude_outliers_from_reference_restraints_selection(
pdb_hierarchy,
restraints_selection):
from mmtbx.validation.ramalyze import ramalyze
# the import below is SLOW!!!
from mmtbx.rotamer.rotamer_eval import RotamerEval
assert restraints_selection is not None
# ramachandran plot outliers
rama_outlier_selection = ramalyze(pdb_hierarchy=pdb_hierarchy,
outliers_only=False).outlier_selection()
rama_outlier_selection = flex.bool(restraints_selection.size(),
rama_outlier_selection)
# rotamer outliers
rota_outlier_selection = flex.size_t()
rotamer_manager = RotamerEval() # SLOW!!!
for model in pdb_hierarchy.models():
for chain in model.chains():
for residue_group in chain.residue_groups():
conformers = residue_group.conformers()
if(len(conformers)>1): continue
for conformer in residue_group.conformers():
residue = conformer.only_residue()
if(rotamer_manager.evaluate_residue(residue)=="OUTLIER"):
rota_outlier_selection.extend(residue.atoms().extract_i_seq())
rota_outlier_selection = flex.bool(restraints_selection.size(),
rota_outlier_selection)
outlier_selection = rama_outlier_selection | rota_outlier_selection
return restraints_selection & (~outlier_selection)
def rebatch(hklin, hklout, first_batch=None, include_range=None, exclude_range=None):
"""Need to implement: include batch range, exclude batches, add N to
batches, start batches at N."""
if include_range is None:
include_range = []
if exclude_range is None:
exclude_range = []
assert not (len(include_range) and len(exclude_range))
assert not (len(include_range) and first_batch)
assert not (len(exclude_range) and first_batch)
mtz_obj = mtz.object(file_name=hklin)
batch_column = None
batch_dataset = None
for crystal in mtz_obj.crystals():
for dataset in crystal.datasets():
for column in dataset.columns():
if column.label() == "BATCH":
batch_column = column
batch_dataset = dataset
if not batch_column:
raise RuntimeError, "no BATCH column found in %s" % hklin
batch_column_values = batch_column.extract_values(not_a_number_substitute=-1)
valid = flex.bool()
if exclude_range:
exclude_sel = flex.bool(batch_column_values.size(), False)
for (start, end) in exclude_range:
exclude_sel.set_selected((batch_column_values >= start) & (batch_column_values <= end), True)
mtz_obj.delete_reflections(exclude_sel.iselection())
elif include_range:
exclude_sel = flex.bool(batch_column_values.size(), True)
for (start, end) in include_range:
exclude_sel.set_selected((batch_column_values >= start) & (batch_column_values <= end), False)
mtz_obj.delete_reflections(exclude_sel.iselection())
# modify batch columns, and also the batch headers
elif first_batch is not None:
offset = first_batch - min(batch_column_values)
batch_column_values = batch_column_values + offset
for batch in mtz_obj.batches():
batch.set_num(int(batch.num() + offset))
# done modifying
batch_column.set_values(values=batch_column_values, selection_valid=valid)
# and write this lot out as hklout
mtz_obj.write(file_name=hklout)
def include_residue_selection(selection, residue_iselection): size = selection.size() selection_ = selection.deep_copy() selection_ = selection_.iselection() selection_.extend(residue_iselection) new_sel = flex.bool(size, selection_) rs = flex.bool(size, residue_iselection).select(new_sel).iselection() return new_sel, rs
def e_pot(O, sites_moved):
if ( O.last_sites_moved is None
or O.last_sites_moved.id() is not sites_moved.id()):
O.last_sites_moved = sites_moved
sites_cart = sites_moved
#
if (O.reduced_geo_manager is None):
flags = None
if (O.orca_experiments):
flags = cctbx.geometry_restraints.flags.flags(
nonbonded=False, default=True)
else:
# computing nonbonded interactions only with geo_manager,
# contributions from other restraints below with reduced_geo_manager
flags = cctbx.geometry_restraints.flags.flags(
nonbonded=True, default=False)
geo_energies = O.geo_manager.energies_sites(
sites_cart=sites_cart,
flags=flags,
custom_nonbonded_function=O.custom_nonbonded_function,
compute_gradients=True)
if (0): # XXX orca_experiments
print "geo_energies:"
geo_energies.show()
if (0): # XXX orca_experiments
O.geo_manager.show_sorted(site_labels=O.site_labels)
O.f = geo_energies.target
O.g = geo_energies.gradients
if (O.reduced_geo_manager is not None):
reduced_geo_energies = O.reduced_geo_manager.energies_sites(
sites_cart=sites_cart,
compute_gradients=True)
O.f += reduced_geo_energies.target
O.g += reduced_geo_energies.gradients
O.last_grms = group_args(geo=flex.mean_sq(O.g.as_double())**0.5)
#
if (O.density_map is not None):
rs_f = maptbx.real_space_target_simple(
unit_cell=O.geo_manager.crystal_symmetry.unit_cell(),
density_map=O.density_map,
sites_cart=sites_cart,
selection=flex.bool(sites_cart.size(),True))
rs_g = maptbx.real_space_gradients_simple(
unit_cell=O.geo_manager.crystal_symmetry.unit_cell(),
density_map=O.density_map,
sites_cart=sites_cart,
delta=O.real_space_gradients_delta,
selection=flex.bool(sites_cart.size(),True))
rs_f *= -O.real_space_target_weight
rs_g *= -O.real_space_target_weight
O.f += rs_f
O.g += rs_g
O.last_grms.real = flex.mean_sq(rs_g.as_double())**0.5
O.last_grms.real_or_xray = "real"
#
O.last_grms.total = flex.mean_sq(O.g.as_double())**0.5
return O.f
def inflate(self, sites_individual = None,
sites_torsion_angles = None,
sites_rigid_body = None,
adp_individual_iso = None,
adp_individual_aniso = None,
adp_group = None,
group_h = None,
adp_tls = None,
s_occupancies = None,
occupancies_group = None,
size_all = None):
# XXX group_anomalous selection should be added
if(sites_individual is not None and self.sites_individual is not None):
assert self.is_bool(sites_individual)
self.sites_individual.extend(sites_individual)
if( sites_torsion_angles is not None
and self.sites_torsion_angles is not None):
assert self.is_bool(sites_torsion_angles)
self.sites_torsion_angles.extend(sites_torsion_angles)
if(adp_individual_iso is not None):
assert self.is_bool(adp_individual_iso)
if(self.adp_individual_iso is None):
assert size_all is not None
self.adp_individual_iso = flex.bool(size_all, False)
# inflate existing iso flags if present
elif (self.adp_individual_iso.size() < size_all) :
n_new = size_all - self.adp_individual_iso.size()
self.adp_individual_iso.extend(flex.bool(n_new, False))
self.adp_individual_iso.extend(adp_individual_iso)
if(adp_individual_aniso is not None):
assert self.is_bool(adp_individual_aniso)
if(self.adp_individual_aniso is None):
assert size_all is not None
self.adp_individual_aniso = flex.bool(size_all, False)
# inflate existing aniso flags if present
elif (self.adp_individual_aniso.size() < size_all) :
n_new = size_all - self.adp_individual_aniso.size()
self.adp_individual_aniso.extend(flex.bool(n_new, False))
self.adp_individual_aniso.extend(adp_individual_aniso)
if(sites_rigid_body is not None):
assert hasattr(sites_rigid_body, 'count')
self.sites_rigid_body.extend(sites_rigid_body)
if(adp_group is not None):
assert hasattr(adp_group, 'count')
self.adp_group.extend(adp_group)
if(group_h is not None):
assert hasattr(group_h, 'count')
self.group_h.extend(group_h)
if(adp_tls is not None):
assert hasattr(adp_tls, 'count')
self.adp_tls.extend(adp_tls)
if(s_occupancies is not None):
assert hasattr(s_occupancies, 'count')
if(self.s_occupancies is not None):
self.s_occupancies.extend(s_occupancies)
self.check_all()
return self
def move_solvent_to_the_end_of_atom_list(self):
solsel = flex.bool(self.model.solvent_selection().count(False), False)
solsel.extend(flex.bool(self.model.solvent_selection().count(True), True))
xrs_sol = self.model.xray_structure.select(self.model.solvent_selection())
if(xrs_sol.hd_selection().count(True) == 0):
self.reset_solvent(
solvent_selection = solsel,
solvent_xray_structure = xrs_sol)
self.model.renumber_water()
self.fmodel.xray_structure = self.model.xray_structure
def __init__ (self, miller_array, settings, merge=None) :
self.miller_array = miller_array
self.settings = settings
self.merge_equivalents = merge
from cctbx import miller
from cctbx import crystal
from cctbx.array_family import flex
self.multiplicities = None
self.process_input_array()
array = self.work_array
uc = array.unit_cell()
self.unit_cell = uc
self.slice_selection = None
self.axis_index = None
if (settings.slice_mode) :
self.axis_index = ["h","k","l"].index(self.settings.slice_axis)
self.slice_selection = miller.simple_slice(
indices=array.indices(),
slice_axis=self.axis_index,
slice_index=settings.slice_index)
#if (self.slice_selection.count(True) == 0) :
#raise ValueError("No data selected!")
index_span = array.index_span()
self.d_min = array.d_min()
self.hkl_range = index_span.abs_range()
self.axes = [ uc.reciprocal_space_vector((self.hkl_range[0],0,0)),
uc.reciprocal_space_vector((0,self.hkl_range[1],0)),
uc.reciprocal_space_vector((0,0,self.hkl_range[2])) ]
self.generate_view_data()
if (self.slice_selection is not None) :
self.indices = self.work_array.indices().select(self.slice_selection)
self.data = self.data.select(self.slice_selection)
else :
self.indices = array.indices()
self.points = uc.reciprocal_space_vector(self.indices) * 100.
self.missing_flags = flex.bool(self.radii.size(), False)
self.sys_absent_flags = flex.bool(self.radii.size(), False)
if (settings.show_missing) :
self.generate_missing_reflections()
if (settings.show_systematic_absences) and (not settings.show_only_missing):
self.generate_systematic_absences()
# XXX hack for process_pick_points
self.visible_points = flex.bool(self.points.size(), True)
n_points = self.points.size()
assert (self.colors.size() == n_points)
assert (self.indices.size() == n_points)
assert (self.radii.size() == n_points)
assert (self.missing_flags.size() == n_points)
assert (self.sys_absent_flags.size() == n_points)
assert (self.data.size() == n_points)
self.clear_labels()
def exercise_1(mon_lib_srv, ener_lib):
pdb_in = simple_pdb()
params = pdb_interpretation.master_params.extract()
processed_pdb_file = pdb_interpretation.process(
mon_lib_srv=mon_lib_srv,
ener_lib=ener_lib,
params=params,
pdb_inp=pdb_in,
log=StringIO())
grm = processed_pdb_file.geometry_restraints_manager()
pdb_hierarchy = processed_pdb_file.all_chain_proxies.pdb_hierarchy
sites_cart = pdb_hierarchy.atoms().extract_xyz()
proxies = reference.add_coordinate_restraints(sites_cart=sites_cart)
assert proxies.size() == 29, "expected 29, got %d" % proxies.size()
import boost.python
ext = boost.python.import_ext("mmtbx_reference_coordinate_ext")
grads = flex.vec3_double(sites_cart.size(), (0.0,0.0,0.0))
residual = ext.reference_coordinate_residual_sum(
sites_cart=sites_cart,
proxies=proxies,
gradient_array=grads)
assert approx_equal(residual, 0.0)
#test selection
ca_selection = pdb_hierarchy.get_peptide_c_alpha_selection()
ca_sites_cart = sites_cart.select(ca_selection)
proxies = reference.add_coordinate_restraints(
sites_cart=ca_sites_cart,
selection=ca_selection)
assert proxies.size() == 3, "expected 3, got %d" % proxies.size()
tst_iselection = flex.size_t()
for atom in pdb_hierarchy.atoms():
if atom.name == " CA " or atom.name == " N ":
tst_iselection.append(atom.i_seq)
tst_sites_cart = sites_cart.select(tst_iselection)
proxies = reference.add_coordinate_restraints(
sites_cart=tst_sites_cart,
selection=tst_iselection)
assert proxies.size() == 6, "expected 6, got %d" % proxies.size()
#test remove
selection = flex.bool([False]*29)
proxies = proxies.proxy_remove(selection=selection)
assert proxies.size() == 6, "expected 6, got %d" % proxies.size()
proxies = proxies.proxy_remove(selection=ca_selection)
assert proxies.size() == 3, "expected 3, got %d" % proxies.size()
selection = flex.bool([True]*29)
proxies = proxies.proxy_remove(selection=selection)
assert proxies.size() == 0, "expected 0, got %d" % proxies.size()
def generate_torsion_restraints(
pdb_hierarchy,
sites_cart,
selection=None,
sigma=2.5,
limit=15.0,
chi_angles_only=False,
top_out_potential=False,
origin_id=2):
torsion_proxies = geometry_restraints.shared_dihedral_proxy()
if pdb_hierarchy.atoms().size() < 4:
return torsion_proxies
assert not pdb_hierarchy.atoms().extract_i_seq().all_eq(0)
bool_pdbh_selection = flex.bool(pdb_hierarchy.atoms_size(), False)
if (selection is not None):
if (isinstance(selection, flex.bool)):
bool_pdbh_selection = selection
elif (isinstance(selection, flex.size_t)):
bool_pdbh_selection.set_selected(selection, True)
if selection is None:
bool_pdbh_selection = flex.bool(pdb_hierarchy.atoms_size(), True)
actual_selection = bool_pdbh_selection.iselection()
assert len(sites_cart) == len(actual_selection)
weight = 1.0 / (sigma**2)
selection_to_sites_map = get_selection_to_sites_map(
sites_cart=sites_cart,
selection=actual_selection)
residue_torsions = collect_residue_torsion_angles(
pdb_hierarchy=pdb_hierarchy,
atom_selection=bool_pdbh_selection,
chi_angles_only=chi_angles_only)
for residue_info in residue_torsions:
for chi in residue_info.chis:
i_seqs = chi.i_seqs
sites = []
for i_seq in i_seqs:
sites.append(selection_to_sites_map[i_seq])
di = geometry_restraints.dihedral(
sites=sites, angle_ideal=0.0, weight=weight)
angle_ideal = di.angle_model
dp = geometry_restraints.dihedral_proxy(
i_seqs=i_seqs,
angle_ideal=angle_ideal,
weight=weight,
limit=limit,
top_out=top_out_potential,
origin_id=origin_id)
torsion_proxies.append(dp)
return torsion_proxies
def add_new_solvent(self):
if(self.params.b_iso is None):
sol_sel = self.model.solvent_selection()
xrs_mac_h = self.model.xray_structure.select(~sol_sel)
hd_mac = self.model.xray_structure.hd_selection().select(~sol_sel)
xrs_mac = xrs_mac_h.select(~hd_mac)
b = xrs_mac.extract_u_iso_or_u_equiv() * math.pi**2*8
b_solv = flex.mean_default(b, None)
if(b_solv is not None and b_solv < self.params.b_iso_min or
b_solv > self.params.b_iso_max):
b_solv = (self.params.b_iso_min + self.params.b_iso_max) / 2.
else:
b_solv = self.params.b_iso
if(self.params.new_solvent == "isotropic"):
new_scatterers = flex.xray_scatterer(
self.sites.size(),
xray.scatterer(occupancy = self.params.occupancy,
b = b_solv,
scattering_type = self.params.scattering_type))
elif(self.params.new_solvent == "anisotropic"):
u_star = adptbx.u_iso_as_u_star(self.model.xray_structure.unit_cell(),
adptbx.b_as_u(b_solv))
new_scatterers = flex.xray_scatterer(
self.sites.size(),
xray.scatterer(
occupancy = self.params.occupancy,
u = u_star,
scattering_type = self.params.scattering_type))
else: raise RuntimeError
new_scatterers.set_sites(self.sites)
solvent_xray_structure = xray.structure(
special_position_settings = self.model.xray_structure,
scatterers = new_scatterers)
xrs_sol = self.model.xray_structure.select(self.model.solvent_selection())
xrs_mac = self.model.xray_structure.select(~self.model.solvent_selection())
xrs_sol = xrs_sol.concatenate(other = solvent_xray_structure)
sol_sel = flex.bool(xrs_mac.scatterers().size(), False)
sol_sel.extend( flex.bool(xrs_sol.scatterers().size(), True) )
self.model.add_solvent(
solvent_xray_structure = solvent_xray_structure,
residue_name = self.params.output_residue_name,
atom_name = self.params.output_atom_name,
chain_id = self.params.output_chain_id,
refine_occupancies = self.params.refine_occupancies,
refine_adp = self.params.new_solvent)
self.fmodel.update_xray_structure(
xray_structure = self.model.xray_structure,
update_f_calc = True)
def tst_loglikelihoods():
fobs = flex.double( range(1000) )/200
fcalc = flex.double( [1]*1000 )
sigmas = flex.double( [0]*1000 )
epsilon = flex.double( [1]*1000 )
centric = flex.bool( [False]*1000 )
beta = flex.double( [1]*1000 )
alpha = flex.double( [0.99]*1000 )
tmp_object = scaling.likelihood_ratio_outlier_test(
fobs,
sigmas,
fcalc,
epsilon,
centric,
alpha,
beta)
cur_lik = tmp_object.log_likelihood()
pm_lik = tmp_object.posterior_mode_log_likelihood()
mode = tmp_object.posterior_mode()
level = 4.5
flags = tmp_object.flag_potential_outliers( 2.0*level )
for fl,pl,l,m,fo in zip(flags,pm_lik,cur_lik,mode,fobs):
if pl-l < level*2.0:
assert fl
else:
assert not fl
def apply_tls(xray_structure, params):
uc = xray_structure.unit_cell()
sg = xray_structure.space_group()
selections_1d = flex.bool(xray_structure.scatterers().size(),True)
selections = [selections_1d.iselection()]
T=random_aniso_adp(space_group=sg, unit_cell=uc, u_scale=params.max_tl,
u_min=params.min_tl)
L=random_aniso_adp(space_group=sg, unit_cell=uc, u_scale=params.max_tl,
u_min=params.min_tl)
print " T: %s"%",".join([("%7.3f"%i).strip() for i in T])
print " L: %s"%",".join([("%7.3f"%i).strip() for i in L])
tlsos = mmtbx.tls.tools.generate_tlsos(
selections = selections,
xray_structure = xray_structure,
T=[T],
L=[L],
S=[[0,0,0,0,0,0,0,0,0]])
u_cart_from_tls = mmtbx.tls.tools.u_cart_from_tls(
sites_cart = xray_structure.sites_cart(),
selections = selections,
tlsos = tlsos)
xray_structure.convert_to_anisotropic()
u_cart = xray_structure.scatterers().extract_u_cart(uc)
utot = u_cart_from_tls+u_cart
xray_structure.set_u_cart(u_cart=utot, selection = selections_1d.iselection())
xray_structure.tidy_us()
return xray_structure
def run():
pdb_file_name = libtbx.env.find_in_repositories(
relative_path="phenix_regression/pdb/adp_restraints.pdb", test=os.path.isfile)
processed_pdb_file = pdb_interpretation.process(
mon_lib_srv = monomer_library.server.server(),
ener_lib = monomer_library.server.ener_lib(),
file_name = pdb_file_name)
xray_structure = processed_pdb_file.xray_structure()
grm = processed_pdb_file.geometry_restraints_manager(
plain_pairs_radius = 5.0)
grm.pair_proxies(sites_cart=xray_structure.sites_cart())
class parameters:
sphere_radius = 1.6
distance_power = 0.0
average_power = 0.0
min_u_sum = 1.e-6
sel = flex.bool(xray_structure.scatterers().size(), True)
xray_structure.scatterers().flags_set_grad_u_iso(sel.iselection())
for use_hd in [True, False]:
energies_adp = cctbx.adp_restraints.energies_iso(
geometry_restraints_manager=grm,
xray_structure=xray_structure,
use_hd = use_hd,
use_u_local_only = False,
parameters=parameters)
u_iso_restraints = grm.harmonic_restraints(
variables = xray_structure.extract_u_iso_or_u_equiv(),
type_indices = None,
type_weights = 1.0)
assert approx_equal(u_iso_restraints.residual_sum, energies_adp.residual_sum)
assert approx_equal(u_iso_restraints.gradients, energies_adp.gradients)
def compute_functional_and_gradients(O):
if (O.number_of_function_evaluations == 0):
O.number_of_function_evaluations += 1
return O.f_start, O.g_start
O.number_of_function_evaluations += 1
O.residue_tardy_model.unpack_q(q_packed=O.x)
O.sites_cart_residue = O.residue_tardy_model.sites_moved()
rs_f = maptbx.real_space_target_simple(
unit_cell=O.unit_cell,
density_map=O.density_map,
sites_cart=O.sites_cart_residue,
selection=flex.bool(O.sites_cart_residue.size(),True))
rs_g = real_space_rigid_body_gradients_simple(
unit_cell=O.unit_cell,
density_map=O.density_map,
sites_cart_0=O.sites_cart_residue_0,
center_of_mass=O.residue_center_of_mass,
q=O.x)
O.rs_f = rs_f
rs_f *= -O.real_space_target_weight
rs_g *= -O.real_space_target_weight
if (O.geometry_restraints_manager is None):
f = rs_f
g = rs_g
else:
O.sites_cart_all.set_selected(O.residue_i_seqs, O.sites_cart_residue)
gr_e = O.geometry_restraints_manager.energies_sites(
sites_cart=O.sites_cart_all, compute_gradients=True)
O.__d_e_pot_d_sites = gr_e.gradients.select(indices=O.residue_i_seqs)
f = rs_f + gr_e.target
g = rs_g + O.residue_tardy_model.d_e_pot_d_q_packed()
return f, g.as_double()
def add_coordinate_restraints(
sites_cart,
selection=None,
sigma=0.5,
limit=1.0,
top_out_potential=False):
import boost.python
ext_rcp = boost.python.import_ext("mmtbx_reference_coordinate_ext")
result = ext_rcp.shared_reference_coordinate_proxy()
if (selection is not None):
if (isinstance(selection, flex.bool)):
selection = selection.iselection()
if selection is None:
selection = flex.bool(
len(sites_cart),
True).iselection()
assert len(sites_cart) == len(selection)
weight = 1.0 / (sigma**2)
for k, i_seq in enumerate(selection):
i_seqs = [i_seq]
ref_sites = sites_cart[k]
proxy = ext_rcp.reference_coordinate_proxy(
i_seqs=i_seqs,
ref_sites=ref_sites,
weight=weight,
limit=limit,
top_out=top_out_potential)
result.append(proxy)
return result
def set_refinable_parameters(xray_structure, parameters, selections,
enforce_positivity=False):
# XXX PVA: Code below is terribly inefficient and MUST be moved into C++
sz = xray_structure.scatterers().size()
i = 0
for sel in selections:
# pre-check for positivity begin
# spread negative occupancies across i_seqs having positive ones
par_all = flex.double()
par_neg = flex.double()
i_p = i
for sel_ in sel:
p = parameters[i_p]
par_all.append(p)
if(p<0): par_neg.append(p)
i_p += 1
if(enforce_positivity and par_neg.size()>0):
par_all = par_all - flex.min(par_all)
fs = flex.sum(par_all)
if(fs != 0):
par_all = par_all / fs
# pre-check for positivity end
for j, sel_ in enumerate(sel):
sel__b = flex.bool(sz, flex.size_t(sel_))
xray_structure.set_occupancies(par_all[j], sel__b)
i+=1
def merge_obs(indices, iobs, sel, symm, anomalous_flag, d_min, d_max):
#indices = flex.miller_index(indices)
#iobs = flex.double(iobs)
if sel is not None and len(sel) > 0:
sel = flex.bool(sel)
indices = indices.select(sel)
iobs = iobs.select(sel)
miller_set = miller.set(crystal_symmetry=symm,
indices=indices,
anomalous_flag=anomalous_flag)
array = miller.array(miller_set=miller_set,
data=iobs,
).set_observation_type_xray_intensity()
array = array.resolution_filter(d_min=d_min, d_max=d_max)
## New way
#array = array.map_to_asu()
#
#merger = yamtbx_dataproc_crystfel_ext.merge_equivalents_crystfel()
#merger.add_observations(array.indices(), array.data())
#merger.merge()
return array.merge_equivalents(algorithm="crystfel") # if sigmas is None, merge_equivalents_real() is used which simply averages.
def compute_functional_and_gradients(self):
if (self.number_of_function_evaluations == 0):
self.number_of_function_evaluations += 1
return self.f_start, self.g_start
self.number_of_function_evaluations += 1
self.residue_tardy_model.unpack_q(q_packed=self.x)
self.sites_cart_residue = self.residue_tardy_model.sites_moved()
if(self.states_collector is not None):
self.states_collector.add(sites_cart = self.sites_cart_residue)
rs_f = maptbx.real_space_target_simple(
unit_cell=self.unit_cell,
density_map=self.density_map,
sites_cart=self.sites_cart_residue,
selection=flex.bool(self.sites_cart_residue.size(),True))
rs_g = real_space_rigid_body_gradients_simple(
unit_cell=self.unit_cell,
density_map=self.density_map,
sites_cart_0=self.sites_cart_residue_0,
center_of_mass=self.residue_center_of_mass,
q=self.x)
self.rs_f = rs_f
rs_f *= -self.real_space_target_weight
rs_g *= -self.real_space_target_weight
if (self.geometry_restraints_manager is None):
f = rs_f
g = rs_g
else:
gr_e = self.geometry_restraints_manager.energies_sites(
sites_cart=self.sites_cart_residue,
flags = self.cctbx_geometry_restraints_flags,
compute_gradients=True)
self.__d_e_pot_d_sites = gr_e.gradients
f = rs_f + gr_e.target
g = rs_g + self.residue_tardy_model.d_e_pot_d_q_packed()
return f, g.as_double()
def read_frames_updated_detail(self):
from cctbx.crystal_orientation import crystal_orientation
from xfel.cxi.util import is_odd_numbered
frames = {'frame_id': flex.int(),
'wavelength': flex.double(),
'cc': flex.double(),
'G': flex.double(),
'BFACTOR': flex.double(),
'RS': flex.double(),
'odd_numbered': flex.bool(),
'thetax': flex.double(),
'thetay': flex.double(),
'orientation': [],
'unit_cell': [],
'unique_file_name': []}
stream = open(self.params.output.prefix + '_frame.db', 'r')
for row in stream:
items = row.split()
CO = crystal_orientation([float(t) for t in items[8:17]], True)
unique_file_name = eval(items[19])
frames['frame_id'].append(int(items[0]))
frames['wavelength'].append(float(items[1]))
frames['cc'].append(float(items[20]))
frames['G'].append(float(items[5]))
frames['BFACTOR'].append(float(items[6]))
frames['RS'].append(float(items[7]))
frames['thetax'].append(float(items[17]))
frames['thetay'].append(float(items[18]))
frames['odd_numbered'].append(is_odd_numbered(unique_file_name))
frames['orientation'].append(CO)
frames['unit_cell'].append(CO.unit_cell())
frames['unique_file_name'].append(unique_file_name)
stream.close()
return frames
def exercise_singular_least_squares():
obs = flex.double([1.234])
weights_2345 = flex.double([2.345])
weights_zero = flex.double([0])
r_free_flags = flex.bool([False])
a = flex.double([0])
b = flex.double([0])
for obs_type in ["F", "I"]:
for weights,scale_factor in [
(weights_2345, 3.456),
(weights_zero, 0)]:
tg = ext.targets_least_squares(
compute_scale_using_all_data=False,
obs_type=obs_type,
obs=obs,
weights=weights,
r_free_flags=r_free_flags,
f_calc=flex.complex_double(a, b),
derivatives_depth=2,
scale_factor=scale_factor)
if (weights is weights_2345):
assert approx_equal(tg.scale_factor(), scale_factor)
assert list(tg.gradients_work()) == [0j]
assert list(tg.hessians_work()) == [(1,1,1)]
else:
assert tg.scale_factor() is None
assert tg.target_work() is None
assert tg.target_test() is None
assert tg.gradients_work().size() == 0
assert tg.hessians_work().size() == 0
def _compute_gradients(self):
return -1.*maptbx.real_space_gradients_simple(
unit_cell = self.unit_cell,
density_map = self.target_map,
sites_cart = self.sites_cart,
delta = self.real_space_gradients_delta,
selection = flex.bool(self.sites_cart.size(), True))
def plot_samples(O, stage):
p = O.params.plot_samples
if (stage not in p.stages):
return
if (p.ix is not None):
O.plot_samples_ix(stage, p.ix)
elif (p.ix_auto == "all"):
for ix in xrange(O.x.size()):
O.plot_samples_ix(stage, ix)
elif (p.ix_auto == "random"):
assert p.ix_random.samples_each_scattering_type is not None
assert p.ix_random.samples_each_scattering_type > 0
assert p.ix_random.random_seed is not None
mt = flex.mersenne_twister(seed=p.ix_random.random_seed)
i_seqs_grouped = O.xray_structure.scatterers() \
.extract_scattering_types().i_seqs_by_value().values()
i_seqs_selected = flex.bool(O.x.size(), False)
for i_seqs in i_seqs_grouped:
ps = i_seqs.size()
ss = min(ps, p.ix_random.samples_each_scattering_type)
isel = mt.random_selection(population_size=ps, sample_size=ss)
i_seqs_selected.set_selected(i_seqs.select(isel), True)
for ix,(i_sc,_) in enumerate(O.x_info):
if (i_seqs_selected[i_sc]):
O.plot_samples_ix(stage, ix)
else:
raise RuntimeError("Unknown plot_samples.ix_auto = %s" % p.ix_auto)
def local_nikonov_scaling(self,out):
print >> out
print >> out, "Nikonev based local scaling"
print >> out, "Maximum depth : %8i"%(self.max_depth)
print >> out, "Target neighbours : %8i"%(self.target_neighbours)
print >> out, "neighbourhood sphere : %8i"%(self.sphere)
print >> out
if self.der_primset.is_xray_intensity_array():
raise Sorry(" For Nikonev target in local scaling, amplitudes must be used")
assert (False)
self.local_scaler = scaling.local_scaling_nikonov(
hkl_master=self.master_set.indices(),
hkl_sets=self.nat_primset.indices(),
data_set_a=self.nat_primset.data(),
data_set_b=self.der_primset.data(),
epsilons=self.der_primset.epsilons().data().as_double(),
centric=flex.bool(self.der_primset.centric_flags().data()),
threshold=self.threshold,
space_group=self.nat_primset.space_group(),
anomalous_flag=self.nat_primset.anomalous_flag(),
radius=self.sphere,
depth=self.max_depth,
target_ref=self.target_neighbours)
def exercise_miller_array_data_types():
miller_set = crystal.symmetry(unit_cell=(10, 10, 10, 90, 90, 90), space_group_symbol="P1").miller_set(
indices=flex.miller_index([(1, 2, 3), (4, 5, 6)]), anomalous_flag=False
)
for data in [
flex.bool([False, True]),
flex.int([0, 1]),
flex.size_t([0, 1]),
flex.double([0, 1]),
flex.complex_double([0, 1]),
]:
miller_array = miller_set.array(data=data)
if op.isfile("tmp_iotbx_mtz.mtz"):
os.remove("tmp_iotbx_mtz.mtz")
assert not op.isfile("tmp_iotbx_mtz.mtz")
miller_array.as_mtz_dataset(column_root_label="DATA").mtz_object().write(file_name="tmp_iotbx_mtz.mtz")
assert op.isfile("tmp_iotbx_mtz.mtz")
mtz_obj = mtz.object(file_name="tmp_iotbx_mtz.mtz")
miller_arrays_read_back = mtz_obj.as_miller_arrays()
assert len(miller_arrays_read_back) == 1
miller_array_read_back = miller_arrays_read_back[0]
assert miller_array_read_back.indices().all_eq(miller_array.indices())
if miller_array.is_integer_array() or miller_array.is_bool_array():
assert miller_array_read_back.data().all_eq(flex.int([0, 1]))
elif miller_array.is_real_array():
assert miller_array_read_back.data().all_eq(flex.double([0, 1]))
elif miller_array.is_complex_array():
assert miller_array_read_back.data().all_eq(flex.complex_double([0, 1]))
else:
raise RuntimeError("Programming error.")
def suggest_likely_candidates( self, acceptable_violations = 1e+90 ):
used = flex.bool( len(self.sg_choices), False )
order = []
all_done = False
count = -1
if (len(self.tuple_score) == 0) :
return []
tmp_scores = []
for tt in self.tuple_score:
tmp_scores.append( tt[0] )
order = flex.sort_permutation( flex.double( tmp_scores ), False )
sorted_rows = []
max_score = flex.min( flex.double( tmp_scores ) )
for ii in order:
sg = self.sg_choices[ii]
tmp_n = self.n[ii]
tmp_violations = self.violations[ii]
tmp_mean_i = self.mean_i[ii]
tmp_mean_isigi = self.mean_isigi[ii]
tuple_score = self.tuple_score[ii]
sorted_rows.append( [str(sg), '%i'%(tmp_n),
'%8.2f '%(tmp_mean_i),
'%8.2f '%(tmp_mean_isigi),
' %i '%(tuple_score[1]),
' %i '%(tuple_score[2]),
' %8.3e '%((tuple_score[0]-max_score))
])
return sorted_rows
def process_input_array(self, arr):
array = arr.deep_copy()
work_array = arr
multiplicities = None
try:
if self.merge_equivalents :
array, multiplicities, merge = MergeData(array, self.settings.show_anomalous_pairs)
settings = self.settings
data = array.data()
#import code, traceback; code.interact(local=locals(), banner="".join( traceback.format_stack(limit=10) ) )
self.missing_set = oop.null()
#if (array.is_xray_intensity_array()):
# data.set_selected(data < 0, flex.double(data.size(), 0.))
if (array.is_unique_set_under_symmetry()) and (settings.map_to_asu):
array = array.map_to_asu()
if (multiplicities is not None):
multiplicities = multiplicities.map_to_asu()
if (settings.d_min is not None):
array = array.resolution_filter(d_min=settings.d_min)
if (multiplicities is not None):
multiplicities = multiplicities.resolution_filter(
d_min=settings.d_min)
self.filtered_array = array.deep_copy()
if (settings.expand_anomalous):
if not array.is_unique_set_under_symmetry():
raise Sorry("Error! Cannot generate bijvoet mates of unmerged reflections.")
array = array.generate_bijvoet_mates()
original_symmetry = array.crystal_symmetry()
if (multiplicities is not None):
multiplicities = multiplicities.generate_bijvoet_mates()
if (self.settings.show_missing):
self.missing_set = array.complete_set().lone_set(array)
if self.settings.show_anomalous_pairs:
self.missing_set = self.missing_set.select(
self.missing_set.centric_flags().data(), negate=True)
if (settings.expand_to_p1):
if not array.is_unique_set_under_symmetry():
raise Sorry("Error! Cannot expand unmerged reflections to P1.")
original_symmetry = array.crystal_symmetry()
array = array.expand_to_p1().customized_copy(
crystal_symmetry=original_symmetry)
#array = array.niggli_cell().expand_to_p1()
#self.missing_set = self.missing_set.niggli_cell().expand_to_p1()
self.missing_set = self.missing_set.expand_to_p1().customized_copy(
crystal_symmetry=original_symmetry)
if (multiplicities is not None):
multiplicities = multiplicities.expand_to_p1().customized_copy(
crystal_symmetry=original_symmetry)
data = array.data()
self.r_free_mode = False
self.phases = flex.double(data.size(), float('nan'))
self.radians = flex.double(data.size(), float('nan'))
self.ampl = flex.double(data.size(), float('nan'))
self.sigmas = None
if isinstance(data, flex.bool):
self.r_free_mode = True
data_as_float = flex.double(data.size(), 0.0)
data_as_float.set_selected(data==True, flex.double(data.size(), 1.0))
data = data_as_float
self.data = data #.deep_copy()
else :
if isinstance(data, flex.double):
self.data = data #.deep_copy()
elif isinstance(data, flex.complex_double):
self.data = data #.deep_copy()
self.ampl = flex.abs(data)
self.phases = flex.arg(data) * 180.0/math.pi
# purge nan values from array to avoid crash in fmod_positive()
b = flex.bool([bool(math.isnan(e)) for e in self.phases])
# replace the nan values with an arbitrary float value
self.phases = self.phases.set_selected(b, 42.4242)
# Cast negative degrees to equivalent positive degrees
self.phases = flex.fmod_positive(self.phases, 360.0)
self.radians = flex.arg(data)
# replace the nan values with an arbitrary float value
self.radians = self.radians.set_selected(b, 0.424242)
elif hasattr(array.data(), "as_double"):
self.data = array.data().as_double()
else:
raise RuntimeError("Unexpected data type: %r" % data)
if (settings.show_data_over_sigma):
if (array.sigmas() is None):
raise Sorry("sigmas not defined.")
sigmas = array.sigmas()
non_zero_sel = sigmas != 0
array = array.select(non_zero_sel)
array = array.customized_copy(data=array.data()/array.sigmas())
self.data = array.data()
if (multiplicities is not None):
multiplicities = multiplicities.select(non_zero_sel)
if array.sigmas() is not None:
self.sigmas = array.sigmas()
else:
self.sigmas = None
work_array = array
except Exception as e:
print(to_str(e) + "".join(traceback.format_stack(limit=10)))
raise e
return None, None
work_array.set_info(arr.info() )
multiplicities = multiplicities
return work_array, multiplicities
def generate_view_data(self):
from scitbx.array_family import flex
from scitbx import graphics_utils
settings = self.settings
data_for_colors = data_for_radii = None
if not self.fullprocessarray:
return
data = self.data #self.work_array.data()
sigmas = self.sigmas
if (isinstance(data, flex.double) and data.all_eq(0)):
data = flex.double(data.size(), 1)
if ((self.multiplicities is not None) and
(settings.scale_colors_multiplicity)):
data_for_colors = self.multiplicities.data().as_double()
assert data_for_colors.size() == data.size()
elif (settings.sqrt_scale_colors) and (isinstance(data, flex.double)):
data_for_colors = flex.sqrt(flex.abs(data))
elif isinstance(data, flex.complex_double):
data_for_colors = self.radians
foms_for_colours = self.foms
self.colourlabel = self.miller_array.info().labels[1]
elif (settings.sigma_color) and sigmas is not None:
data_for_colors = sigmas.as_double()
self.colourlabel = self.miller_array.info().labels[1]
else :
data_for_colors = flex.abs(data.deep_copy())
uc = self.work_array.unit_cell()
self.min_dist = min(uc.reciprocal_space_vector((1,1,1))) * self.renderscale
min_radius = 0.05 * self.min_dist
max_radius = 0.45 * self.min_dist
if ((self.multiplicities is not None) and
(settings.scale_radii_multiplicity)):
data_for_radii = self.multiplicities.data().as_double()
if (settings.sigma_radius) and sigmas is not None:
data_for_radii = sigmas * self.multiplicities.as_double()
assert data_for_radii.size() == data.size()
elif (settings.sigma_radius) and sigmas is not None:
data_for_radii = sigmas.as_double()
else :
data_for_radii = nth_power_scale(flex.abs(data.deep_copy()),
settings.nth_power_scale_radii)
if (settings.slice_mode):
data = data.select(self.slice_selection)
if (not settings.keep_constant_scale):
data_for_radii = data_for_radii.select(self.slice_selection)
data_for_colors = data_for_colors.select(self.slice_selection)
foms_for_colours = foms_for_colours.select(self.slice_selection)
if isinstance(data, flex.complex_double):
if self.isUsingFOMs():
colors = graphics_utils.colour_by_phi_FOM(data_for_colors, foms_for_colours)
else:
colors = graphics_utils.colour_by_phi_FOM(data_for_colors, None)
elif (settings.color_scheme in ["rainbow", "heatmap", "redblue"]):
colors = graphics_utils.color_by_property(
properties=data_for_colors,
selection=flex.bool(data_for_colors.size(), True),
color_all=False,
gradient_type=settings.color_scheme)
elif (settings.color_scheme == "grayscale"):
colors = graphics_utils.grayscale_by_property(
properties=data_for_colors,
selection=flex.bool(data_for_colors.size(), True),
shade_all=False,
invert=settings.black_background)
else :
if (settings.black_background):
base_color = (1.0,1.0,1.0)
else :
base_color = (0.0,0.0,0.0)
colors = flex.vec3_double(data_for_colors.size(), base_color)
if (settings.slice_mode) and (settings.keep_constant_scale):
colors = colors.select(self.slice_selection)
data_for_radii = data_for_radii.select(self.slice_selection)
#if (settings.sqrt_scale_radii) and (not settings.scale_radii_multiplicity):
# data_for_radii = flex.sqrt(flex.abs(data_for_radii))
if len(data_for_radii):
dat2 = flex.abs(flex.double([e for e in data_for_radii if not math.isnan(e)]))
# don't divide by 0 if dealing with selection of Rfree array where all values happen to be zero
scale = max_radius/(flex.max(dat2) + 0.001)
radii = data_for_radii * (self.settings.scale * scale)
assert radii.size() == colors.size()
else:
radii = flex.double()
max_radius = 0
self.radii = radii
self.max_radius = max_radius
self.min_radius = min_radius
self.colors = colors
if isinstance(data, flex.complex_double):
self.foms = foms_for_colours
def get_data_from_xac(params, xac):
if xac.endswith(".pkl"):
tmp = pickle.load(open(xac))
else:
tmp = xds_ascii.XDS_ASCII(xac)
sel_remove = flex.bool(tmp.iobs.size(), False)
if params.min_peak is not None:
sel = tmp.peak < params.min_peak
sel_remove |= sel
elif params.min_peak_percentile is not None:
q = numpy.percentile(tmp.peak, params.min_peak_percentile)
print "percentile %.2f %s" % (q, xac)
sel = tmp.peak < q
sel_remove |= sel
if params.skip_rejected: sel_remove |= (tmp.sigma_iobs <= 0)
if params.dmin is not None:
sel_remove |= ~tmp.as_miller_set().resolution_filter_selection(
d_min=params.dmin)
if params.correct_peak:
sel_remove |= (tmp.peak < 1) # remove PEAK==0
# Remove selected
print "DEBUG:: removing %d reflections" % sel_remove.count(
True) #sum(sel_remove)#
tmp.remove_selection(sel_remove)
if not params.skip_rejected: tmp.sigma_iobs = flex.abs(tmp.sigma_iobs)
# Correct I,sigI if needed
if params.correct_peak:
tmp.iobs *= tmp.peak * .01
tmp.sigma_iobs *= tmp.peak * .01
if params.cancel_rlp:
tmp.iobs /= tmp.rlp
tmp.sigma_iobs /= tmp.rlp
if params.polarization.correct:
# Only works with single-panel detector!!
# Assumes detector fast = (1,0,0), slow = (0,1,0)
sin_sq_2theta = tmp.symm.unit_cell().sin_sq_two_theta(
tmp.indices, tmp.wavelength)
cos_sq_2theta = 1. - sin_sq_2theta
sin_theta = tmp.wavelength / tmp.symm.unit_cell().d(
tmp.indices) / 2.
Eppi = numpy.cross(params.polarization.plane_normal,
params.polarization.incident_beam_direction)
Eppi /= numpy.linalg.norm(Eppi)
S = flex.vec3_double(
tmp.xd - tmp.orgx, tmp.yd - tmp.orgy,
flex.double(tmp.xd.size(), tmp.distance / tmp.qx))
S /= S.norms()
zp = S.dot(Eppi.tolist()) * 2. * sin_theta
cosrho = zp / flex.sqrt(sin_sq_2theta)
P0 = 0.5 * (1. + cos_sq_2theta)
PP = (params.polarization.fraction - 0.5) * (2. * cosrho**2 -
1.) * sin_sq_2theta
P = P0 - PP
# Apply correction
tmp.iobs /= P
tmp.sigma_iobs /= P
if 0: # debug
for x, y, p in zip(tmp.xd, tmp.yd, P):
print "pdebug:: %.2f %.2f %.4e" % (x, y, p)
return tmp.i_obs().customized_copy(anomalous_flag=params.anomalous_flag,
space_group_info=sgtbx.space_group_info(
params.space_group))
def get_data_from_dials(params, files):
from dials.util.options import Importer, flatten_reflections, flatten_experiments
importer = Importer(files, read_experiments=True, read_reflections=True)
reflections = flatten_reflections(importer.reflections)
experiments = flatten_experiments(importer.experiments)
assert len(reflections) == len(experiments) == 1
xs = crystal.symmetry(experiments[0].crystal.get_unit_cell(),
space_group=experiments[0].crystal.get_space_group())
tmp = reflections[0].select(reflections[0]["id"] >= 0)
assert max(tmp["id"]) == 0
if params.dials_data == "sum":
assert "intensity.sum.value" in tmp
assert "intensity.sum.variance" in tmp
else:
assert "intensity.prf.value" in tmp
assert "intensity.prf.variance" in tmp
intensity_key = "intensity.sum.value" if params.dials_data == "sum" else "intensity.prf.value"
variance_key = "intensity.sum.variance" if params.dials_data == "sum" else "intensity.prf.variance"
sel_remove = flex.bool(tmp.size(), False)
if params.min_peak is not None:
sel = tmp["partiality"] < params.min_peak / 100.
sel_remove |= sel
elif params.min_peak_percentile is not None:
q = numpy.percentile(tmp["partiality"], params.min_peak_percentile)
print "percentile %.2f %s" % (q * 100., xac)
sel = tmp["partiality"] < q
sel_remove |= sel
if params.skip_rejected: sel_remove |= tmp[variance_key] <= 0
if params.dmin is not None:
sel_remove |= xs.unit_cell().d(tmp["miller_index"]) < params.dmin
if params.correct_peak:
sel_remove |= (tmp["partiality"] < .01) # remove PEAK==0
# Remove selected
print "DEBUG:: removing %d reflections" % sel_remove.count(
True) #sum(sel_remove)#
tmp = tmp.select(~sel_remove)
ret = miller.array(miller.set(xs, tmp["miller_index"],
params.anomalous_flag),
data=tmp[intensity_key],
sigmas=flex.sqrt(flex.abs(tmp[variance_key])))
scale = flex.double(ret.size(), 1.)
if not params.cancel_rlp and "lp" in tmp: scale *= tmp["lp"]
if "dqe" in tmp: scale /= tmp["dqe"]
if params.correct_peak: scale *= tmp["partiality"]
ret = ret.apply_scaling(factor=scale)
if params.cancel_rlp and params.polarization.correct:
raise "Not implemented"
return ret
def exercise_proxy_show():
if sys.platform.startswith("win") and sys.version_info[:2] < (2,6):
# This appears to be a windows-specific bug with string formatting
# for python versions prior to 2.6, where the exponent is printed
# with 3 digits rather than 2.
print "Skipping exercise_proxy_show()"
return
sites_cart = flex.vec3_double((
(-3.1739,10.8317,7.5653),(-2.5419,9.7567,6.6306),
(-3.3369,8.8794,4.5191),(-3.4640,9.9882,5.3896)))
site_labels = ("C1", "C2", "O16", "N8")
u_cart = flex.sym_mat3_double((
(0.0153,0.0206,0.0234,0.0035,-0.0052,-0.0051),
(0.0185,0.0109,0.0206,0.0005,-0.0010,0.0002),
(0.0295,0.0203,0.0218,-0.0010,-0.0003,-0.0044),
(0.0159,0.0154,0.0206,-0.0003,0.0004,0.0036)))
u_iso = flex.double((-1,-1,-1,-1))
use_u_aniso = flex.bool((True,True,True,True))
#
proxies = adp_restraints.shared_adp_similarity_proxy()
sio = StringIO()
proxies.show_sorted(
by_value="residual",
u_cart=flex.sym_mat3_double(),
u_iso=flex.double(),
use_u_aniso=flex.bool(),
f=sio)
assert not show_diff(sio.getvalue(), """\
ADP similarity restraints: 0
""")
proxies = adp_restraints.shared_adp_similarity_proxy([
adp_restraints.adp_similarity_proxy(i_seqs=[0,1],weight=25),
adp_restraints.adp_similarity_proxy(i_seqs=[2,3],weight=0.3)])
sio = StringIO()
proxies.show_sorted(
by_value="residual",
u_cart=u_cart,
u_iso=u_iso,
use_u_aniso=use_u_aniso,
f=sio,
prefix=":")
assert not show_diff(sio.getvalue(), """\
:ADP similarity restraints: 2
:Sorted by residual:
:scatterers 0
: 1
: delta sigma weight rms_deltas residual
: U11 -3.20e-03 2.00e-01 2.50e+01 4.96e-03 5.54e-03
: U22 9.70e-03 2.00e-01 2.50e+01
: U33 2.80e-03 2.00e-01 2.50e+01
: U12 3.00e-03 2.00e-01 2.50e+01
: U13 -4.20e-03 2.00e-01 2.50e+01
: U23 -5.30e-03 2.00e-01 2.50e+01
:scatterers 2
: 3
: delta sigma weight rms_deltas residual
: U11 1.36e-02 1.83e+00 3.00e-01 6.15e-03 1.02e-04
: U22 4.90e-03 1.83e+00 3.00e-01
: U33 1.20e-03 1.83e+00 3.00e-01
: U12 -7.00e-04 1.83e+00 3.00e-01
: U13 -7.00e-04 1.83e+00 3.00e-01
: U23 -8.00e-03 1.83e+00 3.00e-01
""")
sio = StringIO()
proxies.show_sorted(
by_value="rms_deltas",
site_labels=site_labels,
u_cart=u_cart,
u_iso=flex.double((0.024,0.031,0.021,0.028)),
use_u_aniso=flex.bool((False,False,False,False)),
f=sio,
prefix="=")
assert not show_diff(sio.getvalue(), """\
=ADP similarity restraints: 2
=Sorted by rms_deltas:
=scatterers C1
= C2
= delta sigma weight residual
= Uiso -7.00e-03 2.00e-01 2.50e+01 1.22e-03
=scatterers O16
= N8
= delta sigma weight residual
= Uiso -7.00e-03 1.83e+00 3.00e-01 1.47e-05
""")
#
proxies = adp_restraints.shared_isotropic_adp_proxy()
sio = StringIO()
proxies.show_sorted(
by_value="residual",
u_cart=flex.sym_mat3_double(),
u_iso=u_iso,
use_u_aniso=use_u_aniso,
f=sio)
assert not show_diff(sio.getvalue(), """\
Isotropic ADP restraints: 0
""")
proxies = adp_restraints.shared_isotropic_adp_proxy([
adp_restraints.isotropic_adp_proxy(i_seqs=(0,),weight=25),
adp_restraints.isotropic_adp_proxy(i_seqs=(2,),weight=0.3)])
sio = StringIO()
proxies.show_sorted(
by_value="residual",
site_labels=site_labels,
u_cart=u_cart,
u_iso=u_iso,
use_u_aniso=use_u_aniso,
f=sio,
prefix=" ")
assert not show_diff(sio.getvalue(), """\
Isotropic ADP restraints: 2
Sorted by residual:
scatterer C1
delta sigma weight rms_deltas residual
U11 -4.47e-03 2.00e-01 2.50e+01 4.27e-03 4.11e-03
U22 8.33e-04 2.00e-01 2.50e+01
U33 3.63e-03 2.00e-01 2.50e+01
U12 3.50e-03 2.00e-01 2.50e+01
U13 -5.20e-03 2.00e-01 2.50e+01
U23 -5.10e-03 2.00e-01 2.50e+01
scatterer O16
delta sigma weight rms_deltas residual
U11 5.63e-03 1.83e+00 3.00e-01 3.16e-03 2.69e-05
U22 -3.57e-03 1.83e+00 3.00e-01
U33 -2.07e-03 1.83e+00 3.00e-01
U12 -1.00e-03 1.83e+00 3.00e-01
U13 -3.00e-04 1.83e+00 3.00e-01
U23 -4.40e-03 1.83e+00 3.00e-01
""")
sio = StringIO()
proxies.show_sorted(
by_value="rms_deltas",
u_cart=u_cart,
u_iso=u_iso,
use_u_aniso=use_u_aniso,
f=sio,
prefix="$")
assert not show_diff(sio.getvalue(), """\
$Isotropic ADP restraints: 2
$Sorted by rms_deltas:
$scatterer 0
$ delta sigma weight rms_deltas residual
$ U11 -4.47e-03 2.00e-01 2.50e+01 4.27e-03 4.11e-03
$ U22 8.33e-04 2.00e-01 2.50e+01
$ U33 3.63e-03 2.00e-01 2.50e+01
$ U12 3.50e-03 2.00e-01 2.50e+01
$ U13 -5.20e-03 2.00e-01 2.50e+01
$ U23 -5.10e-03 2.00e-01 2.50e+01
$scatterer 2
$ delta sigma weight rms_deltas residual
$ U11 5.63e-03 1.83e+00 3.00e-01 3.16e-03 2.69e-05
$ U22 -3.57e-03 1.83e+00 3.00e-01
$ U33 -2.07e-03 1.83e+00 3.00e-01
$ U12 -1.00e-03 1.83e+00 3.00e-01
$ U13 -3.00e-04 1.83e+00 3.00e-01
$ U23 -4.40e-03 1.83e+00 3.00e-01
""")
#
proxies = adp_restraints.shared_rigid_bond_proxy()
sio = StringIO()
proxies.show_sorted(
by_value="residual",
sites_cart=flex.vec3_double(),
u_cart=flex.sym_mat3_double(),
f=sio)
assert not show_diff(sio.getvalue(), """\
Rigid bond restraints: 0
""")
proxies = adp_restraints.shared_rigid_bond_proxy([
adp_restraints.rigid_bond_proxy(i_seqs=(0,1),weight=25),
adp_restraints.rigid_bond_proxy(i_seqs=(0,2),weight=15),
adp_restraints.rigid_bond_proxy(i_seqs=(2,3),weight=25),
adp_restraints.rigid_bond_proxy(i_seqs=(3,1),weight=30)])
sio = StringIO()
proxies.show_sorted(
by_value="residual",
sites_cart=sites_cart,
site_labels=site_labels,
u_cart=u_cart,
f=sio,
prefix="*")
assert not show_diff(sio.getvalue(), """\
*Rigid bond restraints: 4
*Sorted by residual:
*scatterers O16
* N8
* delta_z sigma weight residual
* -3.96e-03 2.00e-01 2.50e+01 3.92e-04
*scatterers C1
* C2
* delta_z sigma weight residual
* 1.08e-03 2.00e-01 2.50e+01 2.89e-05
*scatterers C1
* O16
* delta_z sigma weight residual
* 4.03e-04 2.58e-01 1.50e+01 2.44e-06
*scatterers N8
* C2
* delta_z sigma weight residual
* -1.54e-04 1.83e-01 3.00e+01 7.16e-07
""")
sio = StringIO()
proxies.show_sorted(
by_value="delta",
sites_cart=sites_cart,
u_cart=u_cart,
f=sio,
prefix="||",
max_items=2)
assert not show_diff(sio.getvalue(), """\
||Rigid bond restraints: 4
||Sorted by delta:
||scatterers 2
|| 3
|| delta_z sigma weight residual
|| -3.96e-03 2.00e-01 2.50e+01 3.92e-04
||scatterers 0
|| 1
|| delta_z sigma weight residual
|| 1.08e-03 2.00e-01 2.50e+01 2.89e-05
||... (remaining 2 not shown)
""")
def test_2():
n_sites = 1000
d_min = 2.0
volume_per_atom = 50
fraction_missing = (0.0, )
scale = 5.0
# create dummy model
space_group_info = sgtbx.space_group_info("P212121")
structure = random_structure.xray_structure(
space_group_info=space_group_info,
elements=["N"] * (n_sites),
volume_per_atom=volume_per_atom,
random_u_iso=False)
structure.scattering_type_registry(table="wk1995")
f_calc = structure.structure_factors(d_min=d_min,
anomalous_flag=False,
algorithm="direct").f_calc()
f_obs = abs(f_calc)
for fm in fraction_missing:
# partial model
n_keep = int(round(structure.scatterers().size() * (1 - fm)))
partial_structure = xray.structure(special_position_settings=structure)
partial_structure.add_scatterers(structure.scatterers()[:n_keep])
# fcalc (partial model), fobs (fcalc full model)
f_calc_partial = partial_structure.structure_factors(
d_min=d_min, anomalous_flag=False, algorithm="direct").f_calc()
f_calc = abs(f_calc_partial)
# define test set reflections
flags = flex.bool(f_calc_partial.indices().size(), False)
k = 0
for i in xrange(f_calc_partial.indices().size()):
k = k + 1
if (k != 10):
flags[i] = False
else:
k = 0
flags[i] = True
# *********************************************************TEST = 1
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=f_obs,
f_calc=f_calc,
free_reflections_per_bin=f_obs.data().size(),
flags=flags,
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=f_obs,
f_calc=f_calc,
free_reflections_per_bin=f_obs.data().size(),
flags=flags,
interpolation=True,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
# *********************************************************TEST = 2
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flags,
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flags,
interpolation=True,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
# *********************************************************TEST = 3
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flex.bool(f_obs.data().size(), True),
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flex.bool(f_obs.data().size(), True),
interpolation=True,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
# *********************************************************TEST = 4
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flex.bool(f_obs.data().size(), False),
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=miller.array(miller_set=f_obs, data=f_obs.data() * scale),
f_calc=f_calc,
free_reflections_per_bin=200,
flags=flex.bool(f_obs.data().size(), False),
interpolation=True,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.size() == beta.size()
assert alpha.size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.max(alpha.data()), 1.0 / scale, 1.e-2)
assert approx_equal(flex.min(beta.data()), 0.0, 1.e-2)
assert approx_equal(flex.max(beta.data()), 0.0, 1.e-2)
def __init__(self, miller_array, settings, merge=None, foms_array=None,
fullprocessarray=True):
self.miller_array = miller_array
self.renderscale = 100.0
self.foms_workarray = foms_array
self.SceneCreated = False
self.settings = settings
self.merge_equivalents = False
if not self.miller_array.is_unique_set_under_symmetry():
self.merge_equivalents = merge
from cctbx import crystal
from cctbx.array_family import flex
self.multiplicities = None
self.fomlabel = ""
self.foms = flex.double(self.miller_array.size(), float('nan'))
self._is_using_foms = False
self.fullprocessarray = fullprocessarray
if self.miller_array.is_complex_array():
# Colour map coefficient as a circular rainbow with saturation as a function of FOMs
# process the foms miller array and store the foms data for later use when computing colours
if foms_array:
assert ( self.miller_array.size() == foms_array.size() )
self.foms_workarray, dummy = self.process_input_array(foms_array)
if not self.foms_workarray:
return
self.foms = self.foms_workarray.data()
self.fomlabel = foms_array.info().label_string()
self._is_using_foms = True
self.work_array, self.multiplicities = self.process_input_array(self.miller_array)
if not self.work_array:
return
array = self.work_array
uc = array.unit_cell()
self.unit_cell = uc
self.slice_selection = None
self.axis_index = None
if (settings.slice_mode):
self.axis_index = ["h","k","l"].index(self.settings.slice_axis)
self.slice_selection = miller.simple_slice(
indices=array.indices(),
slice_axis=self.axis_index,
slice_index=settings.slice_index)
#if (self.slice_selection.count(True) == 0):
#raise ValueError("No data selected!")
index_span = array.index_span()
self.colourlabel = self.miller_array.info().labels[0]
self.d_min = array.d_min()
self.min_dist = 0.0
self.hkl_range = index_span.abs_range()
self.axes = [ uc.reciprocal_space_vector((self.hkl_range[0],0,0)),
uc.reciprocal_space_vector((0,self.hkl_range[1],0)),
uc.reciprocal_space_vector((0,0,self.hkl_range[2])) ]
self.generate_view_data()
if (self.slice_selection is not None):
self.indices = self.work_array.indices().select(self.slice_selection).deep_copy()
self.data = self.data.select(self.slice_selection)
self.phases = self.phases.select(self.slice_selection)
self.radians = self.radians.select(self.slice_selection)
self.ampl = self.ampl.select(self.slice_selection)
if self.sigmas:
self.sigmas = self.sigmas.select(self.slice_selection)
if foms_array:
self.foms = self.foms.select(self.slice_selection)
else :
self.indices = array.indices()
self.points = uc.reciprocal_space_vector(self.indices) * self.renderscale
n_points = self.points.size()
if not fullprocessarray:
self.radii = flex.double()
self.radii = ExtendAnyData(self.radii, n_points)
self.colors = flex.vec3_double()
self.colors = ExtendAnyData(self.colors, n_points)
self.missing_flags = flex.bool(self.radii.size(), False)
self.sys_absent_flags = flex.bool(self.radii.size(), False)
if (settings.show_missing):
self.generate_missing_reflections()
if (settings.show_systematic_absences) and (not settings.show_only_missing):
self.generate_systematic_absences()
n_points = self.points.size()
assert (self.colors.size() == n_points)
assert (self.indices.size() == n_points)
assert (self.radii.size() == n_points)
assert (self.missing_flags.size() == n_points)
assert (self.sys_absent_flags.size() == n_points)
assert (self.data.size() == n_points)
assert (self.phases.size() == n_points)
assert (self.radians.size() == n_points)
assert (self.ampl.size() == n_points)
if self.sigmas:
assert (self.sigmas.size() == n_points)
if foms_array:
assert (self.foms.size() == n_points)
else:
self.foms = flex.double(n_points, float('nan'))
self.dres = uc.d(self.indices )
self.clear_labels()
self.SceneCreated = True
def run(args, log=sys.stdout, as_gui_program=False):
if (len(args) == 0):
parsed = defaults(log=log)
parsed.show(prefix=" ", out=log)
return
command_line = (option_parser().enable_symmetry_comprehensive().option(
"-q",
"--quiet",
action="store_true",
default=False,
help="suppress output").option("--output_plots",
action="store_true",
default=False)).process(args=args)
parsed = defaults(log=log)
processed_args = mmtbx.utils.process_command_line_args(
args=command_line.args,
cmd_cs=command_line.symmetry,
master_params=parsed,
log=log,
suppress_symmetry_related_errors=True)
processed_args.params.show(out=log)
params = processed_args.params.extract().density_modification
output_plots = command_line.options.output_plots
crystal_symmetry = crystal.symmetry(
unit_cell=params.input.unit_cell,
space_group_info=params.input.space_group)
reflection_files = {}
for rfn in (params.input.reflection_data.file_name,
params.input.experimental_phases.file_name,
params.input.map_coefficients.file_name):
if os.path.isfile(str(rfn)) and rfn not in reflection_files:
reflection_files.setdefault(
rfn,
iotbx.reflection_file_reader.any_reflection_file(
file_name=rfn, ensure_read_access=False))
server = iotbx.reflection_file_utils.reflection_file_server(
crystal_symmetry=crystal_symmetry,
reflection_files=reflection_files.values())
fo = mmtbx.utils.determine_data_and_flags(
server,
parameters=params.input.reflection_data,
extract_r_free_flags=False,
log=log).f_obs
hl_coeffs = mmtbx.utils.determine_experimental_phases(
server,
params.input.experimental_phases,
log=log,
parameter_scope="",
working_point_group=None,
symmetry_safety_check=True,
ignore_all_zeros=True)
if params.input.map_coefficients.file_name is not None:
map_coeffs = server.get_phases_deg(
file_name=params.input.map_coefficients.file_name,
labels=params.input.map_coefficients.labels,
convert_to_phases_if_necessary=False,
original_phase_units=None,
parameter_scope="",
parameter_name="labels").map_to_asu()
else:
map_coeffs = None
ncs_object = None
if params.input.ncs_file_name is not None:
ncs_object = ncs.ncs()
ncs_object.read_ncs(params.input.ncs_file_name)
ncs_object.display_all(log=log)
fo = fo.map_to_asu()
hl_coeffs = hl_coeffs.map_to_asu()
fo = fo.eliminate_sys_absent().average_bijvoet_mates()
hl_coeffs = hl_coeffs.eliminate_sys_absent().average_bijvoet_mates()
model_map = None
model_map_coeffs = None
if len(processed_args.pdb_file_names):
pdb_file = mmtbx.utils.pdb_file(
pdb_file_names=processed_args.pdb_file_names)
xs = pdb_file.pdb_inp.xray_structure_simple()
fo_, hl_ = fo, hl_coeffs
if params.change_basis_to_niggli_cell:
change_of_basis_op = xs.change_of_basis_op_to_niggli_cell()
xs = xs.change_basis(change_of_basis_op)
fo_ = fo_.change_basis(change_of_basis_op).map_to_asu()
hl_ = hl_.change_basis(change_of_basis_op).map_to_asu()
#fo_, hl_ = fo_.common_sets(hl_)
fmodel_refined = mmtbx.utils.fmodel_simple(
f_obs=fo_,
scattering_table=
"wk1995", #XXX pva: 1) neutrons? 2) move up as a parameter.
xray_structures=[xs],
bulk_solvent_correction=True,
anisotropic_scaling=True,
r_free_flags=fo_.array(data=flex.bool(fo_.size(), False)))
fmodel_refined.update(abcd=hl_)
master_phil = mmtbx.maps.map_and_map_coeff_master_params()
map_params = master_phil.fetch(
iotbx.phil.parse("""\
map_coefficients {
map_type = 2mFo-DFc
isotropize = True
}
""")).extract().map_coefficients[0]
model_map_coeffs = mmtbx.maps.map_coefficients_from_fmodel(
fmodel=fmodel_refined, params=map_params)
model_map = model_map_coeffs.fft_map(
resolution_factor=params.grid_resolution_factor).real_map_unpadded(
)
import time
t0 = time.time()
dm = density_modify(params,
fo,
hl_coeffs,
ncs_object=ncs_object,
map_coeffs=map_coeffs,
model_map_coeffs=model_map_coeffs,
log=log,
as_gui_program=as_gui_program)
time_dm = time.time() - t0
print >> log, "Time taken for density modification: %.2fs" % time_dm
# run cns
if 0:
from cctbx.development import cns_density_modification
cns_result = cns_density_modification.run(params, fo, hl_coeffs)
print cns_result.modified_map.all()
print dm.map.all()
dm_map_coeffs = dm.map_coeffs_in_original_setting
from cctbx import maptbx, miller
crystal_gridding = maptbx.crystal_gridding(
dm_map_coeffs.unit_cell(),
space_group_info=dm_map_coeffs.space_group().info(),
pre_determined_n_real=cns_result.modified_map.all())
dm_map = miller.fft_map(crystal_gridding,
dm_map_coeffs).apply_sigma_scaling()
corr = flex.linear_correlation(cns_result.modified_map.as_1d(),
dm_map.real_map_unpadded().as_1d())
print "CNS dm/mmtbx dm correlation:"
corr.show_summary()
if dm.model_map_coeffs is not None:
model_map = miller.fft_map(
crystal_gridding,
dm.miller_array_in_original_setting(
dm.model_map_coeffs)).apply_sigma_scaling()
corr = flex.linear_correlation(
cns_result.modified_map.as_1d(),
model_map.real_map_unpadded().as_1d())
print "CNS dm/model correlation:"
corr.show_summary()
if output_plots:
plots_to_make = (
"fom",
"skewness",
"r1_factor",
"r1_factor_fom",
"mean_solvent_density",
"mean_protein_density",
"f000_over_v",
"k_flip",
"rms_solvent_density",
"rms_protein_density",
"standard_deviation_local_rms",
"mean_delta_phi",
"mean_delta_phi_initial",
)
from matplotlib.backends.backend_pdf import PdfPages
from libtbx import pyplot
stats = dm.get_stats()
pdf = PdfPages("density_modification.pdf")
if len(dm.correlation_coeffs) > 1:
if 0:
start_coeffs, model_coeffs = dm.map_coeffs_start.common_sets(
model_map_coeffs)
model_phases = model_coeffs.phases(deg=True).data()
exptl_phases = nearest_phase(
model_phases,
start_coeffs.phases(deg=True).data(),
deg=True)
corr = flex.linear_correlation(exptl_phases, model_phases)
corr.show_summary()
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title("phases start")
ax.set_xlabel("Experimental phases")
ax.set_ylabel("Phases from refined model")
ax.scatter(exptl_phases, model_phases, marker="x", s=10)
pdf.savefig(fig)
#
dm_coeffs, model_coeffs = dm.map_coeffs.common_sets(
model_map_coeffs)
model_phases = model_coeffs.phases(deg=True).data()
dm_phases = nearest_phase(model_phases,
dm_coeffs.phases(deg=True).data(),
deg=True)
corr = flex.linear_correlation(dm_phases, model_phases)
corr.show_summary()
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title("phases dm")
ax.set_xlabel("Phases from density modification")
ax.set_ylabel("Phases from refined model")
ax.scatter(dm_phases, model_phases, marker="x", s=10)
pdf.savefig(fig)
#
data = dm.correlation_coeffs
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title("correlation coefficient")
ax.plot(range(1, dm.i_cycle + 2), data)
pdf.savefig(fig)
#
data = dm.mean_phase_errors
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title("Mean effective phase errors")
ax.plot(range(1, dm.i_cycle + 2), data)
pdf.savefig(fig)
for plot in plots_to_make:
data = [
getattr(stats.get_cycle_stats(i), plot)
for i in range(1, dm.i_cycle + 2)
]
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title(plot.replace("_", " "))
ax.plot(range(1, dm.i_cycle + 2), data)
pdf.savefig(fig)
data = [
stats.get_cycle_stats(i).rms_solvent_density /
stats.get_cycle_stats(i).rms_protein_density
for i in range(1, dm.i_cycle + 2)
]
fig = pyplot.figure()
ax = fig.add_subplot(1, 1, 1)
ax.set_title("RMS solvent/protein density ratio")
ax.plot(range(1, dm.i_cycle + 2), data)
pdf.savefig(fig)
pdf.close()
dm_map_coeffs = dm.map_coeffs_in_original_setting
dm_hl_coeffs = dm.hl_coeffs_in_original_setting
# output map if requested
map_params = params.output.map
if map_params.file_name is not None:
fft_map = dm_map_coeffs.fft_map(
resolution_factor=params.grid_resolution_factor)
if map_params.scale == "sigma":
fft_map.apply_sigma_scaling()
else:
fft_map.apply_volume_scaling()
gridding_first = gridding_last = None
title_lines = []
if map_params.format == "xplor":
fft_map.as_xplor_map(file_name=map_params.file_name,
title_lines=title_lines,
gridding_first=gridding_first,
gridding_last=gridding_last)
else:
fft_map.as_ccp4_map(file_name=map_params.file_name,
gridding_first=gridding_first,
gridding_last=gridding_last,
labels=title_lines)
# output map coefficients if requested
mtz_params = params.output.mtz
# Decide if we are going to actually write the mtz
if mtz_params.file_name is not None:
orig_fom, final_fom = dm.start_and_end_fom()
if mtz_params.skip_output_if_worse and final_fom < orig_fom:
ok_to_write_mtz = False
print "Not writing out mtz. Final FOM (%7.3f) worse than start (%7.3f)" % (
final_fom, orig_fom)
else: # usual
ok_to_write_mtz = True
else:
ok_to_write_mtz = True
if mtz_params.file_name is not None and ok_to_write_mtz:
label_decorator = iotbx.mtz.ccp4_label_decorator()
fo = dm.miller_array_in_original_setting(
dm.f_obs_complete).common_set(dm_map_coeffs)
mtz_dataset = fo.as_mtz_dataset(column_root_label="F",
label_decorator=label_decorator)
mtz_dataset.add_miller_array(dm_map_coeffs,
column_root_label="FWT",
label_decorator=label_decorator)
phase_source = dm.miller_array_in_original_setting(
dm.phase_source).common_set(dm_map_coeffs)
mtz_dataset.add_miller_array(
phase_source.array(data=flex.abs(phase_source.data())),
column_root_label="FOM",
column_types='W',
label_decorator=label_decorator)
mtz_dataset.add_miller_array(
phase_source.array(data=phase_source.phases(deg=True).data()),
column_root_label="PHIB",
column_types='P',
label_decorator=None)
if mtz_params.output_hendrickson_lattman_coefficients:
mtz_dataset.add_miller_array(dm_hl_coeffs,
column_root_label="HL",
label_decorator=label_decorator)
mtz_dataset.mtz_object().write(mtz_params.file_name)
return result(map_file=map_params.file_name,
mtz_file=mtz_params.file_name,
stats=dm.get_stats())
def exercise_expand():
sg = sgtbx.space_group("P 41 (1,-1,0)")
h = flex.miller_index(((3, 1, -2), (1, -2, 0)))
assert tuple(sg.is_centric(h)) == (0, 1)
p1 = miller.expand_to_p1_iselection(space_group=sg,
anomalous_flag=False,
indices=h,
build_iselection=False)
p1_i0 = ((-3, -1, 2), (-1, 3, 2), (3, 1, 2), (1, -3, 2), (1, -2, 0), (2, 1,
0))
assert tuple(p1.indices) == p1_i0
assert p1.iselection.size() == 0
p1 = miller.expand_to_p1_iselection(space_group=sg,
anomalous_flag=True,
indices=h,
build_iselection=False)
assert tuple(p1.indices) \
== ((3,1,-2), (1,-3,-2), (-3,-1,-2), (-1,3,-2),
(1,-2,0), (-2,-1,0), (-1,2,0), (2,1,0))
p1 = miller.expand_to_p1_iselection(space_group=sg,
anomalous_flag=False,
indices=h,
build_iselection=True)
assert tuple(p1.indices) == p1_i0
assert tuple(p1.iselection) == (0, 0, 0, 0, 1, 1)
a = flex.double((1, 2))
p = flex.double((10, 90))
p1 = miller.expand_to_p1_phases(space_group=sg,
anomalous_flag=False,
indices=h,
data=p,
deg=True)
assert approx_equal(tuple(p1.data), (-10, 110, 110, -10, 90, 30))
p1 = miller.expand_to_p1_phases(space_group=sg,
anomalous_flag=True,
indices=h,
data=p,
deg=True)
assert approx_equal(tuple(p1.data), (10, -110, -110, 10, 90, -30, -90, 30))
p = flex.double([x * math.pi / 180 for x in p])
v = [x * math.pi / 180 for x in p1.data]
p1 = miller.expand_to_p1_phases(space_group=sg,
anomalous_flag=True,
indices=h,
data=p,
deg=False)
assert approx_equal(tuple(p1.data), v)
f = flex.polar(a, p)
p1 = miller.expand_to_p1_complex(space_group=sg,
anomalous_flag=True,
indices=h,
data=f)
assert approx_equal(tuple(flex.abs(p1.data)), (1, 1, 1, 1, 2, 2, 2, 2))
assert approx_equal(tuple(flex.arg(p1.data)), v)
hl = flex.hendrickson_lattman([(1, 2, 3, 4), (5, 6, 7, 8)])
p1 = miller.expand_to_p1_hendrickson_lattman(space_group=sg,
anomalous_flag=True,
indices=h,
data=hl)
assert approx_equal(
p1.data, [[1, 2, 3, 4], [1.232051, -1.866025, -4.964102, 0.5980762],
[1.232051, -1.866025, -4.964102, 0.5980762], [1, 2, 3, 4],
[5, 6, 7, 8], [2.696152, -7.330127, -10.4282, 2.062178],
[-5, -6, 7, 8], [7.696152, -1.330127, 3.428203, -10.06218]])
b = flex.bool([True, False])
p1 = miller.expand_to_p1_iselection(space_group=sg,
anomalous_flag=True,
indices=h,
build_iselection=True)
assert b.select(p1.iselection).all_eq(
flex.bool([True, True, True, True, False, False, False, False]))
i = flex.int([13, 17])
p1 = miller.expand_to_p1_iselection(space_group=sg,
anomalous_flag=True,
indices=h,
build_iselection=True)
assert i.select(p1.iselection).all_eq(
flex.int([13, 13, 13, 13, 17, 17, 17, 17]))
#
assert approx_equal(miller.statistical_mean(sg, False, h, a), 4 / 3.)
assert approx_equal(miller.statistical_mean(sg, True, h, a), 3 / 2.)
def substitute_ss(
model, # changed in place
params=None,
log=null_out(),
reference_map=None,
verbose=False):
"""
Substitute secondary structure elements in real_h hierarchy with ideal
ones _in_place_.
Returns reference torsion proxies - the only thing that cannot be restored
with little effort outside the procedure.
"""
ss_annotation = model.get_ss_annotation()
t0 = time()
if model.get_hierarchy().models_size() > 1:
raise Sorry("Multi model files are not supported")
for m in model.get_hierarchy().models():
for chain in m.chains():
if len(chain.conformers()) > 1:
raise Sorry("Alternative conformations are not supported.")
processed_params = process_params(params)
if not processed_params.enabled:
return None
if ss_annotation is None:
return None
ann = ss_annotation
if model.ncs_constraints_present():
print("Using master NCS to reduce amount of work", file=log)
expected_n_hbonds = 0
for h in ann.helices:
expected_n_hbonds += h.get_n_maximum_hbonds()
edited_h = model.get_hierarchy().deep_copy()
n_atoms_in_real_h = model.get_number_of_atoms()
selection_cache = model.get_atom_selection_cache()
# check the annotation for correctness (atoms are actually in hierarchy)
error_msg = "The following secondary structure annotations result in \n"
error_msg += "empty atom selections. They don't match the structre: \n"
t1 = time()
# Checking for SS selections
deleted_annotations = ann.remove_empty_annotations(
hierarchy=model.get_hierarchy(), asc=selection_cache)
if not deleted_annotations.is_empty():
if processed_params.skip_empty_ss_elements:
if len(deleted_annotations.helices) > 0:
print("Removing the following helices because there are",
file=log)
print("no corresponding atoms in the model:", file=log)
for h in deleted_annotations.helices:
print(h.as_pdb_str(), file=log)
error_msg += " %s\n" % h
if len(deleted_annotations.sheets) > 0:
print("Removing the following sheets because there are",
file=log)
print("no corresponding atoms in the model:", file=log)
for sh in deleted_annotations.sheets:
print(sh.as_pdb_str(), file=log)
error_msg += " %s\n" % sh.as_pdb_str(
strand_id=st.strand_id)
else:
raise Sorry(error_msg)
phil_str = ann.as_restraint_groups()
# gathering initial special position atoms
special_position_settings = crystal.special_position_settings(
crystal_symmetry=model.crystal_symmetry())
site_symmetry_table = \
special_position_settings.site_symmetry_table(
sites_cart = model.get_sites_cart(),
unconditional_general_position_flags=(
model.get_atoms().extract_occ() != 1))
original_spi = site_symmetry_table.special_position_indices()
t2 = time()
# Actually idelizing SS elements
fixed_ss_selection = flex.bool(n_atoms_in_real_h, False)
log.write("Replacing ss-elements with ideal ones:\n")
log.flush()
ss_stats = gather_ss_stats(pdb_h=model.get_hierarchy())
n_idealized_elements = 0
master_bool_sel = model.get_master_selection()
if master_bool_sel is None or master_bool_sel.size() == 0:
master_bool_sel = flex.bool(model.get_number_of_atoms(), True)
elif isinstance(master_bool_sel, flex.size_t):
master_bool_sel = flex.bool(model.get_number_of_atoms(),
master_bool_sel)
assert master_bool_sel.size() == model.get_number_of_atoms()
for h in ann.helices:
log.write(" %s\n" % h.as_pdb_str())
log.flush()
if processed_params.skip_good_ss_elements and ss_element_is_good(
ss_stats, ([h], [])):
log.write(" skipping, good element.\n")
else:
selstring = h.as_atom_selections()
sel = selection_cache.selection(selstring[0])
isel = sel.iselection()
if (master_bool_sel & sel).iselection().size() == 0:
log.write(" skipping, not in NCS master.\n")
continue
n_idealized_elements += 1
log.write(" substitute with idealized one.\n")
fixed_ss_selection.set_selected(isel, True)
all_bsel = flex.bool(n_atoms_in_real_h, False)
all_bsel.set_selected(isel, True)
sel_h = model.get_hierarchy().select(all_bsel, copy_atoms=True)
ideal_h = get_helix(helix_class=h.helix_class,
pdb_hierarchy_template=sel_h,
rotamer_manager=model.get_rotamer_manager())
# edited_h.select(all_bsel).atoms().set_xyz(ideal_h.atoms().extract_xyz())
set_xyz_carefully(dest_h=edited_h.select(all_bsel),
source_h=ideal_h)
# set_xyz_smart(dest_h=edited_h.select(all_bsel), source_h=ideal_h) # does not work here
for sh in ann.sheets:
s = " %s\n" % sh.as_pdb_str()
ss = s.replace("\n", "\n ")
log.write(ss[:-2])
log.flush()
if processed_params.skip_good_ss_elements and ss_element_is_good(
ss_stats, ([], [sh])):
log.write(" skipping, good element.\n")
else:
full_sh_selection = flex.bool(n_atoms_in_real_h, False)
for st in sh.strands:
selstring = st.as_atom_selections()
isel = selection_cache.iselection(selstring)
full_sh_selection.set_selected(isel, True)
if (master_bool_sel & full_sh_selection).iselection().size() == 0:
log.write(" skipping, not in NCS master.\n")
continue
n_idealized_elements += 1
log.write(" substitute with idealized one.\n")
for st in sh.strands:
selstring = st.as_atom_selections()
isel = selection_cache.iselection(selstring)
all_bsel = flex.bool(n_atoms_in_real_h, False)
all_bsel.set_selected(isel, True)
fixed_ss_selection.set_selected(isel, True)
sel_h = model.get_hierarchy().select(all_bsel, copy_atoms=True)
ideal_h = secondary_structure_from_sequence(
pdb_str=beta_pdb_str,
sequence=None,
pdb_hierarchy_template=sel_h,
rotamer_manager=model.get_rotamer_manager(),
)
set_xyz_carefully(edited_h.select(all_bsel), ideal_h)
# edited_h.select(all_bsel).atoms().set_xyz(ideal_h.atoms().extract_xyz())
if n_idealized_elements == 0:
log.write("Nothing was idealized.\n")
# Don't do geometry minimization and stuff if nothing was changed.
return None
# XXX here we want to adopt new coordinates
model.set_sites_cart(sites_cart=edited_h.atoms().extract_xyz())
if model.ncs_constraints_present():
model.set_sites_cart_from_hierarchy(multiply_ncs=True)
t3 = time()
# pre_result_h = edited_h
# pre_result_h.reset_i_seq_if_necessary()
bsel = flex.bool(n_atoms_in_real_h, False)
helix_selection = flex.bool(n_atoms_in_real_h, False)
sheet_selection = flex.bool(n_atoms_in_real_h, False)
other_selection = flex.bool(n_atoms_in_real_h, False)
ss_for_tors_selection = flex.bool(n_atoms_in_real_h, False)
nonss_for_tors_selection = flex.bool(n_atoms_in_real_h, False)
# set all CA atoms to True for other_selection
#isel = selection_cache.iselection("name ca")
isel = selection_cache.iselection("name ca or name n or name o or name c")
other_selection.set_selected(isel, True)
n_main_chain_atoms = other_selection.count(True)
isel = selection_cache.iselection("name ca or name n or name o or name c")
nonss_for_tors_selection.set_selected(isel, True)
main_chain_selection_prefix = "(name ca or name n or name o or name c) %s"
t4 = time()
print("Preparing selections...", file=log)
log.flush()
# Here we are just preparing selections
for h in ann.helices:
ss_sels = h.as_atom_selections()[0]
selstring = main_chain_selection_prefix % ss_sels
isel = selection_cache.iselection(selstring)
helix_selection.set_selected(isel, True)
other_selection.set_selected(isel, False)
isel = selection_cache.iselection(selstring)
ss_for_tors_selection.set_selected(isel, True)
nonss_for_tors_selection.set_selected(isel, False)
for sheet in ann.sheets:
for ss_sels in sheet.as_atom_selections():
selstring = main_chain_selection_prefix % ss_sels
isel = selection_cache.iselection(selstring)
sheet_selection.set_selected(isel, True)
other_selection.set_selected(isel, False)
isel = selection_cache.iselection(selstring)
ss_for_tors_selection.set_selected(isel, True)
nonss_for_tors_selection.set_selected(isel, False)
t5 = time()
isel = selection_cache.iselection(
"not name ca and not name n and not name o and not name c")
other_selection.set_selected(isel, False)
helix_sheet_intersection = helix_selection & sheet_selection
if helix_sheet_intersection.count(True) > 0:
sheet_selection = sheet_selection & ~helix_sheet_intersection
assert ((helix_selection | sheet_selection)
& other_selection).count(True) == 0
from mmtbx.monomer_library.pdb_interpretation import grand_master_phil_str
params_line = grand_master_phil_str
params_line += "secondary_structure {%s}" % secondary_structure.sec_str_master_phil_str
# print "params_line"
# print params_line
params = iotbx.phil.parse(input_string=params_line,
process_includes=True) #.extract()
# This does not work the same way for a strange reason. Need to investigate.
# The number of resulting hbonds is different later.
# w_params = params.extract()
# w_params.pdb_interpretation.secondary_structure.protein.remove_outliers = False
# w_params.pdb_interpretation.peptide_link.ramachandran_restraints = True
# w_params.pdb_interpretation.c_beta_restraints = True
# w_params.pdb_interpretation.secondary_structure.enabled = True
# params.format(python_object=w_params)
# params.show()
# print "="*80
# print "="*80
# print "="*80
grm = model.get_restraints_manager()
ssm_log = null_out()
if verbose:
ssm_log = log
ss_params = secondary_structure.sec_str_master_phil.fetch().extract()
ss_params.secondary_structure.protein.remove_outliers = False
ss_manager = secondary_structure.manager(
pdb_hierarchy=model.get_hierarchy(),
geometry_restraints_manager=grm.geometry,
sec_str_from_pdb_file=ss_annotation,
params=ss_params.secondary_structure,
mon_lib_srv=None,
verbose=-1,
log=ssm_log)
grm.geometry.set_secondary_structure_restraints(
ss_manager=ss_manager, hierarchy=model.get_hierarchy(), log=ssm_log)
model.get_hierarchy().reset_i_seq_if_necessary()
from mmtbx.geometry_restraints import reference
if reference_map is None:
if verbose:
print("Adding reference coordinate restraints...", file=log)
grm.geometry.append_reference_coordinate_restraints_in_place(
reference.add_coordinate_restraints(
sites_cart=model.get_sites_cart().select(helix_selection),
selection=helix_selection,
sigma=processed_params.sigma_on_reference_helix))
grm.geometry.append_reference_coordinate_restraints_in_place(
reference.add_coordinate_restraints(
sites_cart=model.get_sites_cart().select(sheet_selection),
selection=sheet_selection,
sigma=processed_params.sigma_on_reference_sheet))
grm.geometry.append_reference_coordinate_restraints_in_place(
reference.add_coordinate_restraints(
sites_cart=model.get_sites_cart().select(other_selection),
selection=other_selection,
sigma=processed_params.sigma_on_reference_non_ss))
# XXX Somewhere here we actually should check placed side-chains for
# clashes because we used ones that were in original model and just moved
# them to nearest allowed rotamer. The idealization may affect a lot
# the orientation of side chain thus justifying changing rotamer on it
# to avoid clashes.
if processed_params.fix_rotamer_outliers:
print("Fixing/checking rotamers...", file=log)
# pre_result_h.write_pdb_file(file_name="before_rotamers.pdb")
br_txt = model.model_as_pdb()
with open("before_rotamers.pdb", 'w') as f:
f.write(br_txt)
if (reference_map is None):
backbone_sample = False
else:
backbone_sample = True
result = mmtbx.refinement.real_space.fit_residues.run(
pdb_hierarchy=model.get_hierarchy(),
crystal_symmetry=model.crystal_symmetry(),
map_data=reference_map,
rotamer_manager=mmtbx.idealized_aa_residues.rotamer_manager.load(),
sin_cos_table=scitbx.math.sin_cos_table(n=10000),
backbone_sample=backbone_sample,
mon_lib_srv=model.get_mon_lib_srv(),
log=log)
model.set_sites_cart(
sites_cart=result.pdb_hierarchy.atoms().extract_xyz(),
update_grm=True)
if verbose:
print("Adding chi torsion restraints...", file=log)
# only backbone
grm.geometry.add_chi_torsion_restraints_in_place(
pdb_hierarchy = model.get_hierarchy(),
sites_cart = model.get_sites_cart().\
select(ss_for_tors_selection),
selection = ss_for_tors_selection,
chi_angles_only = False,
sigma = processed_params.sigma_on_torsion_ss)
grm.geometry.add_chi_torsion_restraints_in_place(
pdb_hierarchy = model.get_hierarchy(),
sites_cart = model.get_sites_cart().\
select(nonss_for_tors_selection),
selection = nonss_for_tors_selection,
chi_angles_only = False,
sigma = processed_params.sigma_on_torsion_nonss)
# real_h.atoms().set_xyz(pre_result_h.atoms().extract_xyz())
#
# Check and correct for special positions
#
real_h = model.get_hierarchy() # just a shortcut here...
special_position_settings = crystal.special_position_settings(
crystal_symmetry=model.crystal_symmetry())
site_symmetry_table = \
special_position_settings.site_symmetry_table(
sites_cart = model.get_sites_cart(),
unconditional_general_position_flags=(
model.get_atoms().extract_occ() != 1))
spi = site_symmetry_table.special_position_indices()
if spi.size() > 0:
print("Moving atoms from special positions:", file=log)
for spi_i in spi:
if spi_i not in original_spi:
new_coords = (real_h.atoms()[spi_i].xyz[0] + 0.2,
real_h.atoms()[spi_i].xyz[1] + 0.2,
real_h.atoms()[spi_i].xyz[2] + 0.2)
print(" ", real_h.atoms()[spi_i].id_str(), end=' ', file=log)
print(tuple(real_h.atoms()[spi_i].xyz),
"-->",
new_coords,
file=log)
real_h.atoms()[spi_i].set_xyz(new_coords)
model.set_sites_cart_from_hierarchy()
t9 = time()
if processed_params.file_name_before_regularization is not None:
grm.geometry.pair_proxies(sites_cart=model.get_sites_cart())
grm.geometry.update_ramachandran_restraints_phi_psi_targets(
hierarchy=model.get_hierarchy())
print("Outputting model before regularization %s" %
processed_params.file_name_before_regularization,
file=log)
m_txt = model.model_as_pdb()
g_txt = model.restraints_as_geo()
with open(processed_params.file_name_before_regularization, 'w') as f:
f.write(m_txt)
geo_fname = processed_params.file_name_before_regularization[:-4] + '.geo'
print("Outputting geo file for regularization %s" % geo_fname,
file=log)
with open(geo_fname, 'w') as f:
f.write(g_txt)
#testing number of restraints
assert grm.geometry.get_n_den_proxies() == 0
if reference_map is None:
assert grm.geometry.get_n_reference_coordinate_proxies() == n_main_chain_atoms, "" +\
"%d %d" % (grm.geometry.get_n_reference_coordinate_proxies(), n_main_chain_atoms)
refinement_log = null_out()
log.write(
"Refining geometry of substituted secondary structure elements...")
log.flush()
if verbose:
refinement_log = log
t10 = time()
if reference_map is None:
minimize_wrapper_for_ramachandran(
model=model,
original_pdb_h=None,
excl_string_selection="",
log=refinement_log,
number_of_cycles=processed_params.n_iter)
else:
ref_xrs = model.crystal_symmetry()
minimize_wrapper_with_map(model=model,
target_map=reference_map,
refine_ncs_operators=False,
number_of_cycles=processed_params.n_macro,
log=log)
model.set_sites_cart_from_hierarchy()
log.write(" Done\n")
log.flush()
t11 = time()
# print >> log, "Initial checking, init : %.4f" % (t1-t0)
# print >> log, "Checking SS : %.4f" % (t2-t1)
# print >> log, "Initializing selections : %.4f" % (t4-t3)
# print >> log, "Looping for selections : %.4f" % (t5-t4)
# print >> log, "Finalizing selections : %.4f" % (t6-t5)
# print >> log, "PDB interpretation : %.4f" % (t7-t6)
# print >> log, "Get GRM : %.4f" % (t8-t7)
# print >> log, "Adding restraints to GRM : %.4f" % (t9-t8)
# print >> log, "Running GM : %.4f" % (t11-t10)
# print_hbond_proxies(grm.geometry,real_h)
grm.geometry.remove_reference_coordinate_restraints_in_place()
grm.geometry.remove_chi_torsion_restraints_in_place(
nonss_for_tors_selection)
return grm.geometry.get_chi_torsion_proxies()
def exercise_merge_equivalents():
i = flex.miller_index(
((1, 2, 3), (1, 2, 3), (3, 0, 3), (3, 0, 3), (3, 0, 3), (1, 1, 2)))
d = flex.double((1, 2, 3, 4, 5, 6))
m = miller.ext.merge_equivalents_real(i, d)
assert tuple(m.indices) == ((1, 2, 3), (3, 0, 3), (1, 1, 2))
assert approx_equal(m.data, (3 / 2., 4, 6))
assert tuple(m.redundancies) == (2, 3, 1)
assert approx_equal(m.r_linear, (1 / 3., 1 / 6., 0))
assert approx_equal(m.r_square, (0.1, 0.04, 0))
assert approx_equal(m.r_int, (1. + 2.) / (3. + 12.))
#
s = flex.double((1 / 3., 1 / 2., 1 / 4., 1 / 6., 1 / 3., 1 / 5.))
m = miller.ext.merge_equivalents_obs(i, d, s, sigma_dynamic_range=2e-6)
assert tuple(m.indices) == ((1, 2, 3), (3, 0, 3), (1, 1, 2))
assert approx_equal(m.data,
(17 / 13.,
(16 * 3 + 36 * 4 + 9 * 5) / (16 + 36 + 9.), 6))
assert approx_equal(
m.sigmas, (math.sqrt(1 / 2. / 2), 0.84077140277 / 3**0.5, 1 / 5.))
assert m.sigma_dynamic_range == 2e-6
assert tuple(m.redundancies) == (2, 3, 1)
assert approx_equal(m.r_linear, (1 / 3., 0.1762295, 0))
assert approx_equal(m.r_square, (0.1147929, 0.0407901, 0))
assert approx_equal(
m.r_int,
(abs(1 - 17 / 13.) + abs(2 - 17 / 13.) + abs(3 - 237 / 61.) +
abs(4 - 237 / 61.) + abs(5 - 237 / 61.)) / (1 + 2 + 3 + 4 + 5))
#
d = flex.complex_double([
complex(-1.706478, 0.248638),
complex(1.097872, -0.983523),
complex(0.147183, 2.625064),
complex(-0.933310, 2.496886),
complex(1.745500, -0.686761),
complex(-0.620066, 2.097776)
])
m = miller.ext.merge_equivalents_complex(i, d)
assert tuple(m.indices) == ((1, 2, 3), (3, 0, 3), (1, 1, 2))
assert approx_equal(m.data, [
complex(-0.304303, -0.367443),
complex(0.319791, 1.478396),
complex(-0.620066, 2.097776)
])
assert tuple(m.redundancies) == (2, 3, 1)
#
d = flex.hendrickson_lattman([(-1.706478, 0.248638, 1.653352, -2.411313),
(1.097872, -0.983523, -2.756402, 0.294464),
(0.147183, 2.625064, 1.003636, 2.563517),
(-0.933310, 2.496886, 2.040418, 0.371885),
(1.745500, -0.686761, -2.291345, -2.386650),
(-0.620066, 2.097776, 0.099784, 0.268107)])
m = miller.ext.merge_equivalents_hl(i, d)
assert tuple(m.indices) == ((1, 2, 3), (3, 0, 3), (1, 1, 2))
assert approx_equal(m.data,
[(-0.3043030, -0.3674425, -0.5515250, -1.0584245),
(0.3197910, 1.4783963, 0.2509030, 0.1829173),
(-0.6200660, 2.0977760, 0.0997840, 0.2681070)])
assert tuple(m.redundancies) == (2, 3, 1)
#
d = flex.bool((True, True, False, False, False, True))
m = miller.ext.merge_equivalents_exact_bool(i, d)
assert tuple(m.indices) == ((1, 2, 3), (3, 0, 3), (1, 1, 2))
assert list(m.data) == [True, False, True]
assert tuple(m.redundancies) == (2, 3, 1)
d = flex.bool((True, True, False, True, False, True))
try:
m = miller.ext.merge_equivalents_exact_bool(i, d)
except RuntimeError, e:
assert str(e) == "cctbx Error: merge_equivalents_exact:"\
" incompatible flags for hkl = (3, 0, 3)"
def run(args,
command_name="mmtbx.model_vs_data",
show_geometry_statistics=True,
model_size_max_atoms=80000,
data_size_max_reflections=1000000,
unit_cell_max_dimension=800.,
return_fmodel_and_pdb=False,
out=None,
log=sys.stdout):
import mmtbx.f_model.f_model_info
if (len(args) == 0) or (args == ["--help"]):
print >> log, msg
defaults(log=log, silent=False)
return
parsed = defaults(log=log, silent=True)
#
mvd_obj = mvd()
#
processed_args = utils.process_command_line_args(args=args,
log=log,
master_params=parsed)
params = processed_args.params.extract()
#
reflection_files = processed_args.reflection_files
if (len(reflection_files) == 0):
raise Sorry("No reflection file found.")
crystal_symmetry = processed_args.crystal_symmetry
if (crystal_symmetry is None):
raise Sorry("No crystal symmetry found.")
if (len(processed_args.pdb_file_names) == 0):
raise Sorry("No PDB file found.")
pdb_file_names = processed_args.pdb_file_names
#
rfs = reflection_file_server(crystal_symmetry=crystal_symmetry,
reflection_files=reflection_files)
parameters = utils.data_and_flags_master_params().extract()
if (params.f_obs_label is not None):
parameters.labels = params.f_obs_label
if (params.r_free_flags_label is not None):
parameters.r_free_flags.label = params.r_free_flags_label
if (params.high_resolution is not None):
parameters.high_resolution = params.high_resolution
determine_data_and_flags_result = utils.determine_data_and_flags(
reflection_file_server=rfs,
parameters=parameters,
data_parameter_scope="refinement.input.xray_data",
flags_parameter_scope="refinement.input.xray_data.r_free_flags",
data_description="X-ray data",
keep_going=True,
log=StringIO())
f_obs = determine_data_and_flags_result.f_obs
number_of_reflections = f_obs.indices().size()
if (params.ignore_giant_models_and_datasets
and number_of_reflections > data_size_max_reflections):
raise Sorry("Too many reflections: %d" % number_of_reflections)
#
max_unit_cell_dimension = max(f_obs.unit_cell().parameters()[:3])
if (params.ignore_giant_models_and_datasets
and max_unit_cell_dimension > unit_cell_max_dimension):
raise Sorry("Too large unit cell (max dimension): %s" %
str(max_unit_cell_dimension))
#
r_free_flags = determine_data_and_flags_result.r_free_flags
test_flag_value = determine_data_and_flags_result.test_flag_value
if (r_free_flags is None):
r_free_flags = f_obs.array(data=flex.bool(f_obs.data().size(), False))
test_flag_value = None
#
mmtbx_pdb_file = mmtbx.utils.pdb_file(
pdb_file_names=pdb_file_names,
cif_objects=processed_args.cif_objects,
crystal_symmetry=crystal_symmetry,
use_neutron_distances=(params.scattering_table == "neutron"),
ignore_unknown_nonbonded_energy_types=not show_geometry_statistics,
log=log)
mmtbx_pdb_file.set_ppf(stop_if_duplicate_labels=False)
processed_pdb_file = mmtbx_pdb_file.processed_pdb_file
pdb_raw_records = mmtbx_pdb_file.pdb_raw_records
pdb_inp = mmtbx_pdb_file.pdb_inp
#
# just to avoid going any further with bad PDB file....
pdb_inp.xray_structures_simple()
#
acp = processed_pdb_file.all_chain_proxies
atom_selections = group_args(
all=acp.selection(string="all"),
macromolecule=acp.selection(string="protein or dna or rna"),
solvent=acp.selection(string="water"), # XXX single_atom_residue
ligand=acp.selection(string="not (protein or dna or rna or water)"),
backbone=acp.selection(string="backbone"),
sidechain=acp.selection(string="sidechain"))
#
scattering_table = params.scattering_table
exptl_method = pdb_inp.get_experiment_type()
if (exptl_method is not None) and ("NEUTRON" in exptl_method):
scattering_table = "neutron"
xsfppf = mmtbx.utils.xray_structures_from_processed_pdb_file(
processed_pdb_file=processed_pdb_file,
scattering_table=scattering_table,
d_min=f_obs.d_min())
xray_structures = xsfppf.xray_structures
if (0): #XXX normalize occupancies if all models have occ=1 so the total=1
n_models = len(xray_structures)
for xrs in xray_structures:
occ = xrs.scatterers().extract_occupancies()
occ = occ / n_models
xrs.set_occupancies(occ)
model_selections = xsfppf.model_selections
mvd_obj.collect(crystal=group_args(
uc=f_obs.unit_cell(),
sg=f_obs.crystal_symmetry().space_group_info().symbol_and_number(),
n_sym_op=f_obs.crystal_symmetry().space_group_info().type().group(
).order_z(),
uc_vol=f_obs.unit_cell().volume()))
#
hierarchy = pdb_inp.construct_hierarchy()
pdb_atoms = hierarchy.atoms()
pdb_atoms.reset_i_seq()
#
# Extract TLS
pdb_tls = None
pdb_inp_tls = pdb_inp.extract_tls_params(hierarchy)
pdb_tls = group_args(pdb_inp_tls=pdb_inp_tls,
tls_selections=[],
tls_selection_strings=[])
# XXX no TLS + multiple models
if (pdb_inp_tls.tls_present and pdb_inp_tls.error_string is None
and len(xray_structures) == 1):
pdb_tls = mmtbx.tls.tools.extract_tls_from_pdb(
pdb_inp_tls=pdb_inp_tls,
all_chain_proxies=mmtbx_pdb_file.processed_pdb_file.
all_chain_proxies,
xray_structure=xsfppf.xray_structure_all)
if (len(pdb_tls.tls_selections) == len(pdb_inp_tls.tls_params)
and len(pdb_inp_tls.tls_params) > 0):
xray_structures = [
utils.extract_tls_and_u_total_from_pdb(
f_obs=f_obs,
r_free_flags=r_free_flags,
xray_structure=xray_structures[
0], # XXX no TLS + multiple models
tls_selections=pdb_tls.tls_selections,
tls_groups=pdb_inp_tls.tls_params)
]
###########################
geometry_statistics = show_geometry(
xray_structures=xray_structures,
processed_pdb_file=processed_pdb_file,
scattering_table=scattering_table,
hierarchy=hierarchy,
model_selections=model_selections,
show_geometry_statistics=show_geometry_statistics,
mvd_obj=mvd_obj,
atom_selections=atom_selections)
###########################
mp = mmtbx.masks.mask_master_params.extract()
f_obs_labels = f_obs.info().label_string()
f_obs = f_obs.sort(reverse=True, by_value="packed_indices")
r_free_flags = r_free_flags.sort(reverse=True, by_value="packed_indices")
fmodel = utils.fmodel_simple(
xray_structures=xray_structures,
scattering_table=scattering_table,
mask_params=mp,
f_obs=f_obs,
r_free_flags=r_free_flags,
skip_twin_detection=params.skip_twin_detection)
n_outl = f_obs.data().size() - fmodel.f_obs().data().size()
mvd_obj.collect(model_vs_data=show_model_vs_data(fmodel))
# Extract information from PDB file header and output (if any)
pub_r_work = None
pub_r_free = None
pub_high = None
pub_low = None
pub_sigma = None
pub_program_name = None
pub_solv_cont = None
pub_matthews = None
published_results = pdb_inp.get_r_rfree_sigma(file_name=pdb_file_names[0])
if (published_results is not None):
pub_r_work = published_results.r_work
pub_r_free = published_results.r_free
pub_high = published_results.high
pub_low = published_results.low
pub_sigma = published_results.sigma
pub_program_name = pdb_inp.get_program_name()
pub_solv_cont = pdb_inp.get_solvent_content()
pub_matthews = pdb_inp.get_matthews_coeff()
mvd_obj.collect(pdb_header=group_args(program_name=pub_program_name,
year=pdb_inp.extract_header_year(),
r_work=pub_r_work,
r_free=pub_r_free,
high_resolution=pub_high,
low_resolution=pub_low,
sigma_cutoff=pub_sigma,
matthews_coeff=pub_matthews,
solvent_cont=pub_solv_cont,
tls=pdb_tls,
exptl_method=exptl_method))
#
# Recompute R-factors using published cutoffs
fmodel_cut = fmodel
tmp_sel = flex.bool(fmodel.f_obs().data().size(), True)
if (pub_sigma is not None and fmodel.f_obs().sigmas() is not None):
tmp_sel &= fmodel.f_obs().data() > fmodel.f_obs().sigmas() * pub_sigma
if (pub_high is not None
and abs(pub_high - fmodel.f_obs().d_min()) > 0.03):
tmp_sel &= fmodel.f_obs().d_spacings().data() > pub_high
if (pub_low is not None
and abs(pub_low - fmodel.f_obs().d_max_min()[0]) > 0.03):
tmp_sel &= fmodel.f_obs().d_spacings().data() < pub_low
if (tmp_sel.count(True) != tmp_sel.size() and tmp_sel.count(True) > 0):
fmodel_cut = utils.fmodel_simple(
xray_structures=xray_structures,
scattering_table=scattering_table,
f_obs=fmodel.f_obs().select(tmp_sel),
r_free_flags=fmodel.r_free_flags().select(tmp_sel),
skip_twin_detection=params.skip_twin_detection)
mvd_obj.collect(
misc=group_args(r_work_cutoff=fmodel_cut.r_work(),
r_free_cutoff=fmodel_cut.r_free(),
n_refl_cutoff=fmodel_cut.f_obs().data().size()))
mvd_obj.collect(data=show_data(fmodel=fmodel,
n_outl=n_outl,
test_flag_value=test_flag_value,
f_obs_labels=f_obs_labels,
fmodel_cut=fmodel_cut))
# CC* and friends
cc_star_stats = None
if (params.unmerged_data is not None):
import mmtbx.validation.experimental
import mmtbx.command_line
f_obs = fmodel.f_obs().average_bijvoet_mates()
unmerged_i_obs = mmtbx.command_line.load_and_validate_unmerged_data(
f_obs=f_obs,
file_name=params.unmerged_data,
data_labels=params.unmerged_labels,
log=null_out())
cc_star_stats = mmtbx.validation.experimental.merging_and_model_statistics(
f_model=fmodel.f_model().average_bijvoet_mates(),
f_obs=f_obs,
r_free_flags=fmodel.r_free_flags().average_bijvoet_mates(),
unmerged_i_obs=unmerged_i_obs,
n_bins=params.n_bins)
mvd_obj.show(log=out)
if (cc_star_stats is not None):
cc_star_stats.show_model_vs_data(out=out, prefix=" ")
if return_fmodel_and_pdb:
mvd_obj.pdb_file = processed_pdb_file
mvd_obj.fmodel = fmodel
if (len(params.map) > 0):
for map_name_string in params.map:
map_type_obj = mmtbx.map_names(map_name_string=map_name_string)
map_params = mmtbx.maps.map_and_map_coeff_master_params().fetch(
mmtbx.maps.cast_map_coeff_params(map_type_obj)).extract()
maps_obj = mmtbx.maps.compute_map_coefficients(
fmodel=fmodel_cut, params=map_params.map_coefficients)
fn = os.path.basename(processed_args.reflection_file_names[0])
if (fn.count(".")):
prefix = fn[:fn.index(".")]
else:
prefix = fn
file_name = prefix + "_%s_map_coeffs.mtz" % map_type_obj.format()
maps_obj.write_mtz_file(file_name=file_name)
# statistics in bins
if (not fmodel.twin):
print >> log, "Statistics in resolution bins:"
mmtbx.f_model.f_model_info.r_work_and_completeness_in_resolution_bins(
fmodel=fmodel, out=log, prefix=" ")
# report map cc
if (params.comprehensive and not fmodel_cut.twin
and fmodel_cut.xray_structure is not None):
rsc_params = real_space_correlation.master_params().extract()
rsc_params.scattering_table = scattering_table
real_space_correlation.simple(fmodel=fmodel_cut,
pdb_hierarchy=hierarchy,
params=rsc_params,
log=log,
show_results=True)
#
if (params.dump_result_object_as_pickle):
output_prefixes = []
for op in processed_args.pdb_file_names + processed_args.reflection_file_names:
op = os.path.basename(op)
try:
op = op[:op.index(".")]
except Exception:
pass
if (not op in output_prefixes): output_prefixes.append(op)
output_prefix = "_".join(output_prefixes)
easy_pickle.dump("%s.pickle" % output_prefix, mvd_obj)
return mvd_obj
def deep_copy(self): return self.select(flex.bool(self.miller_array.indices().size(),True))
def deep_copy(self):
size = self.pdb_hierarchy.atoms_size()
return self.select(selection=flex.bool(size, True))
def master(frame_token, iparams, activity):
if activity == "scale":
n_batch = 1
frame_objects = frame_token
indices = range(0, len(frame_objects), n_batch)
for i in indices:
i_end = i + n_batch if i + n_batch < len(frame_objects) else len(
frame_objects)
rankreq = comm.recv(source=MPI.ANY_SOURCE)
comm.send((activity, (frame_objects[i:i_end], iparams)),
dest=rankreq)
elif activity == "pre_merge":
frame_objects, avg_mode = frame_token
n_batch = int(len(frame_objects) / (size * 3))
if n_batch < 10: n_batch = 10
indices = range(0, len(frame_objects), n_batch)
for i in indices:
i_end = i + n_batch if i + n_batch < len(frame_objects) else len(
frame_objects)
rankreq = comm.recv(source=MPI.ANY_SOURCE)
comm.send((activity, (frame_objects[i:i_end], iparams, avg_mode)),
dest=rankreq)
elif activity == "merge":
frame_objects, avg_mode = frame_token
its = intensities_scaler()
cpo = its.combine_pre_merge(frame_objects, iparams)
#assign at least 100k reflections at a time
n_batch = int(1e5 / (len(cpo[1]) / cpo[0]))
if n_batch < 1: n_batch = 1
print "Merging with %d batch size" % (n_batch)
indices = range(0, cpo[0], n_batch)
for i in indices:
rankreq = comm.recv(source=MPI.ANY_SOURCE)
i_end = i + n_batch if i + n_batch < cpo[0] else cpo[0]
sel = flex.bool([
sel_l and sel_h
for sel_l, sel_h in zip(cpo[1] >= i, cpo[1] < i_end)
])
batch_prep = [cpo_elem.select(sel) for cpo_elem in cpo[1:13]]
batch_prep.insert(0, i_end - i)
batch_prep[1] -= i
batch_prep.append(cpo[13])
batch_prep.append(cpo[14])
batch_prep.append(cpo[15].select(sel))
batch_prep.append("")
comm.send((activity, (tuple(batch_prep), iparams, avg_mode)),
dest=rankreq)
elif activity == "postref":
frame_files, miller_array_ref, results, avg_mode = frame_token
#convert results to a dict obj so that pickle_filename is the key
pres_dict = {}
for pres in results:
if pres: pres_dict[pres.pickle_filename] = pres
for i in xrange(len(frame_files)):
rankreq = comm.recv(source=MPI.ANY_SOURCE)
comm.send((activity, (i, frame_files[i], iparams, miller_array_ref,
pres_dict[frame_files[i]] if frame_files[i]
in pres_dict else None, avg_mode)),
dest=rankreq)
print "Master for %s is completed. Time to stop all %d clients" % (
activity, size - 1)
# stop clients
for rankreq in range(size - 1):
rankreq = comm.recv(source=MPI.ANY_SOURCE)
comm.send('endrun', dest=rankreq)
def show_geometry(xray_structures, processed_pdb_file, scattering_table,
hierarchy, model_selections, show_geometry_statistics,
mvd_obj, atom_selections):
if (len(xray_structures) > 1):
tmp = xray_structures[0]
for xi in xray_structures[1:]:
tmp = tmp.concatenate(xi)
xray_structures = tmp
else:
xray_structures = xray_structures[0]
##
utils.assert_xray_structures_equal(x1=xray_structures,
x2=processed_pdb_file.xray_structure(),
sites=True,
adp=False,
occupancies=True)
##
hd_sel_all = xray_structures.hd_selection()
if (show_geometry_statistics):
sctr_keys = \
xray_structures.scattering_type_registry().type_count_dict().keys()
has_hd = "H" in sctr_keys or "D" in sctr_keys
geometry = processed_pdb_file.geometry_restraints_manager(
show_energies=False,
plain_pairs_radius=5.0,
assume_hydrogens_all_missing=not has_hd)
restraints_manager_all = mmtbx.restraints.manager(geometry=geometry,
normalization=True)
models = hierarchy.models()
geometry_statistics = []
n_residues_in_altlocs = None
for i_seq, model_selection in enumerate(model_selections):
hierarchy_i_seq = pdb.hierarchy.root()
hierarchy_i_seq.append_model(models[i_seq].detached_copy())
#
overall_counts_i_seq = hierarchy_i_seq.overall_counts()
n_residues_in_altlocs = \
overall_counts_i_seq.n_alt_conf_pure + \
overall_counts_i_seq.n_alt_conf_proper + \
overall_counts_i_seq.n_alt_conf_improper
#
resname_classes = []
for k, v in zip(overall_counts_i_seq.resname_classes.keys(),
overall_counts_i_seq.resname_classes.values()):
resname_classes.append(" ".join([k.replace("common_", ""),
str(v)]))
#
xray_structure = xray_structures.select(model_selection)
assert hierarchy_i_seq.atoms_size() == xray_structure.scatterers(
).size()
hd_sel = xray_structure.hd_selection()
def select_atom_selections(selection=model_selection,
atom_selections=atom_selections):
result = group_args()
result.all = atom_selections.all.select(selection)
result.macromolecule = atom_selections.macromolecule.select(
selection)
result.solvent = atom_selections.solvent.select(selection)
result.ligand = atom_selections.ligand.select(selection)
result.backbone = atom_selections.backbone.select(selection)
result.sidechain = atom_selections.sidechain.select(selection)
return result
atom_selections_i_model = select_atom_selections(
selection=model_selection, atom_selections=atom_selections)
if (hd_sel.count(True) > 0 and scattering_table != "neutron"):
xray_structure_stat = show_xray_structure_statistics(
xray_structure=xray_structure,
hd_sel=hd_sel,
atom_selections=atom_selections_i_model)
else:
xray_structure_stat = show_xray_structure_statistics(
xray_structure=xray_structure,
atom_selections=atom_selections_i_model)
model_statistics_geometry_macromolecule = None
model_statistics_geometry_solvent = None
model_statistics_geometry_ligand = None
model_statistics_geometry_all = None
molprobity_stats_i_seq = None
rms_b_iso_or_b_equiv_bonded = None
if (show_geometry_statistics):
# exclude hydrogens
if (hd_sel.count(True) > 0 and scattering_table != "neutron"):
xray_structure = xray_structure.select(~hd_sel)
model_selection = model_selection.select(~hd_sel)
hierarchy_i_seq = hierarchy_i_seq.select(~hd_sel)
geometry = restraints_manager_all.geometry.select(
selection=~hd_sel_all)
atom_selections_i_model = select_atom_selections(
selection=~hd_sel_all,
atom_selections=atom_selections_i_model)
model_selection_as_bool = flex.bool(
xray_structures.scatterers().size(), model_selection)
geometry = restraints_manager_all.geometry.select(
selection=model_selection_as_bool)
restraints_manager = mmtbx.restraints.manager(geometry=geometry,
normalization=True)
restraints_manager.geometry.pair_proxies(
sites_cart=xray_structure.sites_cart())
###
model_statistics_geometry_all = model_statistics.geometry(
pdb_hierarchy=hierarchy_i_seq,
molprobity_scores=True,
restraints_manager=restraints_manager)
#
if (atom_selections.macromolecule.count(True) > 0):
mac_sel = atom_selections_i_model.macromolecule
model_statistics_geometry_macromolecule = model_statistics.geometry(
pdb_hierarchy=hierarchy_i_seq.select(mac_sel),
molprobity_scores=True,
restraints_manager=restraints_manager.select(mac_sel))
#
if (atom_selections.solvent.count(True) > 0):
sol_sel = atom_selections_i_model.solvent
model_statistics_geometry_solvent = model_statistics.geometry(
pdb_hierarchy=hierarchy_i_seq.select(sol_sel),
molprobity_scores=True,
restraints_manager=restraints_manager.select(sol_sel))
#
if (atom_selections.ligand.count(True) > 0):
lig_sel = atom_selections_i_model.ligand
model_statistics_geometry_ligand = model_statistics.geometry(
pdb_hierarchy=hierarchy_i_seq.select(lig_sel),
molprobity_scores=True,
restraints_manager=restraints_manager.select(lig_sel))
###
rms_b_iso_or_b_equiv_bonded = utils.rms_b_iso_or_b_equiv_bonded(
restraints_manager=restraints_manager,
xray_structure=xray_structure)
#
molprobity_stats_i_seq = molprobity_stats(
model_statistics_geometry=model_statistics_geometry_all,
resname_classes=overall_counts_i_seq.resname_classes)
geometry_statistics.append(
group_args(
n_residues_in_altlocs=n_residues_in_altlocs,
resname_classes=resname_classes,
xray_structure_stat=xray_structure_stat,
rms_b_iso_or_b_equiv_bonded=rms_b_iso_or_b_equiv_bonded,
geometry_all=model_statistics_geometry_all,
geometry_macromolecule=model_statistics_geometry_macromolecule,
geometry_solvent=model_statistics_geometry_solvent,
geometry_ligand=model_statistics_geometry_ligand,
molprobity=molprobity_stats_i_seq))
mvd_obj.collect(models=geometry_statistics)
return geometry_statistics
def run_by_params(self, iparams, avg_mode='average'):
#read all result pickles
try:
DIR = iparams.run_no + '/pickles/'
pickle_results = [
pickle.load(open(DIR + fname, "rb"))
for fname in os.listdir(DIR)
]
n_results = len(pickle_results)
#for the last cycle merging with weak anomalous flag on
#fetch the original observations from the integration pickles
if iparams.flag_weak_anomalous and iparams.target_anomalous_flag and avg_mode == 'final':
prh = postref_handler()
for pres in pickle_results:
if pres is not None:
observations_pickle = pickle.load(
open(pres.pickle_filename, 'rb'))
inputs, _ = prh.organize_input(
observations_pickle,
iparams,
avg_mode,
pickle_filename=pres.pickle_filename)
observations_original, alpha_angle_obs, spot_pred_x_mm, spot_pred_y_mm, detector_distance_mm, identified_isoform, mapped_predictions, xbeam, ybeam = inputs
two_theta = observations_original.two_theta(
wavelength=pres.wavelength).data()
ph = partiality_handler()
partiality_set, delta_xy_set, rs_set, rh_set = ph.calc_partiality_anisotropy_set(
pres.unit_cell, pres.rotx, pres.roty,
observations_original.indices(), pres.ry, pres.rz,
pres.r0, pres.re, pres.voigt_nu, two_theta,
alpha_angle_obs, pres.wavelength,
pres.crystal_orientation, spot_pred_x_mm,
spot_pred_y_mm, detector_distance_mm,
iparams.partiality_model,
iparams.flag_beam_divergence)
observations, alt_hkl = prh.get_observations_non_polar(
observations_original, pres.pickle_filename,
iparams)
pres.observations = observations
pres.observations_original = observations_original
pres.partiality = partiality_set
pres.rs_set = rs_set
pres.rh_set = rh_set
pres.mapped_predictions = mapped_predictions
except Exception:
print "Error reading input pickles."
print "Check if prime.run, prime.genref, or prime.postrefine was called prior to merge."
exit()
from prime.postrefine import prepare_output
prep_output = prepare_output(pickle_results, iparams, avg_mode)
txt_merge_mean = 'prime.merge\n'
if prep_output is not None:
#grab linear lists
cn_group, group_id_list, miller_indices_all_sort, miller_indices_ori_all_sort, I_obs_all_sort, \
sigI_obs_all_sort, G_all_sort, B_all_sort, p_all_sort, rs_all_sort, wavelength_all_sort, \
sin_sq_all_sort, SE_all_sort, uc_mean, wavelength_mean, pickle_filename_all_sort, txt_prep_out = prep_output
#start averaging
from prime.postrefine import calc_avg_I_cpp
calc_average_I_result = calc_avg_I_cpp(
cn_group, group_id_list, miller_indices_all_sort,
miller_indices_ori_all_sort, I_obs_all_sort, sigI_obs_all_sort,
G_all_sort, B_all_sort, p_all_sort, rs_all_sort,
wavelength_all_sort, sin_sq_all_sort, SE_all_sort, avg_mode,
iparams, pickle_filename_all_sort)
#grab average results
miller_index, I_merge, sigI_merge, stats, I_two_halves_tuple, txt_obs_out, txt_reject_out = calc_average_I_result
I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l, I_odd_l = I_two_halves_tuple
#remove stat items with nan
r_meas_w_top, r_meas_w_btm, r_meas_top, r_meas_btm, multiplicity = stats
sel = flex.bool([math.isnan(r_top) or math.isinf(r_top)\
or math.isnan(r_btm) or math.isinf(r_btm) for r_top, r_btm in zip(r_meas_w_top, r_meas_w_btm)])
inverse_sel = ~sel
inverse_isel = inverse_sel.iselection()
miller_index = miller_index.select(inverse_isel)
I_merge = I_merge.select(inverse_isel)
sigI_merge = sigI_merge.select(inverse_isel)
I_even = I_even.select(inverse_isel)
I_odd = I_odd.select(inverse_isel)
I_even_h = I_even_h.select(inverse_isel)
I_odd_h = I_odd_h.select(inverse_isel)
I_even_k = I_even_k.select(inverse_isel)
I_odd_k = I_odd_k.select(inverse_isel)
I_even_l = I_even_l.select(inverse_isel)
I_odd_l = I_odd_l.select(inverse_isel)
stat_all = (r_meas_w_top.select(inverse_isel), r_meas_w_btm.select(inverse_isel), \
r_meas_top.select(inverse_isel), r_meas_btm.select(inverse_isel), multiplicity.select(inverse_isel))
#write out rejected reflections
f = open(iparams.run_no + '/rejections.txt', 'w')
f.write(txt_reject_out)
f.close()
#get the latest no. of mtz file and write out the reflections
DIR = iparams.run_no + '/mtz/'
file_no = len([
int(fname.split('.')[0]) for fname in os.listdir(DIR)
if os.path.join(DIR, fname).endswith('mtz')
])
from prime.postrefine import write_output
miller_array_ref, txt_merge_mean_table = write_output(
miller_index, I_merge, sigI_merge, stat_all,
(I_even, I_odd, I_even_h, I_odd_h, I_even_k, I_odd_k, I_even_l,
I_odd_l), iparams, uc_mean, wavelength_mean, file_no,
avg_mode)
txt_merge_mean += txt_merge_mean_table + txt_prep_out
print txt_merge_mean
f = open(iparams.run_no + '/log.txt', 'a')
f.write(txt_merge_mean)
f.close()
def run(server_info, inp, status):
print "<pre>"
if (inp.format == "cns_sdb"):
print "Minimum distance between symmetrically equivalent sites:",
print float(inp.min_distance_sym_equiv)
print
structures = inp_as_xray_structures(inp)
if (len(structures) == 0):
print "No CNS sdb files found!"
print
print "Note that each file must start with {+ file: some_file_name +}"
print "in order to be recognized."
print
else:
if (inp.ucparams is None): inp.ucparams = ""
if (inp.sgsymbol is None): inp.sgsymbol = "P1"
special_position_settings = io_utils.special_position_settings_from_inp(
inp)
special_position_settings.show_summary()
print "Minimum distance between symmetrically equivalent sites:",
print special_position_settings.min_distance_sym_equiv()
print
structures = [
io_utils.structure_from_inp(inp, status, special_position_settings)
]
d_min = float(inp.d_min)
print "Minimum d-spacing:", d_min
if (d_min <= 0.):
raise ValueError, "d-spacing must be greater than zero."
print
min_peak_distance = float(inp.min_peak_distance)
print "Minimum peak distance:", min_peak_distance
if (min_peak_distance <= 0.):
raise ValueError, "min_peak_distance must be greater than zero."
print
max_reduced_peaks = int(inp.max_reduced_peaks)
print "Maximum number of peaks:", max_reduced_peaks
if (max_reduced_peaks <= 0):
raise ValueError, "max_reduced_peaks must be greater than zero."
print
for structure in structures:
if (inp.format == "cns_sdb"):
structure.show_summary().show_scatterers()
print
if (structure.scatterers().size() == 0): continue
reduced_peaks = phase_o_phrenia.calculate_exp_i_two_phi_peaks(
xray_structure=structure,
d_min=d_min,
min_peak_distance=min_peak_distance,
max_reduced_peaks=max_reduced_peaks)
print "Actual number of peaks:", len(reduced_peaks)
print
plot_nx = min(len(reduced_peaks), 60)
if (plot_nx > 0):
plot_ny = max(10, plot_nx // 3)
if (plot_nx != max_reduced_peaks):
print "Number of peaks used for plot:", plot_nx
print
print "Plot of relative peak heights:"
print
plot = flex.bool(flex.grid(plot_nx, plot_ny))
for i in xrange(plot_nx):
height = reduced_peaks[i].height
h = int(round(height * plot_ny))
h = max(0, min(plot_ny, h))
for j in xrange(h):
plot[(i, j)] = True
for j in xrange(plot_ny - 1, -1, -1):
line = ""
for i in xrange(plot_nx):
if (plot[(i, j)]): line += "*"
else: line += " "
print " |" + line.rstrip()
print " -" + "-" * plot_nx
print
print "Peak list:"
print " Relative"
print " height Fractional coordinates"
for peak in reduced_peaks:
print " %5.1f" % (peak.height *
100), " %8.5f %8.5f %8.5f" % peak.site
print
print "</pre>"
def extract(file_name,
crystal_symmetry,
wavelength_id,
crystal_id,
show_details_if_error,
output_r_free_label,
merge_non_unique_under_symmetry,
map_to_asu,
remove_systematic_absences,
all_miller_arrays=None,
incompatible_flags_to_work_set=False,
ignore_bad_sigmas=False,
extend_flags=False,
return_as_miller_arrays=False,
log=sys.stdout):
import iotbx.cif
from cctbx import miller
if all_miller_arrays is None:
base_array_info = miller.array_info(
crystal_symmetry_from_file=crystal_symmetry)
all_miller_arrays = iotbx.cif.reader(
file_path=file_name).build_miller_arrays(
base_array_info=base_array_info)
if (len(all_miller_arrays) == 0):
raise Sorry(
"No data arrays were found in this CIF file. Please make " +
"sure that the file contains reflection data, rather than the refined "
+ "model.")
column_labels = set()
if (extend_flags):
map_to_asu = True
# TODO: is all_mille_arrays a dict ? If not change back
for (data_name, miller_arrays) in six.iteritems(all_miller_arrays):
for ma in miller_arrays.values():
other_symmetry = crystal_symmetry
try:
crystal_symmetry = other_symmetry.join_symmetry(
other_symmetry=ma.crystal_symmetry(), force=True)
except AssertionError as e:
str_e = str(e)
from six.moves import cStringIO as StringIO
s = StringIO()
if "Space group is incompatible with unit cell parameters." in str_e:
other_symmetry.show_summary(f=s)
ma.crystal_symmetry().show_summary(f=s)
str_e += "\n%s" % (s.getvalue())
raise Sorry(str_e)
else:
raise
if (crystal_symmetry.unit_cell() is None
or crystal_symmetry.space_group_info() is None):
raise Sorry(
"Crystal symmetry is not defined. Please use the --symmetry option."
)
mtz_object = iotbx.mtz.object() \
.set_title(title="phenix.cif_as_mtz") \
.set_space_group_info(space_group_info=crystal_symmetry.space_group_info())
unit_cell = crystal_symmetry.unit_cell()
mtz_crystals = {}
mtz_object.set_hkl_base(unit_cell=unit_cell)
from iotbx.reflection_file_utils import cif_status_flags_as_int_r_free_flags
# generate list of all reflections (for checking R-free flags)
from iotbx.reflection_file_utils import make_joined_set
all_arrays = []
for (data_name, miller_arrays) in six.iteritems(all_miller_arrays):
for ma in miller_arrays.values():
all_arrays.append(ma)
complete_set = make_joined_set(all_arrays)
if return_as_miller_arrays:
miller_array_list = []
for i, (data_name,
miller_arrays) in enumerate(six.iteritems(all_miller_arrays)):
for ma in miller_arrays.values():
ma = ma.customized_copy(
crystal_symmetry=crystal_symmetry).set_info(ma.info())
labels = ma.info().labels
label = get_label(miller_array=ma,
output_r_free_label=output_r_free_label)
if label is None:
print("Can't determine output label for %s - skipping." % \
ma.info().label_string(), file=log)
continue
elif label.startswith(output_r_free_label):
ma, _ = cif_status_flags_as_int_r_free_flags(
ma, test_flag_value="f")
if isinstance(ma.data(), flex.double):
data_int = ma.data().iround()
assert data_int.as_double().all_eq(ma.data())
ma = ma.customized_copy(data=data_int).set_info(ma.info())
elif (
(ma.is_xray_amplitude_array() or ma.is_xray_intensity_array())
and isinstance(ma.data(), flex.int)):
ma = ma.customized_copy(data=ma.data().as_double()).set_info(
ma.info())
crys_id = 0
for l in labels:
if 'crystal_id' in l:
crys_id = int(l.split('=')[-1])
break
if crys_id > 0 and crystal_id is None:
label += "%i" % crys_id
if crystal_id is not None and crys_id > 0 and crys_id != crystal_id:
continue
if crys_id not in mtz_crystals:
mtz_crystals[crys_id] = (mtz_object.add_crystal(
name="crystal_%i" % crys_id,
project_name="project",
unit_cell=unit_cell), {})
crystal, datasets = mtz_crystals[crys_id]
w_id = 0
for l in labels:
if 'wavelength_id' in l:
w_id = int(l.split('=')[-1])
break
if wavelength_id is not None and w_id > 0 and w_id != wavelength_id:
continue
if w_id > 1 and wavelength_id is None:
if (label in column_labels):
label += "%i" % w_id
#print "label is", label
if w_id not in datasets:
wavelength = ma.info().wavelength
if (wavelength is None):
wavelength = 0
datasets[w_id] = crystal.add_dataset(name="dataset",
wavelength=wavelength)
dataset = datasets[w_id]
# if all sigmas for an array are set to zero either raise an error, or set sigmas to None
if ma.sigmas() is not None and (ma.sigmas()
== 0).count(False) == 0:
if ignore_bad_sigmas:
print("Warning: bad sigmas, setting sigmas to None.",
file=log)
ma.set_sigmas(None)
else:
raise Sorry("""Bad sigmas: all sigmas are equal to zero.
Add --ignore_bad_sigmas to command arguments to leave out sigmas from mtz file."""
)
if not ma.is_unique_set_under_symmetry():
if merge_non_unique_under_symmetry:
print("Warning: merging non-unique data", file=log)
if (label.startswith(output_r_free_label)
and incompatible_flags_to_work_set):
merging = ma.merge_equivalents(
incompatible_flags_replacement=0)
if merging.n_incompatible_flags > 0:
print("Warning: %i reflections were placed in the working set " \
"because of incompatible flags between equivalents." %(
merging.n_incompatible_flags), file=log)
else:
try:
merging = ma.merge_equivalents()
except Sorry as e:
if ("merge_equivalents_exact: incompatible"
in str(e)):
raise Sorry(
str(e) + " for %s" % ma.info().labels[-1] +
"\n" +
"Add --incompatible_flags_to_work_set to command line "
"arguments to place incompatible flags to working set."
)
raise
ma = merging.array().customized_copy(
crystal_symmetry=ma).set_info(ma.info())
elif return_as_miller_arrays: # allow non-unique set
pass
else:
n_all = ma.indices().size()
sel_unique = ma.unique_under_symmetry_selection()
sel_dup = ~flex.bool(n_all, sel_unique)
n_duplicate = sel_dup.count(True)
n_uus = sel_unique.size()
msg = (
"Miller indices not unique under symmetry: " + file_name + \
"(%d redundant indices out of %d)" % (n_all-n_uus, n_all) +
"Add --merge to command arguments to force merging data.")
if (show_details_if_error):
print(msg)
ma.show_comprehensive_summary(prefix=" ")
ma.map_to_asu().sort().show_array(prefix=" ")
raise Sorry(msg)
if (map_to_asu):
ma = ma.map_to_asu().set_info(ma.info())
if (remove_systematic_absences):
ma = ma.remove_systematic_absences()
if (label.startswith(output_r_free_label)
and complete_set is not None):
n_missing = len(complete_set.lone_set(other=ma).indices())
if (n_missing > 0):
if (extend_flags):
from cctbx import r_free_utils
# determine flag values
fvals = list(set(ma.data()))
print("fvals", fvals)
fval = None
if (len(fvals) == 1):
fval = fvals[0]
elif (len(fvals) == 2):
f1 = (ma.data()
== fvals[0]).count(True) / ma.data().size()
f2 = (ma.data()
== fvals[1]).count(True) / ma.data().size()
if (f1 < f2): fval = fvals[0]
else: fval = fvals[1]
elif (len(fvals) == 0):
fval = None
else:
fval = 0
if (not fval in fvals):
raise Sorry(
"Cannot determine free-R flag value.")
#
if (fval is not None):
ma = r_free_utils.extend_flags(
r_free_flags=ma,
test_flag_value=fval,
array_label=label,
complete_set=complete_set,
preserve_input_values=True,
allow_uniform_flags=True,
log=sys.stdout)
else:
ma = None
else:
libtbx.warn((
"%d reflections do not have R-free flags in the " +
"array '%s' - this may " +
"cause problems if you try to use the MTZ file for refinement "
+
"or map calculation. We recommend that you extend the flags "
+
"to cover all reflections (--extend_flags on the command line)."
) % (n_missing, label))
# Get rid of fake (0,0,0) reflection in some CIFs
if (ma is not None):
ma = ma.select_indices(indices=flex.miller_index(
((0, 0, 0), )),
negate=True).set_info(ma.info())
if return_as_miller_arrays:
miller_array_list.append(ma)
continue # don't make a dataset
dec = None
if ("FWT" in label):
dec = iotbx.mtz.ccp4_label_decorator()
column_types = None
if ("PHI" in label or "PHWT" in label) and (ma.is_real_array()):
column_types = "P"
elif (label.startswith("DANO") and ma.is_real_array()):
if (ma.sigmas() is not None):
column_types = "DQ"
else:
column_types = "D"
label_base = label
i = 1
while label in column_labels:
label = label_base + "-%i" % (i)
i += 1
if (ma is not None):
column_labels.add(label)
dataset.add_miller_array(ma,
column_root_label=label,
label_decorator=dec,
column_types=column_types)
if return_as_miller_arrays:
return miller_array_list
else:
return mtz_object
def __init__(self,
fmodel,
selections=None,
params=None,
r_initial=None,
t_initial=None,
bss=None,
log=None,
monitors=None):
global time_rigid_body_total
self.params = params
save_original_target_name = fmodel.target_name
save_bss_anisotropic_scaling = None
if (bss is not None):
save_bss_anisotropic_scaling = bss.anisotropic_scaling
timer_rigid_body_total = user_plus_sys_time()
save_r_work = fmodel.r_work()
save_r_free = fmodel.r_free()
save_xray_structure = fmodel.xray_structure.deep_copy_scatterers()
if (log is None): log = sys.stdout
if (selections is None):
selections = []
selections.append(
flex.bool(fmodel.xray_structure.scatterers().size(),
True).iselection())
else:
assert len(selections) > 0
fmodel.xray_structure.scatterers().flags_set_grads(state=False)
fmodel.xray_structure.scatterers().flags_set_grad_site(
iselection=flex.bool(fmodel.xray_structure.scatterers().size(),
True).iselection())
self.total_rotation = []
self.total_translation = []
for item in selections:
self.total_rotation.append(flex.double(3, 0))
self.total_translation.append(flex.double(3, 0))
if (r_initial is None):
r_initial = []
for item in selections:
r_initial.append(flex.double(3, 0))
if (t_initial is None):
t_initial = []
for item in selections:
t_initial.append(flex.double(3, 0))
fmodel_copy = fmodel.deep_copy()
if (fmodel_copy.mask_params is not None):
fmodel_copy.mask_params.verbose = -1
d_mins, target_names = split_resolution_range(
d_spacings=fmodel_copy.f_obs_work().d_spacings().data(),
n_bodies=len(selections),
target=params.target,
target_auto_switch_resolution=params.target_auto_switch_resolution,
n_ref_first=params.min_number_of_reflections,
multi_body_factor_n_ref_first=params.multi_body_factor,
d_low=params.max_low_high_res_limit,
d_high=params.high_resolution,
number_of_zones=params.number_of_zones,
zone_exponent=params.zone_exponent,
log=log)
print >> log
if (fmodel.target_name != target_names[0]):
fmodel.update(target_name=target_names[0])
self.show(fmodel=fmodel,
r_mat=self.total_rotation,
t_vec=self.total_translation,
header="Start",
out=log)
if (params.number_of_zones == 1 or monitors is None):
monitors_call_back_handler = None
else:
monitors_call_back_handler = monitors.call_back_handler
if (monitors_call_back_handler is not None):
monitors_call_back_handler(monitor=None,
model=None,
fmodel=fmodel,
method="rigid_body")
for res, target_name in zip(d_mins, target_names):
xrs = fmodel_copy.xray_structure.deep_copy_scatterers()
fmodel_copy = fmodel.resolution_filter(d_min=res)
if (fmodel_copy.target_name != target_name):
fmodel_copy.update(target_name=target_name)
d_max_min = fmodel_copy.f_obs_work().d_max_min()
line = "Refinement at resolution: "+\
str("%7.2f"%d_max_min[0]).strip() + " - " \
+ str("%6.2f"%d_max_min[1]).strip() \
+ " target=" + fmodel_copy.target_name
print_statistics.make_sub_header(line, out=log)
fmodel_copy.update_xray_structure(xray_structure=xrs,
update_f_calc=True)
rworks = flex.double()
if (len(d_mins) == 1):
n_rigid_body_minimizer_cycles = 1
else:
n_rigid_body_minimizer_cycles = min(int(res), 4)
for i_macro_cycle in xrange(n_rigid_body_minimizer_cycles):
if (bss is not None and params.bulk_solvent_and_scale):
if (fmodel_copy.f_obs().d_min() > 3.0):
bss.anisotropic_scaling = False
fast = True
if (bss.mode == "slow"): fast = False
fmodel_copy.update_all_scales(update_f_part1=False,
params=bss,
fast=fast,
log=log,
remove_outliers=False,
optimize_mask=False,
refine_hd_scattering=False)
if (fmodel_copy.f_obs().d_min() > 3.0):
assert save_bss_anisotropic_scaling is not None
bss.anisotropic_scaling = save_bss_anisotropic_scaling
bss.minimization_b_cart = save_bss_anisotropic_scaling
minimized = rigid_body_minimizer(
fmodel=fmodel_copy,
selections=selections,
r_initial=r_initial,
t_initial=t_initial,
refine_r=params.refine_rotation,
refine_t=params.refine_translation,
max_iterations=params.max_iterations,
euler_angle_convention=params.euler_angle_convention,
lbfgs_maxfev=params.
lbfgs_line_search_max_function_evaluations)
rotation_matrices = []
translation_vectors = []
for i in xrange(len(selections)):
self.total_rotation[i] += flex.double(minimized.r_min[i])
self.total_translation[i] += flex.double(
minimized.t_min[i])
rot_obj = scitbx.rigid_body.euler(
phi=minimized.r_min[i][0],
psi=minimized.r_min[i][1],
the=minimized.r_min[i][2],
convention=params.euler_angle_convention)
rotation_matrices.append(rot_obj.rot_mat())
translation_vectors.append(minimized.t_min[i])
new_xrs = apply_transformation(
xray_structure=minimized.fmodel.xray_structure,
rotation_matrices=rotation_matrices,
translation_vectors=translation_vectors,
selections=selections)
fmodel_copy.update_xray_structure(xray_structure=new_xrs,
update_f_calc=True,
update_f_mask=True)
rwork = minimized.fmodel.r_work()
rfree = minimized.fmodel.r_free()
assert approx_equal(rwork, fmodel_copy.r_work())
fmodel.update_xray_structure(
xray_structure=fmodel_copy.xray_structure,
update_f_calc=True,
update_f_mask=True)
if (bss is not None and params.bulk_solvent_and_scale):
fast = True
if (bss.mode == "slow"): fast = False
fmodel_copy.update_all_scales(update_f_part1=False,
params=bss,
fast=fast,
log=log,
remove_outliers=False,
optimize_mask=False,
refine_hd_scattering=False)
self.show(fmodel=fmodel,
r_mat=self.total_rotation,
t_vec=self.total_translation,
header="Rigid body refinement",
out=log)
if (monitors_call_back_handler is not None):
monitors_call_back_handler(monitor=None,
model=None,
fmodel=fmodel,
method="rigid_body")
if (bss is not None and params.bulk_solvent_and_scale):
fast = True
if (bss.mode == "slow"): fast = False
fmodel_copy.update_all_scales(update_f_part1=False,
params=bss,
fast=fast,
log=log,
remove_outliers=False,
optimize_mask=False,
refine_hd_scattering=False)
print >> log
self.show(fmodel=fmodel,
r_mat=self.total_rotation,
t_vec=self.total_translation,
header="Rigid body end",
out=log)
print >> log
self.evaluate_after_end(fmodel, save_r_work, save_r_free,
save_xray_structure, log)
self.fmodel = fmodel
self.fmodel.update(target_name=save_original_target_name)
time_rigid_body_total += timer_rigid_body_total.elapsed()
in str(e)):
raise Sorry(
str(e) + " for %s" % ma.info().labels[-1] +
"\n" +
"Add --incompatible_flags_to_work_set to command line "
"arguments to place incompatible flags to working set."
)
raise
ma = merging.array().customized_copy(
crystal_symmetry=ma).set_info(ma.info())
elif return_as_miller_arrays: # allow non-unique set
pass
else:
n_all = ma.indices().size()
sel_unique = ma.unique_under_symmetry_selection()
sel_dup = ~flex.bool(n_all, sel_unique)
n_duplicate = sel_dup.count(True)
n_uus = sel_unique.size()
msg = (
"Miller indices not unique under symmetry: " + file_name + \
"(%d redundant indices out of %d)" % (n_all-n_uus, n_all) +
"Add --merge to command arguments to force merging data.")
if (show_details_if_error):
print msg
ma.show_comprehensive_summary(prefix=" ")
ma.map_to_asu().sort().show_array(prefix=" ")
raise Sorry(msg)
if (map_to_asu):
ma = ma.map_to_asu().set_info(ma.info())
if (remove_systematic_absences):
ma = ma.remove_systematic_absences()
def refine(self,
selection,
optimize_weight = True,
start_trial_weight_value = 50,
selection_buffer_radius=5,
box_cushion=2,
rms_bonds_limit = 0.03,
rms_angles_limit = 3.0):
if(self.ncs_groups is None or len(self.ncs_groups)==0): # no NCS constraints
sites_cart_moving = self.sites_cart
selection_within = self.xray_structure.selection_within(
radius = selection_buffer_radius,
selection = selection)
sel = selection.select(selection_within)
iselection = flex.size_t()
for i, state in enumerate(selection):
if state:
iselection.append(i)
box = utils.extract_box_around_model_and_map(
xray_structure = self.xray_structure.select(selection=selection_within),
map_data = self.target_map,
box_cushion = box_cushion)
new_unit_cell = box.xray_structure_box.unit_cell()
geo_box = self.geometry_restraints_manager.select(selection_within)
geo_box = geo_box.discard_symmetry(new_unit_cell=new_unit_cell)
geo_box.shift_sites_cart(box.shift_cart) # disaster happens otherwise
map_box = box.map_box
sites_cart_box = box.xray_structure_box.sites_cart()
rsr_simple_refiner = simple(
target_map = map_box,
selection = sel,
real_space_gradients_delta = self.real_space_gradients_delta,
max_iterations = self.max_iterations,
geometry_restraints_manager = geo_box,
gradients_method = self.gradients_method)
real_space_result = refinery(
refiner = rsr_simple_refiner,
xray_structure = box.xray_structure_box,
optimize_weight = optimize_weight,
start_trial_weight_value = start_trial_weight_value,
rms_bonds_limit = rms_bonds_limit,
rms_angles_limit = rms_angles_limit)
self.weight_optimal = real_space_result.weight_final
sites_cart_box_refined = real_space_result.sites_cart_result
shift_back = [-box.shift_cart[i] for i in range(3)]
sites_cart_box_refined_shifted_back = \
sites_cart_box_refined + shift_back # Sure + not - ?
sites_cart_refined = sites_cart_box_refined_shifted_back.select(sel)
sites_cart_moving = sites_cart_moving.set_selected(
iselection, sites_cart_refined)
self.xray_structure.set_sites_cart(sites_cart_moving)
self.sites_cart = self.xray_structure.sites_cart()
else: # NCS constraints are present
# select on xrs, grm, ncs_groups
grm = self.geometry_restraints_manager.select(selection)
xrs = self.xray_structure.select(selection)
sel = flex.bool(xrs.scatterers().size(), True)
size = self.xray_structure.scatterers().size()
ncs_groups_ = self.ncs_groups.select(selection=selection)
#
rsr_simple_refiner = simple(
target_map = self.target_map,
selection = sel,
real_space_gradients_delta = self.real_space_gradients_delta,
max_iterations = self.max_iterations,
geometry_restraints_manager = grm,
ncs_groups = ncs_groups_)
real_space_result = refinery(
refiner = rsr_simple_refiner,
xray_structure = xrs,
optimize_weight = optimize_weight,
start_trial_weight_value = start_trial_weight_value,
rms_bonds_limit = rms_bonds_limit,
rms_angles_limit = rms_angles_limit)
self.weight_optimal = real_space_result.weight_final
def run(self):
'''
Function that places H atoms
'''
model_has_bogus_cs = False
# TODO temporary fix until the code is moved to model class
# check if box cussion of 5 A is enough to prevent symm contacts
cs = self.model.crystal_symmetry()
if (cs is None) or (cs.unit_cell() is None):
self.model = shift_and_box_model(model=self.model)
model_has_bogus_cs = True
# Remove existing H if requested
self.n_H_initial = self.model.get_hd_selection().count(True)
if not self.keep_existing_H:
self.model = self.model.select(~self.model.get_hd_selection())
t0 = time.time()
# Add H atoms and place them at center of coordinates
pdb_hierarchy = self.add_missing_H_atoms_at_bogus_position()
if print_time:
print("add_missing_H_atoms_at_bogus_position:",
round(time.time() - t0, 2))
# f = open("intermediate1.pdb","w")
# f.write(self.model.model_as_pdb())
# place N-terminal propeller hydrogens
for m in pdb_hierarchy.models():
for chain in m.chains():
rgs = chain.residue_groups()[0]
for ag in rgs.atom_groups():
if ag.get_atom("H"):
ag.remove_atom(ag.get_atom('H'))
rc = add_n_terminal_hydrogens_to_residue_group(rgs)
# rc is always empty list?
pdb_hierarchy.sort_atoms_in_place()
pdb_hierarchy.atoms().reset_serial()
# f = open("intermediate2.pdb","w")
# f.write(self.model.model_as_pdb())
p = mmtbx.model.manager.get_default_pdb_interpretation_params()
p.pdb_interpretation.clash_guard.nonbonded_distance_threshold = None
p.pdb_interpretation.use_neutron_distances = self.use_neutron_distances
p.pdb_interpretation.proceed_with_excessive_length_bonds = True
#p.pdb_interpretation.automatic_linking.link_metals = True
t0 = time.time()
#p.pdb_interpretation.restraints_library.cdl=False # XXX this triggers a bug !=360
ro = self.model.get_restraint_objects()
self.model = mmtbx.model.manager(
model_input=None,
pdb_hierarchy=pdb_hierarchy,
stop_for_unknowns=self.stop_for_unknowns,
crystal_symmetry=self.model.crystal_symmetry(),
restraint_objects=ro,
log=null_out())
self.model.process(pdb_interpretation_params=p, make_restraints=True)
if print_time:
print("get new model obj and grm:", round(time.time() - t0, 2))
#f = open("intermediate3.pdb","w")
#f.write(self.model.model_as_pdb())
# Only keep H that have been parameterized in riding H procedure
sel_h = self.model.get_hd_selection()
if sel_h.count(True) == 0:
return
# get rid of isolated H atoms.
#For example when heavy atom is missing, H needs not to be placed
sel_isolated = self.model.isolated_atoms_selection()
self.sel_lone_H = sel_h & sel_isolated
self.model = self.model.select(~self.sel_lone_H)
t0 = time.time()
# get riding H manager --> parameterize all H atoms
sel_h = self.model.get_hd_selection()
self.model.setup_riding_h_manager(use_ideal_dihedral=True)
sel_h_in_para = flex.bool(
[bool(x) for x in self.model.riding_h_manager.h_parameterization])
sel_h_not_in_para = sel_h_in_para.exclusive_or(sel_h)
self.site_labels_no_para = [
atom.id_str().replace('pdb=', '').replace('"', '') for atom in
self.model.get_hierarchy().atoms().select(sel_h_not_in_para)
]
#
self.model = self.model.select(~sel_h_not_in_para)
if print_time:
print("set up riding H manager and some cleanup:",
round(time.time() - t0, 2))
# t0 = time.time()
# self.exclude_H_on_disulfides()
# #self.exclude_h_on_coordinated_S()
# if print_time:
# print("find disulfides:", round(time.time()-t0, 2))
# f = open("intermediate4.pdb","w")
# f.write(model.model_as_pdb())
if self.validate_e:
t0 = time.time()
self.validate_electrons()
if print_time:
print("validate electrons:", round(time.time() - t0, 2))
t0 = time.time()
# Reset occupancies, ADPs and idealize H atom positions
self.model.reset_adp_for_hydrogens(scale=self.adp_scale)
self.model.reset_occupancy_for_hydrogens_simple()
self.model.idealize_h_riding()
if print_time:
print("reset adp, occ; idealize:", round(time.time() - t0, 2))
# t0 = time.time()
# self.exclude_h_on_coordinated_S()
# if print_time:
# print("coordinated S:", round(time.time()-t0, 2))
t0 = time.time()
self.exclude_H_on_links()
if print_time:
print("all links:", round(time.time() - t0, 2))
self.n_H_final = self.model.get_hd_selection().count(True)
def exercise_adp_similarity():
u_cart = ((1,3,2,4,3,6),(2,4,2,6,5,1))
u_iso = (-1,-1)
use_u_aniso = (True, True)
weight = 1
a = adp_restraints.adp_similarity(
u_cart=u_cart,
weight=weight)
assert approx_equal(a.use_u_aniso, use_u_aniso)
assert a.weight == weight
assert approx_equal(a.residual(), 68)
assert approx_equal(a.gradients2(),
((-2.0, -2.0, 0.0, -8.0, -8.0, 20.0), (2.0, 2.0, -0.0, 8.0, 8.0, -20.0)))
assert approx_equal(a.deltas(), (-1.0, -1.0, 0.0, -2.0, -2.0, 5.0))
assert approx_equal(a.rms_deltas(), 2.7487370837451071)
#
u_cart = ((1,3,2,4,3,6),(-1,-1,-1,-1,-1,-1))
u_iso = (-1,2)
use_u_aniso = (True, False)
a = adp_restraints.adp_similarity(
u_cart[0], u_iso[1], weight=weight)
assert approx_equal(a.use_u_aniso, use_u_aniso)
assert a.weight == weight
assert approx_equal(a.residual(), 124)
assert approx_equal(a.gradients2(),
((-2, 2, 0, 16, 12, 24), (2, -2, 0, -16, -12, -24)))
assert approx_equal(a.deltas(), (-1, 1, 0, 4, 3, 6))
assert approx_equal(a.rms_deltas(), 3.711842908553348)
#
i_seqs_aa = (1,2) # () - ()
i_seqs_ai = (1,0) # () - o
i_seqs_ia = (3,2) # o - ()
i_seqs_ii = (0,3) # o - o
p_aa = adp_restraints.adp_similarity_proxy(i_seqs=i_seqs_aa,weight=weight)
p_ai = adp_restraints.adp_similarity_proxy(i_seqs=i_seqs_ai,weight=weight)
p_ia = adp_restraints.adp_similarity_proxy(i_seqs=i_seqs_ia,weight=weight)
p_ii = adp_restraints.adp_similarity_proxy(i_seqs=i_seqs_ii,weight=weight)
assert p_aa.i_seqs == i_seqs_aa
assert p_aa.weight == weight
u_cart = flex.sym_mat3_double(((-1,-1,-1,-1,-1,-1),
(1,2,2,4,3,6),
(2,4,2,6,5,1),
(-1,-1,-1,-1,-1,-1)))
u_iso = flex.double((1,-1,-1,2))
use_u_aniso = flex.bool((False, True,True,False))
for p in (p_aa,p_ai,p_ia,p_ii):
params = adp_restraint_params(u_cart=u_cart, u_iso=u_iso, use_u_aniso=use_u_aniso)
a = adp_restraints.adp_similarity(params, proxy=p)
assert approx_equal(a.weight, weight)
#
gradients_aniso_cart = flex.sym_mat3_double(u_cart.size(), (0,0,0,0,0,0))
gradients_iso = flex.double(u_cart.size(), 0)
proxies = adp_restraints.shared_adp_similarity_proxy([p,p])
residuals = adp_restraints.adp_similarity_residuals(params, proxies=proxies)
assert approx_equal(residuals, (a.residual(),a.residual()))
deltas_rms = adp_restraints.adp_similarity_deltas_rms(params, proxies=proxies)
assert approx_equal(deltas_rms, (a.rms_deltas(),a.rms_deltas()))
residual_sum = adp_restraints.adp_similarity_residual_sum(
params,
proxies=proxies,
gradients_aniso_cart=gradients_aniso_cart,
gradients_iso=gradients_iso)
assert approx_equal(residual_sum, 2 * a.residual())
fd_grads_aniso, fd_grads_iso = finite_difference_gradients(
restraint_type=adp_restraints.adp_similarity,
proxy=p,
u_cart=u_cart,
u_iso=u_iso,
use_u_aniso=use_u_aniso)
for g,e in zip(gradients_aniso_cart, fd_grads_aniso):
assert approx_equal(g, matrix.col(e)*2)
for g,e in zip(gradients_iso, fd_grads_iso):
assert approx_equal(g, e*2)
#
# check frame invariance of residual
#
u_cart_1 = matrix.sym(sym_mat3=(0.1,0.2,0.05,0.03,0.02,0.01))
u_cart_2 = matrix.sym(sym_mat3=(0.21,0.32,0.11,0.02,0.02,0.07))
u_cart = (u_cart_1.as_sym_mat3(),u_cart_2.as_sym_mat3())
u_iso = (-1, -1)
use_u_aniso = (True, True)
a = adp_restraints.adp_similarity(u_cart, weight=1)
expected_residual = a.residual()
gen = flex.mersenne_twister()
for i in range(20):
R = matrix.rec(gen.random_double_r3_rotation_matrix(),(3,3))
u_cart_1_rot = R * u_cart_1 * R.transpose()
u_cart_2_rot = R * u_cart_2 * R.transpose()
u_cart = (u_cart_1_rot.as_sym_mat3(),u_cart_2_rot.as_sym_mat3())
a = adp_restraints.adp_similarity(u_cart, weight=1)
assert approx_equal(a.residual(), expected_residual)
def __init__(self, **kwargs):
group_args.__init__(self, **kwargs)
# require Dij, d_c
P = Profiler("2. calculate rho density")
print "finished Dij, now calculating rho_i, the density"
from xfel.clustering import Rodriguez_Laio_clustering_2014
# alternative clustering algorithms: see http://scikit-learn.org/stable/modules/clustering.html
# also see https://cran.r-project.org/web/packages/dbscan/vignettes/hdbscan.html
# see also https://en.wikipedia.org/wiki/Hausdorff_dimension
R = Rodriguez_Laio_clustering_2014(distance_matrix=self.Dij,
d_c=self.d_c)
self.rho = rho = R.get_rho()
ave_rho = flex.mean(rho.as_double())
NN = self.Dij.focus()[0]
print "The average rho_i is %5.2f, or %4.1f%%" % (ave_rho,
100 * ave_rho / NN)
i_max = flex.max_index(rho)
P = Profiler("3.transition")
print "the index with the highest density is %d" % (i_max)
delta_i_max = flex.max(
flex.double([self.Dij[i_max, j] for j in xrange(NN)]))
print "delta_i_max", delta_i_max
rho_order = flex.sort_permutation(rho, reverse=True)
rho_order_list = list(rho_order)
P = Profiler("4. delta")
self.delta = delta = R.get_delta(rho_order=rho_order,
delta_i_max=delta_i_max)
P = Profiler("5. find cluster maxima")
#---- Now hunting for clusters ---Lot's of room for improvement (or simplification) here!!!
cluster_id = flex.int(NN, -1) # default -1 means no cluster
delta_order = flex.sort_permutation(delta, reverse=True)
N_CLUST = 10 # maximum of 10 points to be considered as possible clusters
#MAX_PERCENTILE_DELTA = 0.99 # cluster centers have to be in the top 10% percentile delta
MAX_PERCENTILE_RHO = 0.99 # cluster centers have to be in the top 75% percentile rho
n_cluster = 0
#max_n_delta = min(N_CLUST, int(MAX_PERCENTILE_DELTA*NN))
for ic in xrange(NN):
# test the density, rho
item_idx = delta_order[ic]
if delta[item_idx] > 100:
print "A: iteration", ic, "delta", delta[
item_idx], delta[item_idx] < 0.25 * delta[delta_order[0]]
if delta[item_idx] < 0.25 * delta[
delta_order[0]]: # too low (another heuristic!)
continue
item_rho_order = rho_order_list.index(item_idx)
if delta[item_idx] > 100:
print "B: iteration", ic, item_rho_order, item_rho_order / NN, MAX_PERCENTILE_RHO
if item_rho_order / NN < MAX_PERCENTILE_RHO:
cluster_id[item_idx] = n_cluster
print ic, item_idx, item_rho_order, cluster_id[item_idx]
n_cluster += 1
print "Found %d clusters" % n_cluster
for x in xrange(NN):
if cluster_id[x] >= 0:
print "XC", x, cluster_id[x], rho[x], delta[x]
self.cluster_id_maxima = cluster_id.deep_copy()
P = Profiler("6. assign all points")
R.cluster_assignment(rho_order, cluster_id)
self.cluster_id_full = cluster_id.deep_copy()
# assign the halos
P = Profiler("7. assign halos")
halo = flex.bool(NN, False)
border = R.get_border(cluster_id=cluster_id)
for ic in range(n_cluster
): #loop thru all border regions; find highest density
print "cluster", ic, "in border", border.count(True)
this_border = (cluster_id == ic) & (border == True)
print len(this_border), this_border.count(True)
if this_border.count(True) > 0:
highest_density = flex.max(rho.select(this_border))
halo_selection = (rho < highest_density) & (this_border
== True)
if halo_selection.count(True) > 0:
cluster_id.set_selected(halo_selection, -1)
core_selection = (cluster_id == ic) & ~halo_selection
highest_density = flex.max(rho.select(core_selection))
too_sparse = core_selection & (
rho.as_double() < highest_density / 10.
) # another heuristic
if too_sparse.count(True) > 0:
cluster_id.set_selected(too_sparse, -1)
self.cluster_id_final = cluster_id.deep_copy()
print "%d in the excluded halo" % ((cluster_id == -1).count(True))
def generate_view_data (self) :
from scitbx.array_family import flex
from scitbx import graphics_utils
settings = self.settings
data_for_colors = data_for_radii = None
data = self.data #self.work_array.data()
if (isinstance(data, flex.double) and data.all_eq(0)):
data = flex.double(data.size(), 1)
if ((self.multiplicities is not None) and
(settings.scale_colors_multiplicity)) :
data_for_colors = self.multiplicities.data().as_double()
assert data_for_colors.size() == data.size()
elif (settings.sqrt_scale_colors) and (isinstance(data, flex.double)) :
data_for_colors = flex.sqrt(data)
else :
data_for_colors = data.deep_copy()
if ((self.multiplicities is not None) and
(settings.scale_radii_multiplicity)) :
#data_for_radii = data.deep_copy()
data_for_radii = self.multiplicities.data().as_double()
assert data_for_radii.size() == data.size()
elif (settings.sqrt_scale_radii) and (isinstance(data, flex.double)) :
data_for_radii = flex.sqrt(data)
else :
data_for_radii = data.deep_copy()
if (settings.slice_mode) :
data = data.select(self.slice_selection)
if (not settings.keep_constant_scale) :
data_for_radii = data_for_radii.select(self.slice_selection)
data_for_colors = data_for_colors.select(self.slice_selection)
if (settings.color_scheme in ["rainbow", "heatmap", "redblue"]) :
colors = graphics_utils.color_by_property(
properties=data_for_colors,
selection=flex.bool(data_for_colors.size(), True),
color_all=False,
gradient_type=settings.color_scheme)
elif (settings.color_scheme == "grayscale") :
colors = graphics_utils.grayscale_by_property(
properties=data_for_colors,
selection=flex.bool(data_for_colors.size(), True),
shade_all=False,
invert=settings.black_background)
else :
if (settings.black_background) :
base_color = (1.0,1.0,1.0)
else :
base_color = (0.0,0.0,0.0)
colors = flex.vec3_double(data_for_colors.size(), base_color)
if (settings.slice_mode) and (settings.keep_constant_scale) :
colors = colors.select(self.slice_selection)
data_for_radii = data_for_radii.select(self.slice_selection)
uc = self.work_array.unit_cell()
abc = uc.parameters()[0:3]
min_dist = min(uc.reciprocal_space_vector((1,1,1)))
min_radius = 0.20 * min_dist
max_radius = 40 * min_dist
if (settings.sqrt_scale_radii) and (not settings.scale_radii_multiplicity):
data_for_radii = flex.sqrt(data_for_radii)
if len(data_for_radii):
max_value = flex.max(data_for_radii)
scale = max_radius / max_value
radii = data_for_radii * scale
too_small = radii < min_radius
if (too_small.count(True) > 0) :
radii.set_selected(too_small, flex.double(radii.size(), min_radius))
assert radii.size() == colors.size()
else:
radii = flex.double()
max_radius = 0
self.radii = radii
self.max_radius = max_radius
self.colors = colors
def test_1():
tstart = time.time()
fraction_missing = 0.1
d_min = 1.5
# create dummy model
symmetry = crystal.symmetry(unit_cell=(15.67, 25.37, 35.68, 90, 90, 90),
space_group_symbol="P 21 21 21")
structure = xray.structure(crystal_symmetry=symmetry)
for k in xrange(1000):
scatterer = xray.scatterer(
site=((1. + k * abs(math.sin(k))) / 1000.0,
(1. + k * abs(math.cos(k))) / 1000.0, (1. + k) / 1000.0),
u=abs(math.cos(k)) * 100. / (8. * math.pi**2),
occupancy=1.0,
scattering_type="C")
structure.add_scatterer(scatterer)
# partial model
n_keep = int(round(structure.scatterers().size() * (1 - fraction_missing)))
partial_structure = xray.structure(special_position_settings=structure)
partial_structure.add_scatterers(structure.scatterers()[:n_keep])
# fcalc (partial model), fobs (fcalc full model)
f_calc = structure.structure_factors(d_min=d_min,
anomalous_flag=False).f_calc()
f_calc_partial = partial_structure.structure_factors(
d_min=d_min, anomalous_flag=False).f_calc()
f_obs = abs(f_calc)
f_calc = abs(f_calc_partial)
d_star_sq = 1. / flex.pow2(f_obs.d_spacings().data())
assert approx_equal(flex.max(f_calc.data()), 6810.19834824)
assert approx_equal(flex.min(f_calc.data()), 0.019589341727)
assert approx_equal(flex.mean(f_calc.data()), 76.651506629)
assert approx_equal(flex.max(f_obs.data()), 6962.58343229)
assert approx_equal(flex.min(f_obs.data()), 0.00111552904935)
assert approx_equal(flex.mean(f_obs.data()), 74.5148786464)
assert f_obs.size() == f_calc.size()
# define test set reflections
flags = flex.bool(f_calc_partial.indices().size(), False)
k = 0
for i in xrange(f_calc_partial.indices().size()):
k = k + 1
if (k != 10):
flags[i] = False
else:
k = 0
flags[i] = True
assert flags.count(True) == 250
assert flags.count(False) == 2258
assert flags.size() == 2508
# *********************************************************TEST = 1
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=f_obs,
f_calc=f_calc,
free_reflections_per_bin=1000,
flags=flags,
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.data().size() == beta.data().size()
assert alpha.data().size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 0.914152454693)
assert approx_equal(flex.max(alpha.data()), 0.914152454693)
assert approx_equal(flex.min(beta.data()), 818.503411782)
assert approx_equal(flex.max(beta.data()), 818.503411782)
# *********************************************************TEST = 2
alpha, beta = maxlik.alpha_beta_est_manager(
f_obs=f_obs,
f_calc=f_calc,
free_reflections_per_bin=50,
flags=flags,
interpolation=False,
epsilons=f_obs.epsilons().data().as_double()).alpha_beta()
assert alpha.data().size() == beta.data().size()
assert alpha.data().size() == f_obs.size()
assert approx_equal(flex.min(alpha.data()), 0.910350007113)
assert approx_equal(flex.max(alpha.data()), 1.07104387776)
assert approx_equal(flex.min(beta.data()), 21.7374310013)
assert approx_equal(flex.max(beta.data()), 4222.81104745)
# *********************************************************TEST = 3
alpha, beta = maxlik.alpha_beta_calc(f=f_obs,
n_atoms_absent=100,
n_atoms_included=900,
bf_atoms_absent=25.0,
final_error=0.0,
absent_atom_type="C").alpha_beta()
fom = max_lik.fom_and_phase_error(
f_obs=f_obs.data(),
f_model=flex.abs(f_calc.data()),
alpha=alpha.data(),
beta=beta.data(),
epsilons=f_obs.epsilons().data().as_double(),
centric_flags=f_obs.centric_flags().data()).fom()
assert flex.max(fom) <= 1.0
assert flex.min(fom) >= 0.0
assert flex.min(alpha.data()) == flex.max(alpha.data()) == 1.0
assert approx_equal(flex.min(beta.data()), 7.964134920)
assert approx_equal(flex.max(beta.data()), 13695.1589364)
# *********************************************************TEST = 4
xs = crystal.symmetry((3, 4, 5), "P 2 2 2")
mi = flex.miller_index(((1, -2, 3), (0, 0, -4)))
ms = miller.set(xs, mi)
fc = flex.double((1., 2.))
fo = flex.double((1., 2.))
mso = miller.set(xs, mi)
mac = miller.array(ms, fc)
mao = miller.array(ms, fo)
alp = flex.double(2, 0.0)
bet = flex.double(2, 1e+9)
fom = max_lik.fom_and_phase_error(
f_obs=mao.data(),
f_model=mac.data(),
alpha=alp,
beta=bet,
epsilons=mao.epsilons().data().as_double(),
centric_flags=mao.centric_flags().data()).fom()
assert approx_equal(fom, [0.0, 0.0])
alp = flex.double(2, 1.0)
bet = flex.double(2, 1e+9)
fom = max_lik.fom_and_phase_error(
f_obs=mao.data(),
f_model=mac.data(),
alpha=alp,
beta=bet,
epsilons=mao.epsilons().data().as_double(),
centric_flags=mao.centric_flags().data()).fom()
assert approx_equal(fom, [0.0, 0.0])
alp = flex.double(2, 0.0)
bet = flex.double(2, 1e-9)
fom = max_lik.fom_and_phase_error(
f_obs=mao.data(),
f_model=mac.data(),
alpha=alp,
beta=bet,
epsilons=mao.epsilons().data().as_double(),
centric_flags=mao.centric_flags().data()).fom()
assert approx_equal(fom, [0.0, 0.0])
alp = flex.double(2, 1.0)
bet = flex.double(2, 1e-9)
fom = max_lik.fom_and_phase_error(
f_obs=mao.data(),
f_model=mac.data(),
alpha=alp,
beta=bet,
epsilons=mao.epsilons().data().as_double(),
centric_flags=mao.centric_flags().data()).fom()
assert approx_equal(fom, [1.0, 1.0])
def run(args):
import libtbx.load_env
usage = "%s [options]" % libtbx.env.dispatcher_name
parser = OptionParser(usage=usage,
phil=phil_scope,
check_format=False,
epilog=help_message)
params, options, args = parser.parse_args(show_diff_phil=True,
return_unhandled=True)
assert len(args) == 1
from iotbx.reflection_file_reader import any_reflection_file
intensities = None
batches = None
f = args[0]
reader = any_reflection_file(f)
arrays = reader.as_miller_arrays(merge_equivalents=False)
for ma in arrays:
print(ma.info().labels)
if ma.info().labels == ["I", "SIGI"]:
intensities = ma
elif ma.info().labels == ["IMEAN", "SIGIMEAN"]:
intensities = ma
elif ma.info().labels == ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]:
intensities = ma
elif ma.info().labels == ["BATCH"]:
batches = ma
assert intensities is not None
assert batches is not None
if params.d_min is not None:
intensities = intensities.resolution_filter(
d_min=params.d_min).set_info(intensities.info())
refined = refinery(intensities, batches)
refined.plot_scales()
scaled_unmerged = refined.apply_scales()
scales = refined.get_scales()
assert reader.file_type() == "ccp4_mtz"
mtz_obj = reader.file_content()
mtz_dataset = mtz_obj.crystals()[0].datasets()[0]
labels = intensities.info().labels
i_column = mtz_dataset.columns()[mtz_dataset.column_labels().index(
labels[0])]
sigi_column = mtz_dataset.columns()[mtz_dataset.column_labels().index(
labels[1])]
selection_valid = flex.bool(scaled_unmerged.size(), True)
i_column.set_values(scaled_unmerged.data().as_float(),
selection_valid=selection_valid)
sigi_column.set_values(scaled_unmerged.sigmas().as_float(),
selection_valid=selection_valid)
mtz_dataset.add_column("SCALEUSED", "R").set_values(scales.as_float())
mtz_out = "scaled_unmerged.mtz"
print("Writing scaled unmerged intensities to %s" % mtz_out)
mtz_obj.write(mtz_out)
return
def add(model,
use_neutron_distances=False,
adp_scale=1,
exclude_water=True,
protein_only=False,
stop_for_unknowns=True,
remove_first=True):
if(remove_first):
model = model.select(~model.get_hd_selection())
pdb_hierarchy = model.get_hierarchy()
mon_lib_srv = model.get_mon_lib_srv()
get_class = iotbx.pdb.common_residue_names_get_class
"""
for pmodel in pdb_hierarchy.models():
for chain in pmodel.chains():
for residue_group in chain.residue_groups():
for conformer in residue_group.conformers():
for residue in conformer.residues():
print list(residue.atoms().extract_name())
"""
#XXX This breaks for 1jxt, residue 2, TYR
for chain in pdb_hierarchy.only_model().chains():
for rg in chain.residue_groups():
for ag in rg.atom_groups():
#print list(ag.atoms().extract_name())
if(get_class(name=ag.resname) == "common_water"): continue
if(protein_only and
not ag.resname.strip().upper() in aa_codes): continue
actual = [a.name.strip().upper() for a in ag.atoms()]
mlq = mon_lib_query(residue=ag.resname, mon_lib_srv=mon_lib_srv)
expected_h = []
for k, v in six.iteritems(mlq.atom_dict()):
if(v.type_symbol=="H"): expected_h.append(k)
missing_h = list(set(expected_h).difference(set(actual)))
if 0: print(ag.resname, missing_h)
new_xyz = ag.atoms().extract_xyz().mean()
hetero = ag.atoms()[0].hetero
for mh in missing_h:
# TODO: this should be probably in a central place
if len(mh) < 4: mh = (' ' + mh).ljust(4)
a = (iotbx.pdb.hierarchy.atom()
.set_name(new_name=mh)
.set_element(new_element="H")
.set_xyz(new_xyz=new_xyz)
.set_hetero(new_hetero=hetero))
ag.append_atom(a)
pdb_hierarchy.atoms().reset_serial()
#pdb_hierarchy.sort_atoms_in_place()
p = mmtbx.model.manager.get_default_pdb_interpretation_params()
p.pdb_interpretation.clash_guard.nonbonded_distance_threshold=None
p.pdb_interpretation.use_neutron_distances = use_neutron_distances
p.pdb_interpretation.proceed_with_excessive_length_bonds=True
#p.pdb_interpretation.restraints_library.cdl=False # XXX this triggers a bug !=360
ro = model.get_restraint_objects()
model = mmtbx.model.manager(
model_input = None,
pdb_hierarchy = pdb_hierarchy,
build_grm = True,
stop_for_unknowns = stop_for_unknowns,
crystal_symmetry = model.crystal_symmetry(),
restraint_objects = ro,
pdb_interpretation_params = p,
log = null_out())
# # Remove lone H
# sel_h = model.get_hd_selection()
# sel_isolated = model.isolated_atoms_selection()
# sel_lone = sel_h & sel_isolated
# model = model.select(~sel_lone)
# Only keep H which have been parameterized in riding H procedure
sel_h = model.get_hd_selection()
model.setup_riding_h_manager()
sel_h_in_para = flex.bool(
[bool(x) for x in model.riding_h_manager.h_parameterization])
sel_h_not_in_para = sel_h_in_para.exclusive_or(sel_h)
model = model.select(~sel_h_not_in_para)
model = exclude_h_on_SS(model = model)
model = exclude_h_on_coordinated_S(model = model)
# Reset occupancies, ADPs and idealize
model.reset_adp_for_hydrogens(scale = adp_scale)
model.reset_occupancy_for_hydrogens_simple()
model.idealize_h_riding()
return model
def __init__(self,
model,
reference_hierarchy_list=None,
reference_file_list=None,
params=None,
selection=None,
log=None):
assert [reference_hierarchy_list,
reference_file_list].count(None) == 1
if(log is None):
log = sys.stdout
# self.model = model
self.params = params
self.selection = selection
self.mon_lib_srv = model.get_mon_lib_srv()
self.ener_lib = model.get_ener_lib()
self.pdb_hierarchy = model.get_hierarchy()
self.pdb_hierarchy.reset_i_seq_if_necessary()
sites_cart = self.pdb_hierarchy.atoms().extract_xyz()
if self.selection is None:
self.selection = flex.bool(len(sites_cart), True)
if reference_hierarchy_list is None:
reference_hierarchy_list = \
utils.process_reference_files(
reference_file_list=reference_file_list,
log=log)
if reference_file_list is None:
reference_file_list = \
["ref%d" % x for x in range(len(reference_hierarchy_list))]
#
# this takes 20% of constructor time.
self.dihedral_proxies_ref = utils.get_reference_dihedral_proxies(
reference_hierarchy_list=reference_hierarchy_list,
reference_file_list=reference_file_list,
mon_lib_srv=self.mon_lib_srv,
ener_lib=self.ener_lib,
restraint_objects=model.get_restraint_objects(),
monomer_parameters=model.get_monomer_parameters(),
log=log)
self.i_seq_name_hash = utils.build_name_hash(
pdb_hierarchy=self.pdb_hierarchy)
#reference model components
self.sites_cart_ref = {}
self.pdb_hierarchy_ref = {}
self.i_seq_name_hash_ref = {}
self.reference_dihedral_hash = {}
self.reference_file_list = reference_file_list
#triage reference model files
for file, hierarchy in zip(reference_file_list,
reference_hierarchy_list):
self.sites_cart_ref[file] = hierarchy.atoms().extract_xyz()
self.pdb_hierarchy_ref[file] = hierarchy
self.i_seq_name_hash_ref[file] = \
utils.build_name_hash(
pdb_hierarchy=hierarchy)
self.reference_dihedral_hash[file] = \
self.build_dihedral_hash(
dihedral_proxies=self.dihedral_proxies_ref[file],
sites_cart=self.sites_cart_ref[file],
pdb_hierarchy=hierarchy,
include_hydrogens=self.params.hydrogens,
include_main_chain=self.params.main_chain,
include_side_chain=self.params.side_chain)
self.match_map = None
self.proxy_map = None
self.build_reference_dihedral_proxy_hash()
#
# This takes 80% of constructor time!!!
self.residue_match_hash = {} # {key_model: ('file_name', key_ref)}
self.match_map = {} # {'file_name':{i_seq_model:i_seq_ref}}
if params.use_starting_model_as_reference:
self.get_matching_from_self()
else:
self.get_matching_from_ncs(log=log)
if self.match_map == {}:
# making empty container
new_ref_dih_proxies = self.reference_dihedral_proxies = \
cctbx.geometry_restraints.shared_dihedral_proxy()
else:
new_ref_dih_proxies = self.get_reference_dihedral_proxies(model)
