def exercise():
from cctbx import xray
from cctbx import crystal
from cctbx.array_family import flex
sc = xray.scatterer
xs = xray.structure(
# COD entry 2103456, selected atoms
crystal_symmetry=crystal.symmetry(unit_cell=(10.1053, 10.1053, 10.1053,
90, 90, 90),
space_group_symbol="P 21 3"),
scatterers=flex.xray_scatterer((sc(label='ER1',
site=(-0.16698, 0.66698, 0.33302)),
sc(label='ER2',
site=(0.39957, 0.60043, 0.10043)))))
#
from iotbx.shelx import fvar_encoding
xsm = fvar_encoding.move_sites_if_necessary_for_shelx_fvar_encoding(xs)
from six.moves import cStringIO as StringIO
sio = StringIO()
xsm.show_scatterers(f=sio)
from libtbx.test_utils import show_diff
assert not show_diff(
sio.getvalue(), """\
Label, Scattering, Multiplicity, Coordinates, Occupancy, Uiso, Ustar as Uiso
ER1 Er 4 (-0.1670 -0.3330 0.3330) 1.00 0.0000 [ - ]
ER2 Er 4 ( 0.3996 -0.3996 0.1004) 1.00 0.0000 [ - ]
""")
#
from iotbx.shelx.errors import error
from libtbx.test_utils import Exception_expected, approx_equal
try:
fvar_encoding.dev_build_shelx76_fvars(xs)
except error:
pass
else:
raise Exception_expected
#
fvars, encoded_sites = fvar_encoding.dev_build_shelx76_fvars(xsm)
assert approx_equal(fvars, [1, 0.33396, 0.79914])
assert approx_equal(encoded_sites,
[[19.5, -19.5, -20.5], [30.5, 29.5, -30.5]])
def process(params, pickle_file_name):
cod_id = op.basename(pickle_file_name).split(".", 1)[0]
print "cod_id:", cod_id
c_obs, structure_prep, edge_list = easy_pickle.load(
file_name=pickle_file_name)
changes = structure_prep.make_scatterer_labels_shelx_compatible_in_place()
if (params.sites_mod_short):
structure_prep = structure_prep.sites_mod_short()
from iotbx.shelx import fvar_encoding
structure_prep = \
fvar_encoding.move_sites_if_necessary_for_shelx_fvar_encoding(
xray_structure=structure_prep)
structure_prep.show_summary().show_scatterers()
if (len(changes) != 0):
from libtbx.utils import plural_s
print "INFO: %d atom name%s changed for compatibility with SHELXL:" \
% plural_s(len(changes))
for change in changes:
print ' changed: "%s" -> "%s"' % change
structure_prep.scattering_type_registry(table="it1992").show()
fvar_encoding.dev_build_shelx76_fvars(structure_prep) # only an exercise
print "." * 79
#
if (len(params.optimizers) == 0):
return
#
assert c_obs.is_xray_intensity_array() or c_obs.is_xray_amplitude_array()
if (c_obs.is_xray_intensity_array()):
i_obs = c_obs
f_obs = c_obs.f_sq_as_f(algorithm="xtal_3_7")
else:
f_obs = c_obs
i_obs = c_obs.f_as_f_sq(algorithm="shelxl")
process_continue(params=params,
cod_id=cod_id,
c_obs=c_obs,
i_obs=i_obs,
f_obs=f_obs,
structure_prep=structure_prep)
def process(params, pickle_file_name):
cod_id = op.basename(pickle_file_name).split(".",1)[0]
print "cod_id:", cod_id
c_obs, structure_prep, edge_list = easy_pickle.load(
file_name=pickle_file_name)
changes = structure_prep.make_scatterer_labels_shelx_compatible_in_place()
if (params.sites_mod_short):
structure_prep = structure_prep.sites_mod_short()
from iotbx.shelx import fvar_encoding
structure_prep = \
fvar_encoding.move_sites_if_necessary_for_shelx_fvar_encoding(
xray_structure=structure_prep)
structure_prep.show_summary().show_scatterers()
if (len(changes) != 0):
from libtbx.utils import plural_s
print "INFO: %d atom name%s changed for compatibility with SHELXL:" \
% plural_s(len(changes))
for change in changes:
print ' changed: "%s" -> "%s"' % change
structure_prep.scattering_type_registry(table="it1992").show()
fvar_encoding.dev_build_shelx76_fvars(structure_prep) # only an exercise
print "."*79
#
if (len(params.optimizers) == 0):
return
#
assert c_obs.is_xray_intensity_array() or c_obs.is_xray_amplitude_array()
if (c_obs.is_xray_intensity_array()):
i_obs = c_obs
f_obs = c_obs.f_sq_as_f(algorithm="xtal_3_7")
else:
f_obs = c_obs
i_obs = c_obs.f_as_f_sq(algorithm="shelxl")
process_continue(
params=params,
cod_id=cod_id,
c_obs=c_obs, i_obs=i_obs, f_obs=f_obs,
structure_prep=structure_prep)
def exercise():
from cctbx import xray
from cctbx import crystal
from cctbx.array_family import flex
sc = xray.scatterer
xs = xray.structure(
# COD entry 2103456, selected atoms
crystal_symmetry=crystal.symmetry(
unit_cell=(10.1053, 10.1053, 10.1053, 90, 90, 90),
space_group_symbol="P 21 3"),
scatterers=flex.xray_scatterer((
sc(label='ER1', site=(-0.16698, 0.66698, 0.33302)),
sc(label='ER2', site=(0.39957, 0.60043, 0.10043)))))
#
from iotbx.shelx import fvar_encoding
xsm = fvar_encoding.move_sites_if_necessary_for_shelx_fvar_encoding(xs)
from cStringIO import StringIO
sio = StringIO()
xsm.show_scatterers(f=sio)
from libtbx.test_utils import show_diff
assert not show_diff(sio.getvalue(), """\
Label, Scattering, Multiplicity, Coordinates, Occupancy, Uiso, Ustar as Uiso
ER1 Er 4 (-0.1670 -0.3330 0.3330) 1.00 0.0000 [ - ]
ER2 Er 4 ( 0.3996 -0.3996 0.1004) 1.00 0.0000 [ - ]
""")
#
from iotbx.shelx.errors import error
from libtbx.test_utils import Exception_expected, approx_equal
try: fvar_encoding.dev_build_shelx76_fvars(xs)
except error: pass
else: raise Exception_expected
#
fvars, encoded_sites = fvar_encoding.dev_build_shelx76_fvars(xsm)
assert approx_equal(fvars, [1, 0.33396, 0.79914])
assert approx_equal(encoded_sites,
[[19.5, -19.5, -20.5], [30.5, 29.5, -30.5]])
def process_continue(params, cod_id, c_obs, i_obs, f_obs, structure_prep):
p = params.f_calc_options
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure=structure_prep,
algorithm=p.algorithm,
cos_sin_table=p.cos_sin_table).f_calc()
sel = f_obs.f_obs_f_calc_fan_outlier_selection(f_calc=f_calc)
assert sel is not None
n_outliers = sel.count(True)
if (n_outliers != 0):
action = params.f_obs_f_calc_fan_outliers
print "INFO: f_obs_f_calc_fan_outliers = %s: %d" % (action, n_outliers)
if (action == "remove"):
i_obs = i_obs.select(~sel)
f_obs = f_obs.select(~sel)
if (f_obs.anomalous_flag()):
print "INFO: converting anomalous i+f_obs to non-anomalous."
i_obs = i_obs.average_bijvoet_mates()
f_obs = f_obs.average_bijvoet_mates()
sel = ((i_obs.data() == 0) & (i_obs.sigmas() == 0)) \
| ((f_obs.data() == 0) & (f_obs.sigmas() == 0))
n_zero_d_and_s = sel.count(True)
if (n_zero_d_and_s != 0):
print "INFO: removing reflections with i+f_obs=0 and sigma=0:", \
n_zero_d_and_s
i_obs = i_obs.select(~sel)
f_obs = f_obs.select(~sel)
p = params.f_calc_options
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure=structure_prep,
algorithm=p.algorithm,
cos_sin_table=p.cos_sin_table).f_calc()
if (params.use_f_calc_as_f_obs):
print "INFO: using f_calc as i+f_obs"
i_obs = f_calc.intensities().customized_copy(
sigmas=flex.double(f_calc.indices().size(), 0.01))
f_obs = f_calc.amplitudes().customized_copy(
sigmas=flex.double(f_calc.indices().size(), 0.01))
else:
# scaling applied so that the data written in shelx hklf format
# have sufficient significant digits, and FVAR is 1 (shelx76 seems
# to be especially sensitive to FVAR >> 1)
k = f_obs.scale_factor(f_calc=f_calc)
assert k != 0
s = 1 / k**2
print "INFO: scaling i_obs to f_calc by multiplying i_obs with: %.6g" % s
i_obs = i_obs.apply_scaling(factor=s)
s = 1 / k
print "INFO: scaling f_obs to f_calc by multiplying f_obs with: %.6g" % s
f_obs = f_obs.apply_scaling(factor=s)
def show(obs):
obs.show_comprehensive_summary()
from cif_refine import \
report_fraction_of_negative_observations_if_any as _
_(cod_id, obs)
if (c_obs.is_xray_intensity_array()):
show(i_obs)
else:
show(f_obs)
print "." * 79
#
structure_work = structure_prep.deep_copy_scatterers()
sel = structure_work.hd_selection()
print "Removing hydrogen atoms:", sel.count(True)
structure_work = structure_work.select(selection=~sel)
sdt = params.show_distances_threshold
if (sdt > 0):
print "Distances smaller than %.6g A:" % sdt
structure_work.show_distances(distance_cutoff=sdt)
print "." * 79
#
if (params.tardy_samples.iq is not None):
from cctbx.omz import tardy_adaptor
print
tardy_adaptor.sample_e_pot(id_code=cod_id,
f_obs=f_obs,
xray_structure=structure_prep,
edge_list=edge_list,
params=params.tardy_samples)
print
return
#
from iotbx.shelx import fvar_encoding
fvars, encoded_sites = fvar_encoding.dev_build_shelx76_fvars(
structure_work)
print "Number of FVARs for special position constraints:", len(fvars) - 1
print "." * 79
#
show_cc_r1(params, "prep", f_obs, structure_prep)
def cc_r1(label):
show_cc_r1(params, label, f_obs, structure_work)
cc_r1("no_h")
structure_work.convert_to_isotropic()
cc_r1("iso")
structure_iso = structure_work.deep_copy_scatterers()
#
if (params.reset_u_iso is not None):
structure_work.set_u_iso(value=params.reset_u_iso)
cc_r1("setu")
if (params.shake_sites_rmsd is not None):
mt = flex.mersenne_twister(seed=0)
structure_work.shift_sites_in_place(
shift_length=params.shake_sites_rmsd, mersenne_twister=mt)
print "rms difference after shift_sites_in_place: %.3f" \
% structure_iso.rms_difference(structure_work)
cc_r1("shift_xyz")
#
if (params.max_atoms is not None):
n = structure_work.scatterers().size()
if (n > params.max_atoms):
print "Skipping refinement of large model: %d atoms COD %s" % (
n, cod_id)
return
#
structure_work.scatterers().flags_set_grads(state=False)
for sc in structure_work.scatterers():
sc.flags.set_grad_site(True)
assert sc.flags.use_u_iso_only()
sc.flags.set_grad_u_iso(True)
n_refinable_parameters = structure_work.n_parameters(
considering_site_symmetry_constraints=True)
print "Number of refinable parameters:", n_refinable_parameters
#
if (params.iteration_limit < 1):
return
#
if ("dev" not in params.optimizers):
structure_dev = None
else:
structure_dev = structure_work.deep_copy_scatterers()
omz.dev.refinement(
i_obs=i_obs,
f_obs=f_obs,
xray_structure=structure_dev,
params=params,
reference_structure=structure_iso,
expected_n_refinable_parameters=n_refinable_parameters,
plot_samples_id=cod_id)
show_cc_r1(params, "dev", f_obs, structure_dev)
if (params.export_refined):
file_name = "dev_%s_%s_%s.pdb" % (
params.target_type, params.target_obs_type.lower(), cod_id)
open(file_name,
"w").write(structure_dev.as_pdb_file(remarks=[file_name]))
if (params.pickle_refined_dir is not None):
easy_pickle.dump(file_name=op.join(params.pickle_refined_dir,
cod_id + ".pickle"),
obj=(c_obs, structure_dev, None))
print >> open("%s/qi_%s" % (params.pickle_refined_dir, cod_id),
"w"), (structure_dev.scatterers().size(),
c_obs.space_group().order_p(),
c_obs.indices().size(), c_obs.d_min())
#
def use_smtbx_ls(mode):
if ("ls_" + mode not in params.optimizers):
return None
if (not libtbx.env.has_module(name="smtbx")):
print "INFO: smtbx not available: refinement skipped."
return None
result = structure_work.deep_copy_scatterers()
run_smtbx_ls(mode=mode,
cod_id=cod_id,
i_obs=i_obs,
f_obs=f_obs,
xray_structure=result,
params=params)
show_cc_r1(params, "ls_" + mode, f_obs, result)
return result
structure_ls_simple = use_smtbx_ls("simple")
structure_ls_lm = use_smtbx_ls("lm")
#
def use_shelxl(mode):
if ("shelxl_" + mode not in params.optimizers):
return None
result = structure_work.deep_copy_scatterers()
run_shelxl(mode=mode,
cod_id=cod_id,
i_obs=i_obs,
f_obs=f_obs,
xray_structure=result,
params=params,
reference_structure=structure_iso,
expected_n_refinable_parameters=n_refinable_parameters)
if (params.export_refined):
file_name = "shelxl_%s_%s.pdb" % (mode, cod_id)
open(file_name, "w").write(result.as_pdb_file(remarks=[file_name]))
return result
structure_shelxl_fm = use_shelxl("fm")
structure_shelxl_cg = use_shelxl("cg")
#
if ("shelx76" not in params.optimizers):
structure_shelx76 = None
else:
structure_shelx76 = structure_work.deep_copy_scatterers()
run_shelx76(cod_id=cod_id,
f_obs=f_obs,
xray_structure=structure_shelx76,
fvars=fvars,
encoded_sites=encoded_sites,
params=params,
reference_structure=structure_iso)
if (params.export_refined):
file_name = "shelx76_%s.pdb" % cod_id
open(file_name,
"w").write(structure_shelx76.as_pdb_file(remarks=[file_name]))
def process_continue(params, cod_id, c_obs, i_obs, f_obs, structure_prep):
p = params.f_calc_options
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure=structure_prep,
algorithm=p.algorithm,
cos_sin_table=p.cos_sin_table).f_calc()
sel = f_obs.f_obs_f_calc_fan_outlier_selection(f_calc=f_calc)
assert sel is not None
n_outliers = sel.count(True)
if (n_outliers != 0):
action = params.f_obs_f_calc_fan_outliers
print "INFO: f_obs_f_calc_fan_outliers = %s: %d" % (action, n_outliers)
if (action == "remove"):
i_obs = i_obs.select(~sel)
f_obs = f_obs.select(~sel)
if (f_obs.anomalous_flag()):
print "INFO: converting anomalous i+f_obs to non-anomalous."
i_obs = i_obs.average_bijvoet_mates()
f_obs = f_obs.average_bijvoet_mates()
sel = ((i_obs.data() == 0) & (i_obs.sigmas() == 0)) \
| ((f_obs.data() == 0) & (f_obs.sigmas() == 0))
n_zero_d_and_s = sel.count(True)
if (n_zero_d_and_s != 0):
print "INFO: removing reflections with i+f_obs=0 and sigma=0:", \
n_zero_d_and_s
i_obs = i_obs.select(~sel)
f_obs = f_obs.select(~sel)
p = params.f_calc_options
f_calc = f_obs.structure_factors_from_scatterers(
xray_structure=structure_prep,
algorithm=p.algorithm,
cos_sin_table=p.cos_sin_table).f_calc()
if (params.use_f_calc_as_f_obs):
print "INFO: using f_calc as i+f_obs"
i_obs = f_calc.intensities().customized_copy(
sigmas=flex.double(f_calc.indices().size(), 0.01))
f_obs = f_calc.amplitudes().customized_copy(
sigmas=flex.double(f_calc.indices().size(), 0.01))
else:
# scaling applied so that the data written in shelx hklf format
# have sufficient significant digits, and FVAR is 1 (shelx76 seems
# to be especially sensitive to FVAR >> 1)
k = f_obs.scale_factor(f_calc=f_calc)
assert k != 0
s = 1/k**2
print "INFO: scaling i_obs to f_calc by multiplying i_obs with: %.6g" % s
i_obs = i_obs.apply_scaling(factor=s)
s = 1/k
print "INFO: scaling f_obs to f_calc by multiplying f_obs with: %.6g" % s
f_obs = f_obs.apply_scaling(factor=s)
def show(obs):
obs.show_comprehensive_summary()
from cif_refine import \
report_fraction_of_negative_observations_if_any as _
_(cod_id, obs)
if (c_obs.is_xray_intensity_array()):
show(i_obs)
else:
show(f_obs)
print "."*79
#
structure_work = structure_prep.deep_copy_scatterers()
sel = structure_work.hd_selection()
print "Removing hydrogen atoms:", sel.count(True)
structure_work = structure_work.select(selection=~sel)
sdt = params.show_distances_threshold
if (sdt > 0):
print "Distances smaller than %.6g A:" % sdt
structure_work.show_distances(distance_cutoff=sdt)
print "."*79
#
if (params.tardy_samples.iq is not None):
from cctbx.omz import tardy_adaptor
print
tardy_adaptor.sample_e_pot(
id_code=cod_id,
f_obs=f_obs,
xray_structure=structure_prep,
edge_list=edge_list,
params=params.tardy_samples)
print
return
#
from iotbx.shelx import fvar_encoding
fvars, encoded_sites = fvar_encoding.dev_build_shelx76_fvars(structure_work)
print "Number of FVARs for special position constraints:", len(fvars)-1
print "."*79
#
show_cc_r1(params, "prep", f_obs, structure_prep)
def cc_r1(label):
show_cc_r1(params, label, f_obs, structure_work)
cc_r1("no_h")
structure_work.convert_to_isotropic()
cc_r1("iso")
structure_iso = structure_work.deep_copy_scatterers()
#
if (params.reset_u_iso is not None):
structure_work.set_u_iso(value=params.reset_u_iso)
cc_r1("setu")
if (params.shake_sites_rmsd is not None):
mt = flex.mersenne_twister(seed=0)
structure_work.shift_sites_in_place(
shift_length=params.shake_sites_rmsd,
mersenne_twister=mt)
print "rms difference after shift_sites_in_place: %.3f" \
% structure_iso.rms_difference(structure_work)
cc_r1("shift_xyz")
#
if (params.max_atoms is not None):
n = structure_work.scatterers().size()
if (n > params.max_atoms):
print "Skipping refinement of large model: %d atoms COD %s" % (
n, cod_id)
return
#
structure_work.scatterers().flags_set_grads(state=False)
for sc in structure_work.scatterers():
sc.flags.set_grad_site(True)
assert sc.flags.use_u_iso_only()
sc.flags.set_grad_u_iso(True)
n_refinable_parameters = structure_work.n_parameters(
considering_site_symmetry_constraints=True)
print "Number of refinable parameters:", n_refinable_parameters
#
if (params.iteration_limit < 1):
return
#
if ("dev" not in params.optimizers):
structure_dev = None
else:
structure_dev = structure_work.deep_copy_scatterers()
omz.dev.refinement(
i_obs=i_obs,
f_obs=f_obs,
xray_structure=structure_dev,
params=params,
reference_structure=structure_iso,
expected_n_refinable_parameters=n_refinable_parameters,
plot_samples_id=cod_id)
show_cc_r1(params, "dev", f_obs, structure_dev)
if (params.export_refined):
file_name = "dev_%s_%s_%s.pdb" % (
params.target_type, params.target_obs_type.lower(), cod_id)
open(file_name, "w").write(structure_dev.as_pdb_file(
remarks=[file_name]))
if (params.pickle_refined_dir is not None):
easy_pickle.dump(
file_name=op.join(params.pickle_refined_dir, cod_id+".pickle"),
obj=(c_obs, structure_dev, None))
print >> open("%s/qi_%s" % (params.pickle_refined_dir, cod_id), "w"), (
structure_dev.scatterers().size(),
c_obs.space_group().order_p(),
c_obs.indices().size(),
c_obs.d_min())
#
def use_smtbx_ls(mode):
if ("ls_"+mode not in params.optimizers):
return None
if (not libtbx.env.has_module(name="smtbx")):
print "INFO: smtbx not available: refinement skipped."
return None
result = structure_work.deep_copy_scatterers()
run_smtbx_ls(
mode=mode,
cod_id=cod_id,
i_obs=i_obs,
f_obs=f_obs,
xray_structure=result,
params=params)
show_cc_r1(params, "ls_"+mode, f_obs, result)
return result
structure_ls_simple = use_smtbx_ls("simple")
structure_ls_lm = use_smtbx_ls("lm")
#
def use_shelxl(mode):
if ("shelxl_"+mode not in params.optimizers):
return None
result = structure_work.deep_copy_scatterers()
run_shelxl(
mode=mode,
cod_id=cod_id,
i_obs=i_obs,
f_obs=f_obs,
xray_structure=result,
params=params,
reference_structure=structure_iso,
expected_n_refinable_parameters=n_refinable_parameters)
if (params.export_refined):
file_name = "shelxl_%s_%s.pdb" % (mode, cod_id)
open(file_name, "w").write(result.as_pdb_file(
remarks=[file_name]))
return result
structure_shelxl_fm = use_shelxl("fm")
structure_shelxl_cg = use_shelxl("cg")
#
if ("shelx76" not in params.optimizers):
structure_shelx76 = None
else:
structure_shelx76 = structure_work.deep_copy_scatterers()
run_shelx76(
cod_id=cod_id,
f_obs=f_obs,
xray_structure=structure_shelx76,
fvars=fvars,
encoded_sites=encoded_sites,
params=params,
reference_structure=structure_iso)
if (params.export_refined):
file_name = "shelx76_%s.pdb" % cod_id
open(file_name, "w").write(structure_shelx76.as_pdb_file(
remarks=[file_name]))