def run(): random.seed(0) flex.set_random_seed(0) test_rosenbrock_function(3) random.seed(0) flex.set_random_seed(0) test_rosenbrock_function(4)
def run(args): from scitbx.array_family import flex import random random.seed(0) flex.set_random_seed(0) from cctbx.development import debug_utils debug_utils.parse_options_loop_space_groups(args, run_call_back)
def exercise_twin_detwin () :
random.seed(12345)
flex.set_random_seed(12345)
xrs = random_structure.xray_structure(
unit_cell=(12,5,12,90,90,90),
space_group_symbol="P1",
n_scatterers=12,
elements="random")
fc = abs(xrs.structure_factors(d_min=1.5).f_calc())
fc = fc.set_observation_type_xray_amplitude()
mtz_file = "tmp_massage_in.mtz"
fc.as_mtz_dataset(column_root_label="F").mtz_object().write(mtz_file)
massage_data.run(
args=[
mtz_file,
"aniso.action=None",
"outlier.action=None",
"symmetry.action=twin",
"twin_law='l,-k,h'",
"fraction=0.3",
"hklout=tmp_massage_twinned.mtz",
],
out=null_out())
assert op.isfile("tmp_massage_twinned.mtz")
mtz_in = file_reader.any_file("tmp_massage_twinned.mtz")
fc_twin = mtz_in.file_server.miller_arrays[0].f_sq_as_f()
fc_twin, fc_tmp = fc_twin.common_sets(other=fc)
for hkl, f1, f2 in zip(fc_tmp.indices(), fc_tmp.data(), fc_twin.data()) :
if (abs(hkl[0]) != abs(hkl[2])) :
assert not approx_equal(f1, f2, eps=0.01, out=null_out()), (hkl, f1, f2)
massage_data.run(
args=[
mtz_file,
"aniso.action=None",
"outlier.action=None",
"symmetry.action=twin",
"twin_law='l,-k,h'",
"fraction=0.3",
"hklout=tmp_massage_twinned.sca",
],
out=null_out())
assert op.isfile("tmp_massage_twinned.sca")
massage_data.run(
args=[
"tmp_massage_twinned.mtz",
"aniso.action=None",
"outlier.action=None",
"symmetry.action=detwin",
"twin_law='l,-k,h'",
"fraction=0.3",
"hklout=tmp_massage_detwinned.mtz",
],
out=null_out())
mtz_in = file_reader.any_file("tmp_massage_detwinned.mtz")
fc_detwin = mtz_in.file_server.miller_arrays[0].f_sq_as_f()
fc_detwin, fc_tmp = fc_detwin.common_sets(other=fc)
# XXX we appear to lose some accuracy here, possibly due to the use of
# MTZ format
for hkl, f1, f2 in zip(fc_tmp.indices(), fc_tmp.data(), fc_detwin.data()) :
assert approx_equal(f1, f2, eps=0.01), hkl
def run(self, random_seed):
tmp = self.xray_structure.deep_copy_scatterers()
# Shake and minimize to get variable starting points
tmp.shake_sites_in_place(
rms_difference = None,
mean_distance = 5)
sites_cart_ = tmp.sites_cart()
minimized = geometry_minimization.lbfgs(
sites_cart = sites_cart_,
correct_special_position_tolerance= 1.0,
geometry_restraints_manager = self.restraints_manager.geometry,
geometry_restraints_flags = self.grf,
lbfgs_exception_handling_params = self.lbfgs_exception_handling_params,
lbfgs_termination_params = scitbx.lbfgs.termination_parameters(
max_iterations=50))
tmp = tmp.replace_sites_cart(new_sites = sites_cart_)
#
random.seed(random_seed)
flex.set_random_seed(random_seed)
sa.run(
params = self.params,
xray_structure = tmp,
real_space = True,
target_map = self.map_data,
restraints_manager = self.restraints_manager,
wx = self.weight,
wc = 1.,
verbose = False)
return tmp
def run():
flex.set_random_seed(0)
for ii in xrange(10):
test_rosenbrock_function(1)
test_function(1)
test_function(2)
test_function(3)
test_function(4)
def __init__(self): # # FIXME!!!!! # Setting random seed to avoid test failure, need to take anothe look at # this to figure out why this is failing sometimes anyway. # from random import seed from scitbx.array_family import flex seed(0) flex.set_random_seed(0)
def plot_uc_3Dplot(self, info):
assert self.interactive
import numpy as np
from mpl_toolkits.mplot3d import Axes3D # import dependency
fig = self.plt.figure(figsize=(12, 10))
# Extract uc dimensions from info list
a = flex.double([i['a'] for i in info])
b = flex.double([i['b'] for i in info])
c = flex.double([i['c'] for i in info])
alpha = flex.double([i['alpha'] for i in info])
beta = flex.double([i['beta'] for i in info])
gamma = flex.double([i['gamma'] for i in info])
n_total = len(a)
accepted = flex.bool(n_total, True)
for d in [a, b, c, alpha, beta, gamma]:
outliers = self.reject_outliers(d)
accepted &= ~outliers
a = a.select(accepted)
b = b.select(accepted)
c = c.select(accepted)
AA = "a-edge (%.2f +/- %.2f $\AA$)" % (flex.mean(a),
flex.mean_and_variance(a).unweighted_sample_standard_deviation())
BB = "b-edge (%.2f +/- %.2f $\AA$)" % (flex.mean(b),
flex.mean_and_variance(b).unweighted_sample_standard_deviation())
CC = "c-edge (%.2f +/- %.2f $\AA$)" % (flex.mean(c),
flex.mean_and_variance(c).unweighted_sample_standard_deviation())
subset = min(len(a),1000)
flex.set_random_seed(123)
rnd_sel = flex.random_double(len(a))<(subset/n_total)
a = a.select(rnd_sel)
b = b.select(rnd_sel)
c = c.select(rnd_sel)
fig.suptitle('{} randomly selected cells out of total {} images'
''.format(len(a),n_total), fontsize=18)
ax = fig.add_subplot(111, projection='3d')
for ia in xrange(len(a)):
ax.scatter(a[ia],b[ia],c[ia],c='r',marker='+')
ax.set_xlabel(AA)
ax.set_ylabel(BB)
ax.set_zlabel(CC)
def run(
args,
map_data=None,
map_coeffs=None,
pdb_inp=None,
pdb_string=None,
crystal_symmetry=None,
params_edits=None,
out=sys.stdout,
):
# Get the parameters
params = get_params(args=args, out=out)
if params.control.random_seed:
import random
random.seed(params.control.random_seed)
flex.set_random_seed(params.control.random_seed)
print >> out, "\nUsing random seed of %d" % (params.control.random_seed)
# Get map_data if not present
map_data, map_coeffs, crystal_symmetry = get_map_data_and_symmetry(
crystal_symmetry=crystal_symmetry,
map_data=map_data,
map_coeffs=map_coeffs,
map_coeffs_file=params.input_files.map_coeffs_file,
map_coeffs_labels=params.input_files.map_coeffs_labels,
)
# Get the starting model
pdb_inp, cryst1_line, pdb_string = get_pdb_inp(
crystal_symmetry=crystal_symmetry, pdb_inp=pdb_inp, pdb_string=pdb_string, pdb_in=params.input_files.pdb_in
)
pdb_hierarchy, multiple_models_hierarchy = run_one_cycle(
params=params,
map_data=map_data,
pdb_inp=pdb_inp,
pdb_string=pdb_string,
crystal_symmetry=crystal_symmetry,
params_edits=params_edits,
out=out,
)
if params.output_files.pdb_out:
f = open(params.output_files.pdb_out, "w")
print >> f, cryst1_line
print >> f, pdb_hierarchy.as_pdb_string()
print >> out, "\nWrote output model to %s" % (params.output_files.pdb_out)
f.close()
# all done
return pdb_hierarchy, multiple_models_hierarchy
def exercise_space_group_handling () :
flex.set_random_seed(123456)
random.seed(123456)
base = "tst_cc_star_space_group"
pdb_in = iotbx.pdb.hierarchy.input(pdb_string=model_1yjp)
xrs = pdb_in.xray_structure_simple()
xrs.set_inelastic_form_factors(
photon=1.54,
table="sasaki")
fc = abs(xrs.structure_factors(d_min=1.5).f_calc()).average_bijvoet_mates()
fc.set_observation_type_xray_amplitude()
flags = fc.generate_r_free_flags()
mtz = fc.as_mtz_dataset(column_root_label="F")
mtz.add_miller_array(flags, column_root_label="FreeR_flag")
mtz.mtz_object().write(base + ".mtz")
xrs_p1 = xrs.expand_to_p1()
xrs_p1.shake_sites_in_place(rms_difference=0.1)
fc_p1 = xrs_p1.structure_factors(d_min=1.4).f_calc()
fc_p1_extra = fc_p1.randomize_amplitude_and_phase(amplitude_error=1.0,
phase_error_deg=0,
random_seed=123456)
fc_p1 = abs(fc_p1.concatenate(other=fc_p1_extra)).sort(
by_value="packed_indices")
fc_p1.set_observation_type_xray_amplitude()
sg_p2 = sgtbx.space_group_info("P2")
ic = fc_p1.f_as_f_sq().customized_copy(
space_group_info=sg_p2,
sigmas=flex.double(fc_p1.size(), 10.0))
ic.export_as_scalepack_unmerged(file_name=base + ".sca")
open(base + ".pdb", "w").write(model_1yjp)
args = [
base + ".mtz",
base + ".pdb",
"unmerged_data=%s.sca" % base,
]
cc_star.run(args=args, out=null_out())
# now with .sca in P1 (raises Sorry)
ic2 = fc_p1.f_as_f_sq().customized_copy(
sigmas=flex.double(fc_p1.size(), 10.0))
ic2.export_as_scalepack_unmerged(file_name=base + "_p1.sca")
args = [
base + ".mtz",
base + ".pdb",
"unmerged_data=%s_p1.sca" % base,
]
try :
cc_star.run(args=args, out=null_out())
except Sorry, s :
assert (str(s) == "Incompatible space groups in merged and unmerged data:P 1 21 1 versus P 1"), s
def exercise_1():
random.seed(0)
flex.set_random_seed(0)
pi = get_pdb_inputs(pdb_str=pdb_str_1)
f_obs = abs(pi.xrs.structure_factors(d_min = 2.5).f_calc())
r_free_flags = f_obs.generate_r_free_flags(use_lattice_symmetry=False)
if(0):
pi.ph.adopt_xray_structure(pi.xrs)
pi.ph.write_pdb_file(file_name="start.pdb",
crystal_symmetry = pi.xrs.crystal_symmetry())
xrs_poor = shake_sites(xrs = pi.xrs.deep_copy_scatterers(), random=False,
shift = 1.5, grm=pi.grm)
if(0):
pi.ph.adopt_xray_structure(xrs_poor)
pi.ph.write_pdb_file(file_name="poor.pdb",
crystal_symmetry = xrs_poor.crystal_symmetry())
fmodel = mmtbx.f_model.manager(
f_obs = f_obs,
r_free_flags = r_free_flags,
xray_structure = xrs_poor)
print "start r_work:", fmodel.r_work()
#
params = sa.master_params().extract()
params.start_temperature=3000
params.final_temperature=0
params.cool_rate = 100
params.number_of_steps = 100
params.update_grads_shift = 0.
#
sa.run(
params = params,
fmodel = fmodel,
restraints_manager = pi.grm,
wx = 20,
wc = 1,
verbose = True)
#
r = fmodel.r_work()
print "final r_work:", r
assert r < 0.03, r
dist = flex.mean(flex.sqrt((pi.xrs.sites_cart() -
fmodel.xray_structure.sites_cart()).dot()))
print "Distance(refined, answer): %6.4f"%dist
assert dist < 0.25, dist
if(0):
pi.ph.adopt_xray_structure(fmodel.xray_structure)
pi.ph.write_pdb_file(file_name="refined.pdb",
crystal_symmetry = fmodel.xray_structure.crystal_symmetry())
def generate_reflections(self):
sweep_range = self.scan.get_oscillation_range(deg=False)
resolution = 2.0
index_generator = IndexGenerator(self.crystal.get_unit_cell(),
space_group(space_group_symbols(1).hall()).type(),
resolution)
indices = index_generator.to_array()
# Predict rays within the sweep range
ray_predictor = ScansRayPredictor(self.experiments, sweep_range)
obs_refs = ray_predictor(indices)
# Take only those rays that intersect the detector
intersects = ray_intersection(self.detector, obs_refs)
obs_refs = obs_refs.select(intersects)
# Re-predict using the Experiments predictor for all these reflections. The
# result is the same, but we gain also the flags and xyzcal.px columns
obs_refs['id'] = flex.int(len(obs_refs), 0)
obs_refs = self.ref_predictor(obs_refs)
# Set 'observed' centroids from the predicted ones
obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm']
# Invent some variances for the centroid positions of the simulated data
im_width = 0.1 * pi / 180.
px_size = self.detector[0].get_pixel_size()
var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2)
var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2)
var_phi = flex.double(len(obs_refs), (im_width / 2.)**2)
obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi)
# set the flex random seed to an 'uninteresting' number
flex.set_random_seed(12407)
# take 5 random reflections for speed
reflections = obs_refs.select(flex.random_selection(len(obs_refs), 5))
# use a BlockCalculator to calculate the blocks per image
from dials.algorithms.refinement.reflection_manager import BlockCalculator
block_calculator = BlockCalculator(self.experiments, reflections)
reflections = block_calculator.per_image()
return reflections
def exercise(d_min=3.5):
pi = get_pdb_inputs(pdb_str=pdb_str_1)
selection = flex.bool(pi.xrs.scatterers().size(), True)
f_obs = abs(pi.xrs.structure_factors(d_min=d_min).f_calc())
r_free_flags = f_obs.generate_r_free_flags()
for d_min in [1, 2, 3]:
print "d_min:", d_min
f_calc = pi.xrs.structure_factors(d_min=d_min).f_calc()
fft_map = f_calc.fft_map(resolution_factor=0.25)
fft_map.apply_sigma_scaling()
target_map = fft_map.real_map_unpadded()
from mmtbx.refinement import real_space
rsr_simple_refiner = real_space.simple(
target_map=target_map,
selection=selection,
real_space_gradients_delta=d_min / 4,
max_iterations=150,
geometry_restraints_manager=pi.grm.geometry,
)
for shake_size in [1]:
print " shake_size:", shake_size
for p in [(0.01, 1.0), (0.03, 3.0)]:
print " target:", p
w_opt = flex.double()
for start_value in [0, 1000]:
xrs_poor = pi.xrs.deep_copy_scatterers()
random.seed(0)
flex.set_random_seed(0)
xrs_poor.shake_sites_in_place(mean_distance=shake_size)
#
refined = real_space.refinery(
refiner=rsr_simple_refiner,
xray_structure=xrs_poor,
start_trial_weight_value=start_value,
rms_bonds_limit=p[0],
rms_angles_limit=p[1],
)
w_opt.append(refined.weight_final)
dist = flex.mean(flex.sqrt((pi.xrs.sites_cart() - refined.sites_cart_result).dot()))
print " start_value:", start_value, refined.weight_final, refined.rms_bonds_final, refined.rms_angles_final, dist
assert refined.rms_bonds_final <= p[0]
assert refined.rms_angles_final <= p[1]
assert flex.max(flex.abs(w_opt - flex.mean(w_opt))) < 1.0
def exercise_01 () :
pdb_raw = """\
CRYST1 10.000 15.000 10.000 90.00 90.00 90.00 P 1
ATOM 6407 N GLY A 388 -0.783 9.368 -16.436 1.00 51.96 N
ATOM 6408 CA GLY A 388 -0.227 9.888 -15.197 1.00 54.04 C
ATOM 6409 C GLY A 388 -0.637 11.320 -14.897 1.00 55.86 C
ATOM 6410 O GLY A 388 -1.728 11.738 -15.347 1.00 56.70 O
ATOM 6411 OXT GLY A 388 0.129 12.024 -14.203 1.00 56.98 O
ATOM 6412 D GLY A 388 -0.460 9.727 -17.309 1.00 51.44 D
ATOM 6413 HA2 GLY A 388 -0.561 9.258 -14.385 1.00 54.07 H
ATOM 6414 HA3 GLY A 388 0.843 9.835 -15.243 1.00 54.13 H
TER 6415 GLY A 388
HETATM 6416 D D8U A 401 2.236 5.695 -12.992 1.00 15.23 D
HETATM 6417 O DOD A1001 -4.151 4.107 -16.592 1.00 13.40 O
HETATM 6418 D1 DOD A1001 -4.760 3.026 -11.326 1.00 15.45 D
HETATM 6419 D2 DOD A1001 -4.625 2.741 -13.845 1.00 14.81 D
"""
random.seed(12345)
flex.set_random_seed(12345)
pdb_in = iotbx.pdb.hierarchy.input(pdb_string=pdb_raw)
xrs = pdb_in.input.xray_structure_simple()
pdb_in.hierarchy.atoms().reset_i_seq()
open("tst_validate_experimental.pdb", "w").write(
pdb_in.hierarchy.as_pdb_string(crystal_symmetry=xrs))
f_calc =abs( xrs.structure_factors(d_min=2.0).f_calc())
f_calc.set_observation_type_xray_amplitude()
flags = f_calc.generate_r_free_flags()
mtz = f_calc.as_mtz_dataset(column_root_label="F")
mtz.add_miller_array(flags, column_root_label="FreeR_flag")
mtz.mtz_object().write("tst_validate_experimental.mtz")
args = [
"tst_validate_experimental.pdb",
"tst_validate_experimental.mtz",
]
args.append("flags.clashscore=%s" % libtbx.env.has_module("probe"))
result = molprobity.run(args=args,
ignore_missing_modules=True,
out=null_out()).validation
assert result.real_space is not None
ds = result.data_stats
assert (ds.n_free > 0)
def exercise():
# Exercise "simple" target
pi = get_pdb_inputs(pdb_str=pdb_str_1)
selection = flex.bool(pi.xrs.scatterers().size(), True)
for d_min in [1, 2, 3]:
print "d_min:", d_min
f_calc = pi.xrs.structure_factors(d_min = d_min).f_calc()
fft_map = f_calc.fft_map(resolution_factor=0.25)
fft_map.apply_sigma_scaling()
target_map = fft_map.real_map_unpadded()
rsr_simple_refiner = individual_sites.simple(
target_map = target_map,
selection = selection,
real_space_gradients_delta = d_min/4,
max_iterations = 150,
geometry_restraints_manager = pi.grm.geometry)
for shake_size in [1,]:
print " shake_size:", shake_size
for p in [(0.01, 1.0), (0.03, 3.0), (0.1, 10.0)]:
print " target:", p
w_opt = flex.double()
for start_value in [0.001, 0.01, 0.1, 0, 1, 10, 100, 1000]:
xrs_poor = pi.xrs.deep_copy_scatterers()
random.seed(0)
flex.set_random_seed(0)
xrs_poor.shake_sites_in_place(mean_distance = shake_size)
#
refined = individual_sites.refinery(
refiner = rsr_simple_refiner,
xray_structure = xrs_poor,
start_trial_weight_value = start_value,
rms_bonds_limit = p[0],
rms_angles_limit = p[1])
w_opt.append(refined.weight_final)
dist = flex.mean(flex.sqrt((pi.xrs.sites_cart() -
refined.sites_cart_result).dot()))
print " w_start,w_final,b,a,dist: %9.4f %9.4f %6.3f %6.3f %6.3f"%(
start_value, refined.weight_final, refined.rms_bonds_final,
refined.rms_angles_final, dist)
assert refined.rms_bonds_final <= p[0]
assert refined.rms_angles_final <= p[1]
def test_curve_scaler(): flex.set_random_seed( 12345 ) x = flex.double( range(10) )/10.0 y = flex.exp( -x ) y0 = y y1 = y*100+50 y2 = y*1000+500 v0 = y0*0+1 v1 = y0*0+1.0 v2 = v1 scaler_object = linear_scaler( [y0,y1,y2], [v0,v1,v2], 0 , factor=5,show_progress=False) s,o,m = scaler_object.retrieve_results() assert approx_equal(s[1],0.01,eps=1e-3) assert approx_equal(s[2],0.001,eps=1e-3) assert approx_equal(o[1],-50,eps=1e-2) assert approx_equal(o[2],-500,eps=1e-2) s,o = scale_it_pairwise([y0,y1,y2],[v0,v1,v2],0, show_progress=False) assert approx_equal(s[1],0.01,eps=1e-3) assert approx_equal(s[2],0.001,eps=1e-3) assert approx_equal(o[1],-50,eps=1e-2) assert approx_equal(o[2],-500,eps=1e-2)
def exercise () :
flex.set_random_seed(12345)
random.seed(12345)
xrs = random_structure.xray_structure(
sgtbx.space_group_info("P21212"),
elements=["const"]*100)
f_calc = xrs.structure_factors(d_min=2.5).f_calc()
i_calc = abs(f_calc).f_as_f_sq().set_observation_type_xray_intensity()
i_calc = i_calc.customized_copy(
space_group_info=sgtbx.space_group_info("P222"),
sigmas=flex.double(i_calc.size(), 1.0))
complete_set = i_calc.complete_set()
lone_set = complete_set.lone_set(other=i_calc)
i_abs = lone_set.array(data=flex.double(lone_set.size(), 5),
sigmas=flex.double(lone_set.size(), 10))
i_calc = i_calc.concatenate(other=i_abs).set_observation_type_xray_intensity()
i_calc.export_as_scalepack_unmerged(file_name="tst_sys_absent.sca")
args = ["tst_sys_absent.sca"]
out = StringIO()
try :
show_systematic_absences.run(args=args, out=out)
except Sorry :
pass
else :
raise Exception_expected
args.append(",".join([ str(x) for x in xrs.unit_cell().parameters() ]))
show_systematic_absences.run(args=args, out=out)
assert (out.getvalue().count(" ( 0, 3, 0): i/sigi = 0.5") == 4)
i_calc = i_calc.customized_copy(sigmas=None)
i_calc.as_mtz_dataset(column_root_label="I").mtz_object().write("tst_sys_absent.mtz")
args = ["tst_sys_absent.mtz"]
try :
show_systematic_absences.run(args=args, out=out)
except Sorry :
pass
else :
raise Exception_expected
def __call__ (self, i_proc) :
import mmtbx.building
from scitbx.array_family import flex
seed = self.params.random_seed
if (seed is None) :
seed = int(time.time() / os.getpid())
random.seed(seed)
flex.set_random_seed(seed)
sites_new = mmtbx.building.run_real_space_annealing(
xray_structure=self.xray_structure.deep_copy_scatterers(),
pdb_hierarchy=self.pdb_hierarchy.deep_copy(),
processed_pdb_file=self.processed_pdb_file,
selection=self.selection,
target_map=self.target_map,
d_min=self.d_min,
resolution_factor=self.resolution_factor,
params=self.params.simulated_annealing,
target_map_rsr=self.two_fofc_map,
rsr_after_anneal=self.params.rsr_after_anneal,
reference_sigma=self.params.reference_sigma,
wc=self.params.wc,
out=self.out,
debug=self.debug)
return sites_new
def find_tls (params,
pdb_inp,
pdb_hierarchy,
xray_structure,
return_as_list=False,
ignore_pdb_header_groups=False,
out=None) :
if (out is None) :
out = sys.stdout
print_statistics.make_header("Analyzing inputs", out=out)
if (params.random_seed is None) :
params.random_seed = flex.get_random_seed()
random.seed(params.random_seed)
flex.set_random_seed(params.random_seed)
xray_structure.convert_to_isotropic()
sites_cart = xray_structure.sites_cart()
u_cart = None
u_iso = xray_structure.extract_u_iso_or_u_equiv()#*adptbx.u_as_b(1.) # ?
bad_i_seqs = check_adp(u_iso=u_iso, out=out)
if (bad_i_seqs is not None) :
atoms = pdb_hierarchy.atoms()
bad_atom_strings = []
for i_seq in bad_i_seqs[:10] :
atom_str = atoms[i_seq].format_atom_record()
bad_atom_strings.append(atom_str)
if (len(bad_i_seqs) > 10) :
bad_atom_strings.append("... (remaining %d not shown)" %
(len(bad_i_seqs)-10))
raise Sorry(("%d atoms in the model contain isotropic B-factors <= 0:\n"+
"\n".join(bad_atom_strings)) % (len(bad_i_seqs)))
#
ssm = mmtbx.secondary_structure.manager(
pdb_hierarchy = pdb_hierarchy,
sec_str_from_pdb_file = None,
params = None)
alpha_h_selection = ssm.alpha_selection()
secondary_structure_selection = ssm.alpha_selection() | \
ssm.beta_selection() | ssm.base_pair_selection()
if(u_cart is not None):
assert secondary_structure_selection.size() == u_cart.size()
else:
assert secondary_structure_selection.size() == u_iso.size()
ssm.show_summary(log=out)
chains_and_residue_selections, secondary_structure_selection = chains_and_atoms(
pdb_hierarchy = pdb_hierarchy,
secondary_structure_selection = secondary_structure_selection,
out = out)
chains_and_permutations = []
chains_and_atom_selection_strings = []
print_statistics.make_header("Processing chains", out=out)
if (params.nproc is None) :
params.nproc = 1
for crs in chains_and_residue_selections:
print_statistics.make_sub_header("Processing chain '%s'"%crs[0],
out=out)
chain_selection = chain_selection_from_residues(crs[1])
groups, perms = get_model_partitioning(residues = crs[1],
secondary_structure_selection = secondary_structure_selection,
out = out)
#
#print
#selection_arrays = sels_as_selection_arrays(sels = groups)
#merge_groups_by_connectivity(
# pdb_hierarchy = pdb_hierarchy,
# xray_structure = xray_structure,
# selection_arrays = selection_arrays)
#assert 0
#
if(len(perms)==1):
print >> out, " Whole chain is considered as one TLS group."
chains_and_atom_selection_strings.append([crs[0],
permutations_as_atom_selection_string(groups, perms[0])])
else:
print >> out, " Fitting TLS matrices..."
dic = {}
target_best = 1.e+9
if (params.nproc is Auto) or (params.nproc > 1) :
process_perms = analyze_permutations(
groups=groups,
sites_cart=sites_cart,
u_cart=u_cart,
u_iso=u_iso)
from libtbx import easy_mp
stdout_and_targets = easy_mp.pool_map(
processes=params.nproc,
fixed_func=process_perms,
args=perms,
chunksize=100,
func_wrapper="buffer_stdout_stderr")
targets = [ t for so, t in stdout_and_targets ]
for (perm, target) in zip(perms, targets) :
dic.setdefault(len(perm), []).append([target,perm])
else :
for i_perm, perm in enumerate(perms):
if i_perm%500==0:
print >> out, " ...perm %d of %d"%(i_perm, len(perms))
selections = tls_group_selections(groups, perm)
target = 0
for selection in selections:
mo = tls_refinery(
u_cart = u_cart,
u_iso = u_iso,
sites_cart = sites_cart,
selection = selection)
target += mo.f
dic.setdefault(len(perm), []).append([target,perm])
#print " perm %d of %d: target=%8.3f (TLS groups: %s), permutation:"%(
# i_perm, len(perms),target,len(perm)),perm
print >> out, " Best fits:"
print >> out, " No. of Targets"
print >> out, " groups best rand.pick diff. score permutation"
score_best = -1.e+9
perm_choice = None
for k, v in zip(dic.keys(),dic.values()):
t_best = v[0][0]
perm_best = v[0][1]
for v_ in v:
if(v_[0]<t_best):
t_best = v_[0]
perm_best = v_[1]
if(u_cart is not None):
u_cart_ = u_cart.select(chain_selection)
else: u_cart_ = None
if(u_iso is not None):
u_iso_ = u_iso.select(chain_selection)
else: u_iso_ = None
r = tls_refinery_random_groups(
u_cart = u_cart_,
u_iso = u_iso_,
sites_cart = sites_cart.select(chain_selection),
n_groups = k)
score = (r-t_best)/(r+t_best)*100.
print >> out, " %3d %6.3f %6.3f %6.2f %6.3f"%(
k,t_best, r, r-t_best, score), perm_best
if(score > score_best):
score_best = score
perm_choice = perm_best[:]
#
chains_and_permutations.append([crs[0],perm_choice])
chains_and_atom_selection_strings.append([crs[0],
permutations_as_atom_selection_string(groups, perm_choice)])
#
if (pdb_inp is not None) and (not ignore_pdb_header_groups) :
external_tls_selections = external_tls(
pdb_inp = pdb_inp,
pdb_hierarchy = pdb_hierarchy,
sites_cart = sites_cart,
u_iso = u_iso,
out = out)
print_statistics.make_header("SUMMARY", out=out)
#print "Optimal TLS groups:"
#for chain_and_permutation in chains_and_permutations:
# print chain_and_permutation
#print
print >> out, "TLS atom selections for phenix.refine:"
groups_out = cStringIO.StringIO()
selection_strings = []
print >> groups_out, "refinement.refine.adp {"
for r in chains_and_atom_selection_strings:
prefix = "chain '%s'"%r[0]
if(len(r[1])>0 and len(r[1:])>0):
prefix += " and "
for r_ in r[1:]:
for r__ in r_:
if(len(r__)>0):
group_selection = prefix+"(%s)"%r__
print >> groups_out, " tls = \"%s\"" % group_selection
selection_strings.append("%s" % group_selection)
else:
print >> groups_out, " tls = \"%s\"" % prefix
selection_strings.append("%s" % prefix)
print >> groups_out, "}"
print >> out, groups_out.getvalue()
print >> out
#XXX
if 0:
merge_groups_by_connectivity(
pdb_hierarchy = pdb_hierarchy,
xray_structure = xray_structure,
selection_strings = selection_strings)
#XXX
if(len(selection_strings)>0):
total_target = total_score(
pdb_hierarchy = pdb_hierarchy,
sites_cart = sites_cart,
u_iso = u_iso,
selection_strings = selection_strings)
print >> out, "Overall best total target for automatically found groups: %10.1f"%total_target
print >> out
if (return_as_list) :
return selection_strings
else :
return groups_out.getvalue()
def run(args, log = sys.stdout):
print >> log, legend
# XXX: pre-processing for GUI; duplicates some of mmtbx.utils
sources = []
for arg in args :
if os.path.isfile(arg) :
try :
file_phil = iotbx.phil.parse(file_name=arg)
except KeyboardInterrupt :
raise
except RuntimeError :
pass
else :
if len(file_phil.objects) != 0 :
sources.append(file_phil)
if len(sources) > 0 :
cmdline_phil = fmodel_from_xray_structure_master_params.fetch(
sources=sources)
params = cmdline_phil.extract()
if len(params.pdb_file) > 0 :
args.extend(params.pdb_file)
if params.reference_file is not None :
args.append(params.reference_file)
# end of preprocessing
processed_args = mmtbx.utils.process_command_line_args(args = args, log = log,
master_params = fmodel_from_xray_structure_master_params)
pdb_combined = iotbx.pdb.combine_unique_pdb_files(
file_names = processed_args.pdb_file_names)
pdb_combined.report_non_unique(out = log)
print >> log, "-"*79
print >> log, "\nParameters to compute Fmodel::\n"
processed_args.params.show(out = log, prefix=" ")
params = processed_args.params.extract()
if(params.random_seed is not None):
random.seed(params.random_seed)
flex.set_random_seed(params.random_seed)
pdb_file_names = processed_args.pdb_file_names
if len(pdb_file_names) == 0 :
pdb_file_names = params.pdb_file # for GUI
pdb_combined = iotbx.pdb.combine_unique_pdb_files(file_names=pdb_file_names)
pdb_combined.report_non_unique(out = log)
if(len(pdb_combined.unique_file_names) == 0):
raise Sorry("Model file is not provided.")
print >> log, "-"*79
print >> log, "\nInput model file(s):", " ".join(processed_args.pdb_file_names)
pdb_inp = iotbx.pdb.input(source_info = None,
lines = flex.std_string(pdb_combined.raw_records))
# select miller array to use as a set of miller indices for f_model
miller_array = None
if(len(processed_args.reflection_files) > 1):
raise Sorry("Multiple reflection files found at input.")
# FIXME this does not pick up the reference_file parameter! in fact, it
# appears to be ignored completely when run on the command line.
if(len(processed_args.reflection_files) == 1):
print >> log, "-"*79
print >> log, "Input reflection data:", \
" ".join(processed_args.reflection_file_names)
if([params.high_resolution, params.low_resolution].count(None) != 2):
raise Sorry("high_resolution and low_resolution must be undefined "+
"if reflection data file is given.")
miller_arrays = processed_args.reflection_files[0].as_miller_arrays()
data_sizes = flex.int([ma.data().size() for ma in miller_arrays])
if(data_sizes.all_eq(data_sizes[0])): miller_array = miller_arrays[0]
else:
all_labels = []
for ma in miller_arrays:
if(params.data_column_label is not None and
ma.info().label_string() == params.data_column_label):
miller_array = ma
break
all_labels.append(",".join(ma.info().labels))
if(miller_array is None):
raise Sorry("Multiple data available in input reflection file:\n%s\n%s"%(
"\n".join(all_labels),"Please select one using 'data_column_label=' keyword."))
else:
miller_array.show_comprehensive_summary(f = log, prefix=" ")
if(miller_array is not None):
miller_array = miller_array.map_to_asu().customized_copy(
data = flex.double(miller_array.data().size(), 1))
#
cryst1 = pdb_inp.crystal_symmetry_from_cryst1()
if(cryst1 is None and miller_array is not None):
cryst1 = miller_array.crystal_symmetry()
if (cryst1 is not None) and (params.generate_fake_p1_symmetry) :
raise Sorry("The input reference data already define crystal symmetry; "+
"you may not use this in combination with the option "+
"generate_fake_p1_symmetry=True.")
if (not params.generate_fake_p1_symmetry) :
if(cryst1 is None) :
raise Sorry(
"CRYST1 record in input PDB file is incomplete or missing. "+
"If you want the program to generate P1 symmetry automatically, set "+
"generate_fake_p1_symmetry=True.")
else:
if([cryst1.unit_cell(), cryst1.space_group_info()].count(None) != 0):
raise Sorry(
"CRYST1 record in input PDB file is incomplete or missing. "+
"If you want the program to generate P1 symmetry automatically, "+
"set generate_fake_p1_symmetry=True.")
pdb_hierarchy = pdb_inp.construct_hierarchy()
# need to preserve the order in the hierarchy in case we have to perform an
# atom selection later
xray_structure = pdb_hierarchy.extract_xray_structure(
crystal_symmetry = cryst1)
if (cryst1 is None) :
cryst1 = xray_structure.crystal_symmetry()
if (miller_array is not None) :
if (miller_array.crystal_symmetry() is None) :
miller_array = miller_array.customized_copy(crystal_symmetry=cryst1)
xray_structure.show_summary(f = log, prefix=" ")
if(len(params.anomalous_scatterers.group) != 0):
pdb_atoms = pdb_hierarchy.atoms()
pdb_atoms.reset_i_seq()
set_fp_fdp_for_anomalous_scatterers(
pdb_hierarchy = pdb_hierarchy,
xray_structure = xray_structure,
anomalous_scatterer_groups = params.anomalous_scatterers.group)
elif (params.wavelength is not None) :
if (params.scattering_table == "neutron") :
raise Sorry("Wavelength parameter not supported when the neutron "+
"scattering table is used.")
print >> log, "Setting inelastic form factors for wavelength = %g" % \
params.wavelength
xray_structure.set_inelastic_form_factors(
photon=params.wavelength,
table="sasaki")
#
validate_params_command_line(params)
#
print >> log, "-"*79
print >> log, "Computing model structure factors, Fmodel:"
if(params.output.format == "cns"): extension = ".hkl"
elif(params.output.format == "mtz"): extension = ".mtz"
ofn = params.output.file_name
if(ofn is None):
ofn = os.path.basename(processed_args.pdb_file_names[0])
if(len(processed_args.pdb_file_names)==1): ofn = ofn + extension
else: ofn = ofn + "_et_al" + extension
if([miller_array, params.high_resolution].count(None)==2):
raise Sorry("Input data file or high_resolution has to be provided.")
mmtbx.utils.fmodel_from_xray_structure(
xray_structure = xray_structure,
f_obs = miller_array,
add_sigmas = params.add_sigmas,
params = params,
twin_law = params.twin_law,
twin_fraction = params.twin_fraction,
out = log).write_to_file(file_name = ofn,
obs_type=params.output.obs_type)
print >> log, "Output file name:", ofn
print >> log, "All done."
print >> log, "-"*79
return ofn
def exercise () :
flex.set_random_seed(123456)
random.seed(123456)
base = "tst_table_one"
pdb_in = iotbx.pdb.hierarchy.input(pdb_string=model_1yjp)
xrs = pdb_in.xray_structure_simple()
xrs.set_inelastic_form_factors(
photon=1.54,
table="sasaki")
fc = abs(xrs.structure_factors(d_min=1.5).f_calc()).average_bijvoet_mates()
fc.set_observation_type_xray_amplitude()
flags = fc.generate_r_free_flags()
mtz = fc.as_mtz_dataset(column_root_label="F",
wavelength=1.54)
mtz.add_miller_array(flags, column_root_label="FreeR_flag")
mtz.mtz_object().write(base + ".mtz")
xrs_p1 = xrs.expand_to_p1()
xrs_p1.shake_sites_in_place(rms_difference=0.1)
fc_p1 = xrs_p1.structure_factors(d_min=1.4).f_calc()
fc_p1_extra = fc_p1.randomize_amplitude_and_phase(amplitude_error=1.0,
phase_error_deg=0,
random_seed=123456)
fc_p1 = abs(fc_p1.concatenate(other=fc_p1_extra)).sort(
by_value="packed_indices")
fc_p1.set_observation_type_xray_amplitude()
sg_p2 = sgtbx.space_group_info("P2")
ic = fc_p1.f_as_f_sq().customized_copy(
space_group_info=sg_p2,
sigmas=flex.double(fc_p1.size(), 10.0))
ic.export_as_scalepack_unmerged(file_name=base + ".sca")
open(base + ".pdb", "w").write(model_1yjp)
args = [
base + ".mtz",
base + ".pdb",
"unmerged_data=%s.sca" % base,
"prefix=tst_table_one_1",
]
table_one.run(args=args, out=null_out(),
use_current_directory_if_not_specified=True)
# now with unmerged data in SHELX format
f = open(base + ".hkl", "w")
ic.export_as_shelx_hklf(file_object=f)
f.close()
args = [
base + ".mtz",
base + ".pdb",
"unmerged_data=%s.hkl=hklf4" % base,
"prefix=tst_table_one_2",
]
table_one.run(args=args, out=null_out(),
use_current_directory_if_not_specified=True)
# now with phil file
f = open("tst_table_one_3.eff", "w")
f.write("""\
table_one {
structure {
name = %(base)s
pdb_file = %(base)s.pdb
mtz_file = %(base)s.mtz
unmerged_data = %(base)s.hkl=hklf4
}
output {
directory = os.getcwd()
base_name = %(base)s_3
}
}""" % {"base" : base })
args = [ "tst_table_one_3.eff" ]
table_one.run(args=args, out=null_out(),
use_current_directory_if_not_specified=True)
obs_refs['id'] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor.predict(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs['xyzobs.mm.value'] = obs_refs['xyzcal.mm'] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180. px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.)**2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.)**2) var_phi = flex.double(len(obs_refs), (im_width / 2.)**2) obs_refs['xyzobs.mm.variance'] = flex.vec3_double(var_x, var_y, var_phi) # set the flex random seed to an 'uninteresting' number flex.set_random_seed(12407) # take 5 random reflections for speed reflections = obs_refs.select(flex.random_selection(len(obs_refs), 5)) # use a BlockCalculator to calculate the blocks per image from dials.algorithms.refinement.reflection_manager import BlockCalculator block_calculator = BlockCalculator(experiments, reflections) reflections = block_calculator.per_image() # use a ReflectionManager to exclude reflections too close to the spindle, # plus set the frame numbers from dials.algorithms.refinement.reflection_manager import ReflectionManager refman = ReflectionManager(reflections, experiments, outlier_detector=None)
def exercise_load_unmerged():
flex.set_random_seed(123456)
random.seed(123456)
base = "tst_load_unmerged"
pdb_in = iotbx.pdb.hierarchy.input(pdb_string=model_1yjp)
xrs = pdb_in.xray_structure_simple()
xrs.set_inelastic_form_factors(
photon=1.54,
table="sasaki")
fc = abs(xrs.structure_factors(d_min=1.5).f_calc()).average_bijvoet_mates()
fc.set_observation_type_xray_amplitude()
flags = fc.generate_r_free_flags()
mtz = fc.as_mtz_dataset(column_root_label="F")
mtz.add_miller_array(flags, column_root_label="FreeR_flag")
mtz.mtz_object().write(base + ".mtz")
xrs_p1 = xrs.expand_to_p1()
xrs_p1.shake_sites_in_place(rms_difference=0.1)
fc_p1 = xrs_p1.structure_factors(d_min=1.4).f_calc()
fc_p1_extra = fc_p1.randomize_amplitude_and_phase(amplitude_error=1.0,
phase_error_deg=0,
random_seed=123456)
fc_p1 = abs(fc_p1.concatenate(other=fc_p1_extra)).sort(
by_value="packed_indices")
fc_p1.set_observation_type_xray_amplitude()
sg_p2 = sgtbx.space_group_info("P2")
ic = fc_p1.f_as_f_sq().customized_copy(
space_group_info=sg_p2,
sigmas=flex.double(fc_p1.size(), 10.0))
ic.export_as_scalepack_unmerged(file_name=base + ".sca")
with open(base + ".pdb", "w") as f:
f.write(model_1yjp)
args = [
base + ".mtz",
base + ".pdb",
"unmerged_data=%s.sca" % base,
]
master_phil = mmtbx.command_line.generate_master_phil_with_inputs(
phil_string="",
enable_unmerged_data=True)
cmdline = mmtbx.command_line.load_model_and_data(
args=[ base + ext for ext in [".pdb",".mtz",] ] +
["unmerged_data.file_name=%s.sca" % base ],
master_phil=master_phil,
out=StringIO(),
create_fmodel=False,
process_pdb_file=False,
create_log_buffer=True)
# now with .sca in P1 (raises Sorry)
ic2 = fc_p1.f_as_f_sq().customized_copy(
sigmas=flex.double(fc_p1.size(), 10.0))
ic2.export_as_scalepack_unmerged(file_name=base + "_p1.sca")
args = [
base + ".mtz",
base + ".pdb",
"unmerged_data=%s_p1.sca" % base,
]
try :
cmdline = mmtbx.command_line.load_model_and_data(
args=[ base + ext for ext in [".pdb",".mtz",] ] +
["unmerged_data.file_name=%s_p1.sca" % base ],
master_phil=master_phil,
out=StringIO(),
create_fmodel=False,
process_pdb_file=False,
create_log_buffer=True)
except Sorry as s :
assert (str(s) == "Incompatible space groups in merged and unmerged data:P 1 21 1 versus P 1"), s
else :
raise Exception_expected
# XXX
f = open(base + ".cif", "w")
ic.as_cif_simple(array_type="meas",
out=f)
f.close()
args = [
base + ".mtz",
base + ".pdb",
"unmerged_data=%s.cif" % base,
]
cmdline = mmtbx.command_line.load_model_and_data(
args=[ base + ext for ext in [".pdb",".mtz",] ] +
["unmerged_data.file_name=%s.cif" % base ],
master_phil=master_phil,
out=StringIO(),
create_fmodel=False,
process_pdb_file=False,
create_log_buffer=True)
# bad unit cell
uc2 = uctbx.unit_cell((23,6.5,23.5,90,108,90))
ic3 = ic.customized_copy(unit_cell=uc2)
f = open(base + "_new_uc.cif", "w")
ic3.as_cif_simple(array_type="meas",
out=f)
f.close()
args = [
base + ".mtz",
base + ".pdb",
"unmerged_data=%s_new_uc.cif" % base,
]
try :
cmdline = mmtbx.command_line.load_model_and_data(
args=[ base + ext for ext in [".pdb",".mtz",] ] +
["unmerged_data.file_name=%s_new_uc.cif" % base ],
master_phil=master_phil,
out=StringIO(),
create_fmodel=False,
process_pdb_file=False,
create_log_buffer=True)
except Sorry as s :
assert ("Incompatible symmetry definitions" in str(s)), s
else :
raise Exception_expected
import random
import time
import sys
import mmtbx.f_model
from scitbx.array_family import flex
from libtbx import group_args
import mmtbx.utils
import restraints
import macro_cycle
import mmtbx.command_line
# ase / mopac specific imports
from ase.calculators.mopac import Mopac
if 1:
random.seed(1)
flex.set_random_seed(1)
def get_master_phil():
return mmtbx.command_line.generate_master_phil_with_inputs(
phil_string="""
sf_algorithm = *direct fft
.type = choice(multi=False)
restraints = cctbx *qm
.type = choice(multi=False)
number_of_macro_cycles=50
.type = int
data_weight=None
.type = float
file_out_prefix=None
.type = str
use_convergence_test = True
def manage_random_seed(random_seed):
if(random_seed is None):
random_seed = flex.get_random_seed()
random.seed(random_seed)
flex.set_random_seed(random_seed)
finite_grad_differences_test = True,
use_geometry_restraints = True,
shake_site_mean_distance = 1.5,
d_min = 2)
t.run_test()
t.clean_up_temp_test_files()
def exercise_transformation_refinement():
""" Test transformation refinement """
print 'Running ',sys._getframe().f_code.co_name
t = ncs_minimization_test(
n_macro_cycle = 40,
sites = False,
u_iso = False,
transformations = True,
finite_grad_differences_test = True,
use_geometry_restraints = True,
shake_angles_sigma = 0.032,
shake_translation_sigma = 0.5,
d_min = 2)
t.run_test()
t.clean_up_temp_test_files()
if __name__ == "__main__":
flex.set_random_seed(12345)
random.seed(12345)
exercise_without_geometry_restaints()
exercise_site_refinement()
exercise_u_iso_refinement()
exercise_transformation_refinement()
def run(): random.seed(0) flex.set_random_seed(0) test_rosenbrock_function(1) print "OK"
def exercise () :
random.seed(12345)
flex.set_random_seed(12345)
flags_1 = assign_random_r_free_flags(n_refl=100000, fraction_free=0.05)
assert (flags_1.count(True) == 5000)
flags_1_ccp4 = assign_random_r_free_flags(n_refl=100000, fraction_free=0.05,
format="ccp4")
# XXX this is the best we can do with the current method
assert (flags_1_ccp4.count(0) > 4000) and (flags_1_ccp4.count(0) < 6000)
flags_1_shelx = assign_random_r_free_flags(n_refl=100000, fraction_free=0.05,
format="shelx")
assert (flags_1_shelx.count(-1) == 5000)
flags_2 = assign_r_free_flags_by_shells(n_refl=100000,
fraction_free=0.05,
n_bins=50)
assert (flags_2.count(True) == 5000)
ccp4_flags = export_r_free_flags_for_ccp4(flags_1, True)
assert (ccp4_flags.count(0) == flags_1.count(True))
assert (flex.max(ccp4_flags) == 19)
shelx_flags = export_r_free_flags_for_shelx(flags_1, True)
assert ((shelx_flags==-1).all_eq((ccp4_flags==0)))
flags_3 = assign_random_r_free_flags(n_refl=100000, fraction_free=0.025)
assert (flags_3.count(True) == 2500)
ccp4_flags = export_r_free_flags_for_ccp4(flags_3, True)
assert (ccp4_flags.count(0) == flags_3.count(True))
assert (flex.max(ccp4_flags) == 39)
# now with an actual Miller array
symm = crystal.symmetry(
space_group_info=sgtbx.space_group_info("P212121"),
unit_cell=uctbx.unit_cell((6,7,8,90,90,90)))
set1 = miller.build_set(
crystal_symmetry=symm,
anomalous_flag=True,
d_min=1.0)
flags_4 = set1.generate_r_free_flags()
stats = get_r_free_stats(flags_4, True)
assert (19 <= stats[0] <= 25) # XXX is this even necessary?
# much larger for the last few tests
symm = crystal.symmetry(
space_group_info=sgtbx.space_group_info("P212121"),
unit_cell=uctbx.unit_cell((60,70,80,90,90,90)))
set1 = miller.build_set(
crystal_symmetry=symm,
anomalous_flag=True,
d_min=1.0)
#print set1.indices().size()
flags_5 = set1.generate_r_free_flags(fraction=0.1, max_free=None)
flags_6 = adjust_fraction(flags_5, 0.15, log=null_out())
frac_6 = flags_6.data().count(True) / flags_6.data().size()
assert approx_equal(frac_6, 0.15, eps=0.001)
flags_7 = adjust_fraction(flags_5, 0.05, log=null_out())
frac_7 = flags_7.data().count(True) / flags_7.data().size()
assert approx_equal(frac_7, 0.05, eps=0.001)
n_flipped = 0
for i_hkl, (h,k,l) in enumerate(flags_5.indices()) :
if (i_hkl % 100 == 0) and (h > 0) and (k > 0) and (l > 0) :
flag = flags_5.data()[i_hkl]
if (not flag) :
n_flipped += 1
flags_5.data()[i_hkl] = True
# XXX check this for reproducibility on other systems
#assert (n_flipped == 1559) # setting the random seed should ensure this
try :
flags_8 = flags_5.average_bijvoet_mates()
except Sorry :
pass
else :
raise Exception_expected
out = StringIO()
flags_9 = remediate_mismatches(flags_5, log=out)
# XXX check this for reproducibility on other systems
#assert (out.getvalue() == " 1559 reflections moved to test set\n")
flags_10 = flags_9.average_bijvoet_mates()
def test_fast_mcd_large():
from scitbx.array_family import flex
from dials.algorithms.statistics.fast_mcd import FastMCD
# set random seeds to try to avoid assertion errors due to occasionally
# finding less common solutions
import random
random.seed(42)
flex.set_random_seed(42)
# test large dataset algorithm
import libtbx.load_env # required for libtbx.env.find_in_repositories
if not libtbx.env.has_module("dials_regression"):
print "Skipping test_fast_mcd_large(): dials_regression not available."
return
# load data
import os
dials_regression = libtbx.env.find_in_repositories(
relative_path="dials_regression",
test=os.path.isdir)
data_pth = os.path.join(dials_regression, "refinement_test_data",
"outlier_rejection", "residuals.dat")
with(open(data_pth, "r")) as f:
residuals = f.readlines()
# ignore first line, which is a header
residuals = [[float(val) for val in e.split()] for e in residuals[1:]]
X_resid_mm, Y_resid_mm, Phi_resid_mm = zip(*residuals)
X_resid_mm = flex.double(X_resid_mm)
Y_resid_mm = flex.double(Y_resid_mm)
Phi_resid_mm = flex.double(Phi_resid_mm)
# Fast MCD raw estimates
fast_mcd = FastMCD([X_resid_mm, Y_resid_mm, Phi_resid_mm])
T, S = fast_mcd.get_raw_T_and_S()
from libtbx.test_utils import approx_equal
assert approx_equal(T,
[-0.009702392946856687, 0.008866136837504363, -0.04909037126352747])
assert approx_equal(S, flex.double(
[[0.00527965256891, 0.000864300169087, -0.00145971018701],
[0.000864300169087, 0.00842807897907, -0.00184047321286],
[-0.00145971018701, -0.00184047321286, 0.00698461269031]]))
# Fast MCD corrected estimates
T, S = fast_mcd.get_corrected_T_and_S()
assert approx_equal(T,
[-0.009702392946856687, 0.008866136837504363, -0.04909037126352747])
assert approx_equal(S, flex.double(
[[0.0129950608638, 0.00212734325892, -0.00359285435473],
[0.00212734325892, 0.0207444330604, -0.00453004456394],
[-0.00359285435473, -0.00453004456394, 0.0171915605878]]))
# Correction factors
assert approx_equal(fast_mcd._consistency_fac, 2.45659976388)
assert approx_equal(fast_mcd._finite_samp_fac, 1.00193273884)
print "OK"
return
def exercise_analyze_absences():
"""
Looping over common space groups in macromolecular structures, generate
synthetic data converted to the derived intensity group with absent
reflections present (I=0), and run the analysis to check that the actual
space group is the best-scoring.
"""
# space groups found in structures with data in the PDB as of Sep. 2011,
# minus a few unusual ones
symbols = [
"C 2 2 21",
"P 31 1 2",
"P 62 2 2",
"P 2 3",
"P 3",
"P 1",
"P 63",
"C 1 2 1",
"F 2 2 2",
"A 1 2 1",
"B 1 1 2",
"P 61",
"P 61 2 2",
"P 6 2 2",
"P 2 21 21",
"P 21 21 21",
"R 3 2 :R",
"P 4 3 2",
"P 1 21 1",
"P 32 2 1",
"P 42 2 2",
"P 43 21 2",
"P 21 2 21",
"I 2 3",
"P 43 3 2",
"I 41 3 2",
"P 3 1 2",
"P 2 2 2",
"R 3 :H",
"P 21 21 2",
"P 65 2 2",
"P 21 21 2",
"F 2 3",
"I 41 2 2",
"P 65",
"P 1 1 21",
"P 2 2 21",
"P 4 2 2",
"I 2 2 2",
"I 4 3 2",
"R 3 :R",
"P 4",
"P 42",
"P 64 2 2",
"I 1 2 1",
"P 64",
"C 1 2 1",
"I 41",
"P 63 2 2",
"P 3 2 1",
"P 41 2 2",
"P 62",
"P 1",
"P 32 1 2",
"F 4 3 2",
"P 31 2 1",
"I 4 2 2",
"P 21 3",
"P 43 2 2",
"C 2 2 2",
"P 4 21 2",
"B 2 21 2",
"P 41 21 2",
"P 31",
"P 41",
"P 41 3 2",
"P 32",
"P 6",
"P 1 2 1",
"I 21 3",
"F 41 3 2",
"P 42 3 2",
"P 42 21 2",
"P 43",
"I 21 21 21",
"R 3 2 :H",
"I 4",
]
random.seed(987654321)
flex.set_random_seed(987654321) # 12345)
for symbol in ["P 42 3 2"]: # symbols :
xrs = random_structure.xray_structure(space_group_symbol=symbol, elements=["O"] * 200, n_scatterers=200)
fc = abs(xrs.structure_factors(d_min=1.5).f_calc())
fc.set_observation_type_xray_amplitude()
i_calc = fc.f_as_f_sq()
space_group_info = i_calc.crystal_symmetry().space_group_info()
n_absent = i_calc.sys_absent_flags().data().count(True)
assert n_absent == 0
ig = space_group_info.group().build_derived_reflection_intensity_group(False)
complete_set = i_calc.complete_set()
missing_set = complete_set.lone_set(i_calc)
assert missing_set.size() == 0 # should be complete so far
i_calc_orig = i_calc.deep_copy().customized_copy(sigmas=flex.double(i_calc.size(), 0.01))
symm = i_calc.crystal_symmetry().customized_copy(space_group_info=ig.info())
i_calc = i_calc.customized_copy(crystal_symmetry=symm)
# i_calc.show_summary()
complete_set = i_calc.complete_set()
missing_set = complete_set.lone_set(i_calc) # these are the absences
data = flex.double(missing_set.size(), 0.01)
sys_abs = missing_set.array(data=data)
i_calc = i_calc.concatenate(sys_abs)
sigmas = flex.double(i_calc.size(), 0.01)
i_calc = i_calc.customized_copy(sigmas=sigmas)
i_calc = i_calc.set_observation_type_xray_intensity()
# i_calc is now complete with respect to the derived reflection intensity
# group, with I/sigma=1 for all systematic absences
psgc = protein_space_group_choices(miller_array=i_calc, original_data=i_calc_orig)
table = psgc.suggest_likely_candidates()
min_score = min([row[-1] for row in table])
expected_failures = ["I 4", "I 4 2 2", "P 42 3 2"]
# The actual space group won't always be the first on the list (if the
# absence rules are the same for more than one related space group), but
# it should always have the lowest score. (I think...)
for row in table:
group = row[0]
score = row[-1]
n_abs_viol = int(row[4])
n_pres_viol = int(row[5])
if group == str(space_group_info):
assert n_abs_viol == 0, row
if n_pres_viol != 0:
print group, n_pres_viol
psgc.show()
# XXX Currently this fails for the following space groups:
# I 4
# I 4 2 2
# P 42 3 2
assert (score == min_score) or (str(space_group_info) in expected_failures)
from __future__ import division
from libtbx.test_utils import approx_equal
import mmtbx.f_model
import random, time
from scitbx.array_family import flex
from mmtbx import bulk_solvent
from cctbx import adptbx
from cctbx import sgtbx
from cctbx.development import random_structure
import boost.python
ext = boost.python.import_ext("mmtbx_f_model_ext")
from mmtbx.bulk_solvent import scaler
if(1):
random.seed(0)
flex.set_random_seed(0)
def run_0(symbol = "C 2"):
space_group_info = sgtbx.space_group_info(symbol = symbol)
xrs = random_structure.xray_structure(
space_group_info = space_group_info,
elements = ["N"]*50,
volume_per_atom = 100.0,
random_u_iso = True)
#
b_cart = adptbx.random_traceless_symmetry_constrained_b_cart(
crystal_symmetry=xrs.crystal_symmetry())
u_star = adptbx.u_cart_as_u_star(xrs.unit_cell(), adptbx.b_as_u(b_cart))
#
F = xrs.structure_factors(d_min = 1.5).f_calc()
k_anisotropic = mmtbx.f_model.ext.k_anisotropic(F.indices(), u_star)
def test_fast_mcd_small():
from scitbx.array_family import flex
from dials.algorithms.statistics.fast_mcd import FastMCD
# set random seeds to try to avoid assertion errors due to occasionally
# finding less common solutions
import random
random.seed(42)
flex.set_random_seed(42)
# some test data, from R package robustbase: Hawkins, Bradu, Kass's Artificial Data
hbk = """10.1 19.6 28.3
9.5 20.5 28.9
10.7 20.2 31.0
9.9 21.5 31.7
10.3 21.1 31.1
10.8 20.4 29.2
10.5 20.9 29.1
9.9 19.6 28.8
9.7 20.7 31.0
9.3 19.7 30.3
11.0 24.0 35.0
12.0 23.0 37.0
12.0 26.0 34.0
11.0 34.0 34.0
3.4 2.9 2.1
3.1 2.2 0.3
0.0 1.6 0.2
2.3 1.6 2.0
0.8 2.9 1.6
3.1 3.4 2.2
2.6 2.2 1.9
0.4 3.2 1.9
2.0 2.3 0.8
1.3 2.3 0.5
1.0 0.0 0.4
0.9 3.3 2.5
3.3 2.5 2.9
1.8 0.8 2.0
1.2 0.9 0.8
1.2 0.7 3.4
3.1 1.4 1.0
0.5 2.4 0.3
1.5 3.1 1.5
0.4 0.0 0.7
3.1 2.4 3.0
1.1 2.2 2.7
0.1 3.0 2.6
1.5 1.2 0.2
2.1 0.0 1.2
0.5 2.0 1.2
3.4 1.6 2.9
0.3 1.0 2.7
0.1 3.3 0.9
1.8 0.5 3.2
1.9 0.1 0.6
1.8 0.5 3.0
3.0 0.1 0.8
3.1 1.6 3.0
3.1 2.5 1.9
2.1 2.8 2.9
2.3 1.5 0.4
3.3 0.6 1.2
0.3 0.4 3.3
1.1 3.0 0.3
0.5 2.4 0.9
1.8 3.2 0.9
1.8 0.7 0.7
2.4 3.4 1.5
1.6 2.1 3.0
0.3 1.5 3.3
0.4 3.4 3.0
0.9 0.1 0.3
1.1 2.7 0.2
2.8 3.0 2.9
2.0 0.7 2.7
0.2 1.8 0.8
1.6 2.0 1.2
0.1 0.0 1.1
2.0 0.6 0.3
1.0 2.2 2.9
2.2 2.5 2.3
0.6 2.0 1.5
0.3 1.7 2.2
0.0 2.2 1.6
0.3 0.4 2.6"""
# unpack the data into vectors
rows = [[float(e) for e in row.split()] for row in hbk.splitlines()]
x1, x2, x3 = [flex.double(e) for e in zip(*rows)]
# Fast MCD raw estimates
fast_mcd = FastMCD([x1, x2, x3])
T, S = fast_mcd.get_raw_T_and_S()
from libtbx.test_utils import approx_equal
assert approx_equal(T,
[1.5333333333333334, 2.4564102564102566, 1.6076923076923078])
assert approx_equal(S, flex.double(
[[1.18964912281, 0.00464912280702, 0.217368421053],
[0.00464912280702, 0.37620782726, 0.182186234818],
[0.217368421053, 0.182186234818, 0.910728744939]]))
# Fast MCD corrected estimates
T, S = fast_mcd.get_corrected_T_and_S()
assert approx_equal(T,
[1.5333333333333334, 2.4564102564102566, 1.6076923076923078])
assert approx_equal(S, flex.double(
[[3.17735853174, 0.012417047794, 0.58055555535],
[0.01241704779, 1.00478967011, 0.486589681332],
[0.58055555535, 0.486589681332, 2.43240775146]]))
# Correction factors
assert approx_equal(fast_mcd._consistency_fac, 2.36792847084)
assert approx_equal(fast_mcd._finite_samp_fac, 1.12792118859)
print "OK"
return
