def check_cb_op_perm(cb_op, perm): mi_cb = cb_op.apply(ra.miller_indices) miis = flex.random_permutation(size=ra.miller_indices.size())[:2] k = cb_op.apply(ra.miller_indices.select(miis)) matches = miller.match_indices(k, ra.miller_indices) pairs = matches.pairs() assert pairs.column(0).all_eq(flex.size_t_range(k.size())) miis_cb = pairs.column(1) assert perm.select(miis).all_eq(miis_cb)
def check_cb_op_perm(cb_op, perm):
mi_cb = cb_op.apply(ra.miller_indices)
miis = flex.random_permutation(size=ra.miller_indices.size())[:2]
k = cb_op.apply(ra.miller_indices.select(miis))
matches = miller.match_indices(k, ra.miller_indices)
pairs = matches.pairs()
assert pairs.column(0).all_eq(flex.size_t_range(k.size()))
miis_cb = pairs.column(1)
assert perm.select(miis).all_eq(miis_cb)
def exercise_crystallographic():
crystal_symmetry = crystal.symmetry(
unit_cell=(10, 10, 10, 90, 90, 90),
space_group_symbol="P 1 1 2")
sites_frac = flex.vec3_double([
(0.1, 0.1, 0.0),
(0.9, 0.1, 0.0)])
for distance_cutoff in [1,2]:
pair_asu_table = \
crystal_symmetry.special_position_settings().pair_asu_table(
distance_cutoff=distance_cutoff,
sites_frac=sites_frac)
for strictly_in_asu in [True, False]:
cluster = crystal.asu_clusters(
pair_asu_table=pair_asu_table,
strictly_in_asu=strictly_in_asu).sort_index_groups_by_size()
cluster_sizes = [cluster.size() for cluster in cluster.index_groups]
if (distance_cutoff == 1 or strictly_in_asu):
assert cluster_sizes == [1, 1]
else:
assert cluster_sizes == [2]
sites_frac = flex.vec3_double([
(0.1, 0.1, 0.0),
(0.2, 0.2, 0.0),
(0.1, 0.3, 0.0),
(0.9, 0.1, 0.0),
(0.8, 0.2, 0.0)])
for i_trial in xrange(10):
if (i_trial > 0):
sites_frac = sites_frac.select(
flex.random_permutation(size=sites_frac.size()))
for distance_cutoff in [1.5,2]:
asu_mappings = crystal_symmetry.asu_mappings(
buffer_thickness=distance_cutoff).process_sites_frac(
original_sites=sites_frac)
pair_asu_table = crystal.pair_asu_table(
asu_mappings=asu_mappings).add_all_pairs(
distance_cutoff=distance_cutoff)
for strictly_in_asu in [True, False]:
cluster = crystal.asu_clusters(
pair_asu_table=pair_asu_table,
strictly_in_asu=strictly_in_asu).sort_index_groups_by_size()
cluster_sizes = [cluster.size() for cluster in cluster.index_groups]
if (distance_cutoff == 1.5 or strictly_in_asu):
assert cluster_sizes == [3, 2]
else:
assert cluster_sizes == [5]
def exercise_non_crystallographic():
sites_cart = flex.vec3_double([
(7.767, 5.853, 7.671),
(6.935, 5.032, 8.622),
(5.918, 4.176, 8.140),
(7.161, 5.107, 10.012),
(5.126, 3.395, 9.038),
(6.382, 4.336, 10.930),
(5.360, 3.476, 10.439),
(7.956, 7.811, 6.133),
(8.506, 7.237, 5.169),
(8.143, 9.010, 6.428),
(6.253, 5.840, 5.439),
(5.364, 7.253, 5.745),
(5.875, 6.461, 6.183),
(5.216, 5.927, 6.782),
(7.000, 7.000, 7.000)])
have_one_unsorted = False
for i_trial in xrange(10):
if (i_trial > 0):
sites_cart = sites_cart.select(
flex.random_permutation(size=sites_cart.size()))
asu_mappings = crystal.direct_space_asu.non_crystallographic_asu_mappings(
sites_cart=sites_cart)
for distance_cutoff, expected_cluster_sizes in [
(1.0, [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]),
(1.1, [4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]),
(1.5, [6, 5, 3, 1]),
(1.6, [15])]:
pair_asu_table = crystal.pair_asu_table(asu_mappings=asu_mappings)
pair_asu_table.add_all_pairs(distance_cutoff=distance_cutoff)
clusters = crystal.asu_clusters(
pair_asu_table=pair_asu_table).sort_index_groups_by_size()
assert [cluster.size() for cluster in clusters.index_groups] \
== expected_cluster_sizes
if (distance_cutoff == 1.5):
for cluster in clusters.index_groups:
sorted_indices = list(cluster)
sorted_indices.sort()
if (list(cluster) != sorted_indices):
have_one_unsorted = True
clusters.sort_indices_in_each_group()
for cluster in clusters.index_groups:
sorted_indices = list(cluster)
sorted_indices.sort()
assert list(cluster) == sorted_indices
assert have_one_unsorted
def exercise(space_group_info,
n_scatterers=8,
d_min=2.5,
anomalous_flag=False,
verbose=0):
f_calc = random_f_calc(space_group_info=space_group_info,
n_scatterers=n_scatterers,
d_min=d_min,
anomalous_flag=anomalous_flag,
verbose=verbose)
if (f_calc is None): return
data = flex.norm(f_calc.data())
scale_factor = 9999998 / flex.max(data)
data = data * scale_factor + 1
f_calc = miller.array(miller_set=f_calc, data=data, sigmas=data /
10).set_observation_type_xray_intensity()
f_calc = f_calc.select(flex.random_permutation(size=data.size()))
recycle(miller_array=f_calc)
recycle(miller_array=f_calc.f_sq_as_f())
def exercise(space_group_info, n_scatterers=8, d_min=2.5,
anomalous_flag=False, verbose=0):
f_calc = random_f_calc(
space_group_info=space_group_info,
n_scatterers=n_scatterers,
d_min=d_min,
anomalous_flag=anomalous_flag,
verbose=verbose)
if (f_calc is None): return
data = flex.norm(f_calc.data())
scale_factor = 9999998/flex.max(data)
data = data * scale_factor + 1
f_calc = miller.array(
miller_set=f_calc,
data=data,
sigmas=data/10).set_observation_type_xray_intensity()
f_calc = f_calc.select(flex.random_permutation(size=data.size()))
recycle(miller_array=f_calc)
recycle(miller_array=f_calc.f_sq_as_f())
def discover_better_experimental_model(imagesets,
spot_lists,
params,
dps_params,
nproc=1,
wide_search_binning=1):
assert len(imagesets) == len(spot_lists)
assert len(imagesets) > 0
# XXX should check that all the detector and beam objects are the same
from dials.algorithms.indexing.indexer import indexer_base
spot_lists_mm = [
indexer_base.map_spots_pixel_to_mm_rad(spots, imageset.get_detector(),
imageset.get_scan())
for spots, imageset in zip(spot_lists, imagesets)
]
spot_lists_mm = []
max_cell_list = []
detector = imagesets[0].get_detector()
beam = imagesets[0].get_beam()
beam_panel = detector.get_panel_intersection(beam.get_s0())
if beam_panel == -1:
from libtbx.utils import Sorry
raise Sorry('input beam does not intersect detector')
for imageset, spots in zip(imagesets, spot_lists):
if 'imageset_id' not in spots:
spots['imageset_id'] = spots['id']
spots_mm = indexer_base.map_spots_pixel_to_mm_rad(
spots=spots,
detector=imageset.get_detector(),
scan=imageset.get_scan())
indexer_base.map_centroids_to_reciprocal_space(
spots_mm,
detector=imageset.get_detector(),
beam=imageset.get_beam(),
goniometer=imageset.get_goniometer())
if dps_params.d_min is not None:
d_spacings = 1 / spots_mm['rlp'].norms()
sel = d_spacings > dps_params.d_min
spots_mm = spots_mm.select(sel)
# derive a max_cell from mm spots
if params.max_cell is None:
from dials.algorithms.indexing.indexer import find_max_cell
max_cell = find_max_cell(spots_mm,
max_cell_multiplier=1.3,
step_size=45).max_cell
max_cell_list.append(max_cell)
if (params.max_reflections is not None
and spots_mm.size() > params.max_reflections):
logger.info('Selecting subset of %i reflections for analysis' %
params.max_reflections)
perm = flex.random_permutation(spots_mm.size())
sel = perm[:params.max_reflections]
spots_mm = spots_mm.select(sel)
spot_lists_mm.append(spots_mm)
if params.max_cell is None:
max_cell = flex.median(flex.double(max_cell_list))
else:
max_cell = params.max_cell
args = [(imageset, spots, max_cell, dps_params)
for imageset, spots in zip(imagesets, spot_lists_mm)]
from libtbx import easy_mp
results = easy_mp.parallel_map(func=run_dps,
iterable=args,
processes=nproc,
method="multiprocessing",
preserve_order=True,
asynchronous=True,
preserve_exception_message=True)
solution_lists = [r["solutions"] for r in results]
amax_list = [r["amax"] for r in results]
assert len(solution_lists) > 0
detector = imagesets[0].get_detector()
beam = imagesets[0].get_beam()
# perform calculation
if dps_params.indexing.improve_local_scope == "origin_offset":
discoverer = better_experimental_model_discovery(
imagesets,
spot_lists_mm,
solution_lists,
amax_list,
dps_params,
wide_search_binning=wide_search_binning)
new_detector = discoverer.optimize_origin_offset_local_scope()
old_panel, old_beam_centre = detector.get_ray_intersection(
beam.get_s0())
new_panel, new_beam_centre = new_detector.get_ray_intersection(
beam.get_s0())
old_beam_centre_px = detector[old_panel].millimeter_to_pixel(
old_beam_centre)
new_beam_centre_px = new_detector[new_panel].millimeter_to_pixel(
new_beam_centre)
logger.info("Old beam centre: %.2f, %.2f mm" % old_beam_centre +
" (%.1f, %.1f px)" % old_beam_centre_px)
logger.info("New beam centre: %.2f, %.2f mm" % new_beam_centre +
" (%.1f, %.1f px)" % new_beam_centre_px)
logger.info(
"Shift: %.2f, %.2f mm" %
(matrix.col(old_beam_centre) - matrix.col(new_beam_centre)).elems +
" (%.1f, %.1f px)" % (matrix.col(old_beam_centre_px) -
matrix.col(new_beam_centre_px)).elems)
return new_detector, beam
elif dps_params.indexing.improve_local_scope == "S0_vector":
raise NotImplementedError()
def exercise_get_r_free_flags():
crystal_symmetry = crystal.symmetry(
unit_cell=(30,31,32,85,95,100),
space_group_symbol="P 1")
miller_set = miller.build_set(
crystal_symmetry=crystal_symmetry,
anomalous_flag=False,
d_min=3)
n = miller_set.indices().size()
exercise_flag_arrays = []
exercise_flag_arrays.append(
flex.int(list(flex.random_permutation(size=n)%10)))
exercise_flag_arrays.append(flex.int(xrange(n)))
exercise_flag_arrays.append(flex.int(n, 0))
for style in ["ccp4", "cns", "shelx", "bool"]:
for i_exercise,exercise_flag_array in enumerate(exercise_flag_arrays):
for reversed in [False, True]:
if (style == "ccp4"):
if (reversed): break
data = exercise_flag_array
test_flag_value = 3
else:
if (not reversed):
data = (exercise_flag_array == 0)
test_flag_value = True
else:
data = (exercise_flag_array != 0)
test_flag_value = False
if (style == "cns"):
data = data.as_int()
test_flag_value = int(test_flag_value)
elif (style == "shelx"):
data = -data.as_int()
data.set_selected((data == 0), 1)
if (not reversed): test_flag_value = -1
else: test_flag_value = 1
input_array = miller_set.array(data=data)
mtz_dataset = input_array.as_mtz_dataset(
column_root_label="FreeRflags")
mtz_dataset.mtz_object().write("tmp.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp.mtz")]
err = StringIO()
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=err)
for trial_test_flag_value in [None, test_flag_value]:
for trial_label in [None, "free", "foo"]:
try:
r_free_flags, actual_test_flag_value = \
reflection_file_srv.get_r_free_flags(
file_name=None,
label=trial_label,
test_flag_value=trial_test_flag_value,
disable_suitability_test=False,
parameter_scope="r_free_flags")
except Sorry, e:
if (trial_label != "foo"):
assert i_exercise > 0
if (trial_label is None):
assert str(e) == """\
No array of R-free flags found.
For manual selection define:
r_free_flags.label
r_free_flags.test_flag_value
r_free_flags.disable_suitability_test=True"""
else:
assert str(e) == \
"Not a suitable array of R-free flags:" \
+ " r_free_flags.label=free\n" \
+ "To override the suitability test define:" \
+ " r_free_flags.disable_suitability_test=True"
else:
assert str(e) == "No matching array: r_free_flags.label=foo"
if (i_exercise == 0):
assert err.getvalue() == """\
No matching array: r_free_flags.label=foo
Possible choices:
tmp.mtz:FreeRflags
Please use r_free_flags.label
to specify an unambiguous substring of the target label.
"""
else:
assert err.getvalue() == """\
No matching array: r_free_flags.label=foo
"""
err = reflection_file_srv.err = StringIO()
else:
assert i_exercise == 0
actual_test_flag_value_2 = guess_r_free_flag_value(
miller_array=r_free_flags,
test_flag_value=trial_test_flag_value)
assert (actual_test_flag_value_2 == actual_test_flag_value)
def discover_better_experimental_model(
imagesets, spot_lists, params, dps_params, nproc=1, wide_search_binning=1):
assert len(imagesets) == len(spot_lists)
assert len(imagesets) > 0
# XXX should check that all the detector and beam objects are the same
from dials.algorithms.indexing.indexer import indexer_base
spot_lists_mm = [
indexer_base.map_spots_pixel_to_mm_rad(
spots, imageset.get_detector(), imageset.get_scan())
for spots, imageset in zip(spot_lists, imagesets)]
spot_lists_mm = []
max_cell_list = []
detector = imagesets[0].get_detector()
beam = imagesets[0].get_beam()
beam_panel = detector.get_panel_intersection(beam.get_s0())
if beam_panel == -1:
from libtbx.utils import Sorry
raise Sorry, 'input beam does not intersect detector'
for imageset, spots in zip(imagesets, spot_lists):
if 'imageset_id' not in spots:
spots['imageset_id'] = spots['id']
spots_mm = indexer_base.map_spots_pixel_to_mm_rad(
spots=spots, detector=imageset.get_detector(), scan=imageset.get_scan())
indexer_base.map_centroids_to_reciprocal_space(
spots_mm, detector=imageset.get_detector(), beam=imageset.get_beam(),
goniometer=imageset.get_goniometer())
if dps_params.d_min is not None:
d_spacings = 1/spots_mm['rlp'].norms()
sel = d_spacings > dps_params.d_min
spots_mm = spots_mm.select(sel)
# derive a max_cell from mm spots
if params.max_cell is None:
from dials.algorithms.indexing.indexer import find_max_cell
max_cell = find_max_cell(spots_mm, max_cell_multiplier=1.3,
step_size=45,
nearest_neighbor_percentile=0.05).max_cell
max_cell_list.append(max_cell)
if (params.max_reflections is not None and
spots_mm.size() > params.max_reflections):
logger.info('Selecting subset of %i reflections for analysis'
%params.max_reflections)
perm = flex.random_permutation(spots_mm.size())
sel = perm[:params.max_reflections]
spots_mm = spots_mm.select(sel)
spot_lists_mm.append(spots_mm)
if params.max_cell is None:
max_cell = flex.median(flex.double(max_cell_list))
else:
max_cell = params.max_cell
args = [(imageset, spots, max_cell, dps_params)
for imageset, spots in zip(imagesets, spot_lists_mm)]
from libtbx import easy_mp
results = easy_mp.parallel_map(
func=run_dps,
iterable=args,
processes=nproc,
method="multiprocessing",
preserve_order=True,
asynchronous=True,
preserve_exception_message=True)
solution_lists = [r["solutions"] for r in results]
amax_list = [r["amax"] for r in results]
assert len(solution_lists) > 0
detector = imagesets[0].get_detector()
beam = imagesets[0].get_beam()
# perform calculation
if dps_params.indexing.improve_local_scope == "origin_offset":
discoverer = better_experimental_model_discovery(
imagesets, spot_lists_mm, solution_lists, amax_list, dps_params,
wide_search_binning=wide_search_binning)
new_detector = discoverer.optimize_origin_offset_local_scope()
old_beam_centre = detector.get_ray_intersection(beam.get_s0())[1]
new_beam_centre = new_detector.get_ray_intersection(beam.get_s0())[1]
logger.info("Old beam centre: %.2f mm, %.2f mm" %old_beam_centre)
logger.info("New beam centre: %.2f mm, %.2f mm" %new_beam_centre)
logger.info("Shift: %.2f mm, %.2f mm" %(
matrix.col(old_beam_centre)-matrix.col(new_beam_centre)).elems)
return new_detector, beam
elif dps_params.indexing.improve_local_scope=="S0_vector":
raise NotImplementedError()
def exercise_get_r_free_flags():
crystal_symmetry = crystal.symmetry(
unit_cell=(30,31,32,85,95,100),
space_group_symbol="P 1")
miller_set = miller.build_set(
crystal_symmetry=crystal_symmetry,
anomalous_flag=False,
d_min=3)
n = miller_set.indices().size()
exercise_flag_arrays = []
exercise_flag_arrays.append(
flex.int(list(flex.random_permutation(size=n)%10)))
exercise_flag_arrays.append(flex.int(range(n)))
exercise_flag_arrays.append(flex.int(n, 0))
for style in ["ccp4", "cns", "shelx", "bool"]:
for i_exercise,exercise_flag_array in enumerate(exercise_flag_arrays):
for reversed in [False, True]:
if (style == "ccp4"):
if (reversed): break
data = exercise_flag_array
test_flag_value = 3
else:
if (not reversed):
data = (exercise_flag_array == 0)
test_flag_value = True
else:
data = (exercise_flag_array != 0)
test_flag_value = False
if (style == "cns"):
data = data.as_int()
test_flag_value = int(test_flag_value)
elif (style == "shelx"):
data = -data.as_int()
data.set_selected((data == 0), 1)
if (not reversed): test_flag_value = -1
else: test_flag_value = 1
input_array = miller_set.array(data=data)
mtz_dataset = input_array.as_mtz_dataset(
column_root_label="FreeRflags")
mtz_dataset.mtz_object().write("tmp.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp.mtz")]
err = StringIO()
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=err)
for trial_test_flag_value in [None, test_flag_value]:
for trial_label in [None, "free", "foo"]:
try:
r_free_flags, actual_test_flag_value = \
reflection_file_srv.get_r_free_flags(
file_name=None,
label=trial_label,
test_flag_value=trial_test_flag_value,
disable_suitability_test=False,
parameter_scope="r_free_flags")
except Sorry as e:
if (trial_label != "foo"):
assert i_exercise > 0
if (trial_label is None):
assert str(e) == """\
No array of R-free flags found.
For manual selection define:
r_free_flags.label
r_free_flags.test_flag_value
r_free_flags.disable_suitability_test=True"""
else:
assert str(e) == \
"Not a suitable array of R-free flags:" \
+ " r_free_flags.label=free\n" \
+ "To override the suitability test define:" \
+ " r_free_flags.disable_suitability_test=True"
else:
assert str(e) == "No matching array: r_free_flags.label=foo"
if (i_exercise == 0):
assert err.getvalue() == """\
No matching array: r_free_flags.label=foo
Possible choices:
tmp.mtz:FreeRflags
Please use r_free_flags.label
to specify an unambiguous substring of the target label.
"""
else:
assert err.getvalue() == """\
No matching array: r_free_flags.label=foo
"""
err = reflection_file_srv.err = StringIO()
else:
assert i_exercise == 0
actual_test_flag_value_2 = guess_r_free_flag_value(
miller_array=r_free_flags,
test_flag_value=trial_test_flag_value)
assert (actual_test_flag_value_2 == actual_test_flag_value)
for second_label in ["test", "foo"]:
input_array = miller_set.array(data=exercise_flag_arrays[0])
mtz_dataset = input_array.as_mtz_dataset(
column_root_label="FreeRflags")
mtz_dataset.add_miller_array(
miller_array=input_array,
column_root_label=second_label)
mtz_dataset.mtz_object().write("tmp.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp.mtz")]
err = StringIO()
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=err)
try:
reflection_file_srv.get_r_free_flags(
file_name=None,
label=None,
test_flag_value=None,
disable_suitability_test=False,
parameter_scope="r_free_flags")
except Sorry as e:
assert str(e)=="Multiple equally suitable arrays of R-free flags found."
assert err.getvalue() == """\
Multiple equally suitable arrays of R-free flags found.
Possible choices:
tmp.mtz:FreeRflags
tmp.mtz:test
Please use r_free_flags.label
to specify an unambiguous substring of the target label.
"""
err = reflection_file_srv.err = StringIO()
else:
assert str(r_free_flags.info()) == "tmp.mtz:FreeRflags"
r_free_flags, actual_test_flag_value = \
reflection_file_srv.get_r_free_flags(
file_name=None,
label="FreeRflags",
test_flag_value=3,
disable_suitability_test=True,
parameter_scope="r_free_flags")
assert r_free_flags.info().label_string() == "FreeRflags"
assert actual_test_flag_value == 3
for label,test_flag_value in [(None,3), ("FreeRflags",None)]:
try:
reflection_file_srv.get_r_free_flags(
file_name=None,
label=label,
test_flag_value=test_flag_value,
disable_suitability_test=True,
parameter_scope="r_free_flags")
except Sorry as e:
assert str(e) == "r_free_flags.disable_suitability_test=True:" \
" Suitability test for R-free flags can only be disabled if both" \
" r_free_flags.label and r_free_flags.test_flag_value are defined."
else: raise Exception_expected
# test corrupted R-free flags
r_free_flags = miller_set.generate_r_free_flags()
int_flags = r_free_flags.data().as_int()
int_flags[100] = 10000000
r_free_flags = r_free_flags.customized_copy(data=int_flags)
mtz_dataset = r_free_flags.as_mtz_dataset(
column_root_label="TEST")
mtz_dataset.mtz_object().write("tmp.mtz")
reflection_files = [reflection_file_reader.any_reflection_file(
file_name="tmp.mtz")]
err = StringIO()
reflection_file_srv = reflection_file_server(
crystal_symmetry=crystal_symmetry,
force_symmetry=True,
reflection_files=reflection_files,
err=err)
flags, value = reflection_file_srv.get_r_free_flags(
file_name=None,
label=None,
test_flag_value=None,
disable_suitability_test=False,
parameter_scope="r_free_flags")
assert (value == 1)
