def test_experimentlist_from_file(monkeypatch, dials_regression, tmpdir):
# With the default check_format=True this file should fail to load with an
# appropriate error as we can't find the images on disk
with monkeypatch.context() as m:
m.delenv("DIALS_REGRESSION", raising=False)
with pytest.raises(IOError) as e:
exp_list = ExperimentList.from_file(
os.path.join(dials_regression, "experiment_test_data",
"experiment_1.json"))
assert e.value.errno == errno.ENOENT
assert "No such file or directory" in str(e.value)
assert "centroid_0001.cbf" in str(e.value)
# Setting check_format=False should allow the file to load
exp_list = ExperimentList.from_file(
os.path.join(dials_regression, "experiment_test_data",
"experiment_1.json"),
check_format=False,
)
assert len(exp_list) == 1
assert exp_list[0].beam
# This allows expansion of environment variables in regression files, enabling the
# file to load with check_format=True
with monkeypatch.context() as m:
m.setenv("DIALS_REGRESSION", dials_regression)
exp_list = ExperimentList.from_file(
os.path.join(dials_regression, "experiment_test_data",
"experiment_1.json"))
assert len(exp_list) == 1
assert exp_list[0].beam
def test_experimentlist_from_file(dials_regression, tmpdir):
# This allows expansion of environment variables in regression files
os.environ["DIALS_REGRESSION"] = dials_regression
exp_list = ExperimentList.from_file(
os.path.join(dials_regression, "experiment_test_data",
"experiment_1.json"))
assert len(exp_list) == 1
assert exp_list[0].beam
# Try loading from a pickle
exp_list.as_pickle(tmpdir / "el.pickle")
exp_list_pk = ExperimentList.from_file(tmpdir / "el.pickle")
assert len(exp_list_pk) == 1
assert exp_list[0].beam
def test_correct_correction(dials_data):
"""Test that the anvil absorption correction is producing expected values."""
data_dir = dials_data("centroid_test_data")
# We'll need an integrated reflection table and an experiment list.
reflections_file = data_dir.join("integrated.pickle")
experiments_file = data_dir.join("experiments.json")
# We need only test with the first ten reflections.
reflections = flex.reflection_table.from_file(reflections_file)
reflections = reflections.select(flex.size_t_range(10))
experiment = ExperimentList.from_file(experiments_file)[0]
# Test the correction that would be applied to a DAC with 1.5mm-thick anvils,
# aligned along the z-axis at goniometer zero-datum.
old_reflections = copy.deepcopy(reflections)
correct_intensities_for_dac_attenuation(experiment, reflections, (0, 0, 1),
1.5)
cases = {
"intensity.sum.value": reflections.flags.integrated_sum,
"intensity.sum.variance": reflections.flags.integrated_sum,
"intensity.prf.value": reflections.flags.integrated_prf,
"intensity.prf.variance": reflections.flags.integrated_prf,
}
corrections = flex.double([
0,
6.653068275094517,
6.522657529202368,
6.3865190053761,
6.587270967838122,
6.43403642876391,
6.39216742203502,
0,
6.152148372872684,
6.0474840161407375,
])
for case, flag in cases.items():
flagged = reflections.get_flags(flag)
target_correction = corrections.select(flagged)
if "variance" in case:
target_correction = flex.pow2(target_correction)
intensity_correction = (reflections[case] /
old_reflections[case]).select(flagged)
# Check that the un-integrated reflections are unchanged.
assert pytest.approx(reflections[case].select(
~flagged)) == old_reflections[case].select(~flagged), (
"Un-integrated reflections have been erroneously "
"'corrected'.")
# Check that the applied corrections are correct.
assert pytest.approx(
intensity_correction, rel=1e-5
) == list(target_correction), (
"The applied intensity correction to %s doesn't seem to be correct."
% case)
def test_mtz_primitive_cell(dials_data, tmpdir):
scaled_expt = dials_data("insulin_processed") / "scaled.expt"
scaled_refl = dials_data("insulin_processed") / "scaled.refl"
# First reindex to the primitive setting
expts = ExperimentList.from_file(scaled_expt.strpath, check_format=False)
cs = expts[0].crystal.get_crystal_symmetry()
cb_op = cs.change_of_basis_op_to_primitive_setting()
procrunner.run(
[
"dials.reindex",
scaled_expt.strpath,
scaled_refl.strpath,
'change_of_basis_op="%s"' % cb_op,
],
working_directory=tmpdir.strpath,
)
# Now export the reindexed experiments/reflections
procrunner.run(
["dials.export", "reindexed.expt", "reindexed.refl"],
working_directory=tmpdir.strpath,
)
mtz_obj = mtz.object(os.path.join(tmpdir.strpath, "scaled.mtz"))
cs_primitive = cs.change_basis(cb_op)
assert mtz_obj.space_group() == cs_primitive.space_group()
refl = flex.reflection_table.from_file(scaled_refl.strpath)
refl = refl.select(~refl.get_flags(refl.flags.bad_for_scaling, all=False))
for ma in mtz_obj.as_miller_arrays():
assert ma.crystal_symmetry().is_similar_symmetry(cs_primitive)
assert ma.d_max_min() == pytest.approx(
(flex.max(refl["d"]), flex.min(refl["d"]))
)
def test_experimentlist_change_basis(dials_data):
experiments = ExperimentList()
for i in range(4):
experiments.extend(
ExperimentList.from_file(
dials_data("vmxi_proteinase_k_sweeps") /
("experiments_%i.expt" % i),
check_format=False,
))
reindexed_uc = (68.368, 103.968, 68.368, 90.000, 90.000, 90.000)
reindexed_sg = sgtbx.space_group_info("P 4 2 2 (b,c,a)").group()
cb_op = sgtbx.change_of_basis_op("-a,-c,-b")
for cb_op in (cb_op, [cb_op] * len(experiments)):
expts_rdx = experiments.change_basis(cb_op)
for expt in expts_rdx:
assert expt.crystal.get_unit_cell().parameters() == pytest.approx(
reindexed_uc, abs=0.1)
assert expt.crystal.get_space_group() == reindexed_sg
experiments.change_basis(cb_op, in_place=True)
for expt in experiments:
assert expt.crystal.get_unit_cell().parameters() == pytest.approx(
reindexed_uc, abs=0.1)
assert expt.crystal.get_space_group() == reindexed_sg
with pytest.raises(AssertionError):
experiments.change_basis([cb_op, cb_op])
def test_all_expt_ids_have_expts(dials_data, tmpdir):
result = procrunner.run(
[
"dials.index",
dials_data("vmxi_thaumatin_grid_index").join("split_07602.expt"),
dials_data("vmxi_thaumatin_grid_index").join("split_07602.refl"),
"stills.indexer=sequences",
"indexing.method=real_space_grid_search",
"space_group=P4",
"unit_cell=58,58,150,90,90,90",
"max_lattices=8",
"beam.fix=all",
"detector.fix=all",
],
working_directory=tmpdir,
)
assert not result.returncode and not result.stderr
assert tmpdir.join("indexed.expt").check(file=1)
assert tmpdir.join("indexed.refl").check(file=1)
refl = flex.reflection_table.from_file(tmpdir / "indexed.refl")
expt = ExperimentList.from_file(tmpdir / "indexed.expt",
check_format=False)
assert flex.max(refl["id"]) + 1 == len(expt)
def _run_integration(scope, experiments_file, reflections_file):
# type: (scope, str, str) -> Tuple[ExperimentList, flex.reflection_table]
"""Run integration programatically, compatible with multiple DIALS versions.
Args:
scope: The dials.integrate phil scope
experiments_file: Path to the experiment list file
reflections_file: Path to the reflection table file
"""
if hasattr(dials.command_line.integrate, "run_integration"):
# DIALS 3.1+ interface
expts, refls, _ = dials.command_line.integrate.run_integration(
scope.extract(),
ExperimentList.from_file(experiments_file),
flex.reflection_table.from_file(reflections_file),
)
elif hasattr(dials.command_line.integrate, "Script"):
# Pre-3.1-style programmatic interface
expts, refls = dials.command_line.integrate.Script(phil=scope).run(
[experiments_file, reflections_file])
else:
raise RuntimeError(
"Could not find dials.integrate programmatic interface 'run_integration' or 'Script'"
)
return expts, refls
def load_reference_geometries(geometry_file_list):
logger.debug("Collecting reference instrument models.")
ref_geoms = {
# Note that 'index' is the index of the experiment in the expt list file,
# as per dials.show, rather than the UID string of the experiment.
(expt.detector, expt.beam, f, index)
for f in geometry_file_list for index, expt in enumerate(
ExperimentList.from_file(f, check_format=False))
}
logger.debug("Removing duplicate reference geometries.")
duplicates = set()
for a, b in filter(duplicates.isdisjoint,
itertools.combinations(ref_geoms, 2)):
if compare_geometries(a[0], b[0]):
logger.debug(f"Experiment {b[3]} of {b[2]} is a duplicate.")
duplicates.add(b)
ref_geoms -= duplicates
n = len(ref_geoms)
logger.debug(
f"Found {n} unique reference geometr{'ies' if n != 1 else 'y'}.")
for geometry in ref_geoms:
logger.debug(f"Experiment {geometry[3]} of {geometry[2]} is unique.")
return [{
"detector": geometry[0],
"beam": geometry[1]
} for geometry in ref_geoms]
def _import(self, files): # type: (List[str]) -> None
"""
Try to run a quick call of dials.import. Failing that, run a slow call.
Try initially to construct file name templates contiguous groups of files.
Failing that, pass a full list of the files to the importer (slower).
Args:
files: List of image filenames.
"""
info("\nImporting data...")
if len(files) == 1:
if os.path.isdir(files[0]):
debug("You specified a directory. Importing all CBF files in "
"that directory.")
# TODO Support HDF5.
files = [
os.path.join(files[0], f) for f in os.listdir(files[0])
if f.endswith(".cbf") or f.endswith(".cbf.gz")
or f.endswith(".cbf.bz2")
]
elif len(files[0].split(":")) == 3:
debug("You specified an image range in the xia2 format. "
"Importing all specified files.")
template, start, end = files[0].split(":")
template = screen19.make_template(template)[0]
start, end = int(start), int(end)
if not self._quick_import_templates([(template,
(start, end))]):
warning("Could not import specified image range.")
sys.exit(1)
info("Quick import successful.")
return
elif files[0].endswith(".expt"):
debug("You specified an existing experiment list file. "
"No import necessary.")
try:
self.expts = ExperimentList.from_file(files[0])
except (IOError, PickleError, ValueError):
pass
else:
self.params.dials_import.output.experiments = files[0]
if self.expts:
return
if not files:
warning("No images found matching input.")
sys.exit(1)
# Can the files be quick-imported?
if self._quick_import(files):
info("Quick import successful.")
return
self.params.dials_import.input.experiments = files
self._run_dials_import()
def test_generate_mask_trusted_range(dials_data, tmpdir):
# https://github.com/dials/dials/issues/978
image_files = [
f.strpath for f in dials_data("x4wide").listdir("*.cbf", sort=True)
]
with tmpdir.as_cwd():
# Import as usual
importer = ImageImporter(import_phil_scope)
importer.import_image(["output.experiments=no-overloads.expt"] +
image_files)
experiments = ExperimentList.from_file(
tmpdir.join("no-overloads.expt").strpath)
params = phil_scope.fetch(
phil.parse("untrusted.rectangle=100,200,100,200")).extract()
params.output.mask = "pixels1.mask"
generate_mask(experiments, params)
# Import with narrow trusted range to produce overloads
importer = ImageImporter(import_phil_scope)
importer.import_image(
["trusted_range=-1,100", "output.experiments=overloads.expt"] +
image_files)
experiments = ExperimentList.from_file(
tmpdir.join("overloads.expt").strpath)
params = phil_scope.fetch(
phil.parse("untrusted.rectangle=100,200,100,200")).extract()
params.output.mask = "pixels2.mask"
generate_mask(experiments, params)
with tmpdir.join("pixels1.mask").open("rb") as fh:
mask1 = pickle.load(fh)
with tmpdir.join("pixels2.mask").open("rb") as fh:
mask2 = pickle.load(fh)
# Overloads should not be included in the mask
assert (mask1[0] == mask2[0]).all_eq(True)
def load_expt_from_df(df, opt=False):
"""
:param df: a hopper-formatted pandas dataframe with a single row
:return: experiment
"""
if opt:
data_expt_name = df.opt_exp_name.values[0]
else:
data_expt_name = df.exp_name.values[0]
assert os.path.exists(data_expt_name)
data_expt = ExperimentList.from_file(data_expt_name)[0]
return data_expt
def test_generate_mask_with_untrusted_rectangle(experiments_masks, tmpdir):
experiments, masks = experiments_masks
params = phil_scope.fetch(
phil.parse("untrusted.rectangle=100,200,100,200")).extract()
params.output.experiments = "masked.expt"
with tmpdir.as_cwd():
generate_mask(experiments, params)
assert all(tmpdir.join(mask).check() for mask in masks)
assert tmpdir.join("masked.expt").check()
experiments = ExperimentList.from_file(tmpdir.join("masked.expt").strpath)
imageset = experiments.imagesets()[0]
assert imageset.external_lookup.mask.filename == tmpdir.join(
masks[0]).strpath
def test_generate_mask_function_with_untrusted_rectangle(experiments_masks, tmpdir):
experiments, masks = experiments_masks
masks = [tmpdir.join(mask.replace("pixels", "pixels4")) for mask in masks]
params = phil_scope.fetch().extract()
params.output.mask = tmpdir.join("pixels4.mask").strpath
params.output.experiments = tmpdir.join("masked.expt").strpath
params.untrusted.rectangle = [100, 200, 100, 200]
generate_mask(experiments, params)
assert all(mask.check() for mask in masks)
assert tmpdir.join("masked.expt").check()
experiments = ExperimentList.from_file(tmpdir.join("masked.expt").strpath)
associated_masks = [
imageset.external_lookup.mask.filename for imageset in experiments.imagesets()
]
assert all(assoc_mask == mask for assoc_mask, mask in zip(associated_masks, masks))
def test_blank_counts_analysis(dials_data):
expts = ExperimentList.from_file(dials_data("insulin_processed") /
"imported.expt",
check_format=False)
refl = flex.reflection_table.from_file(
dials_data("insulin_processed") / "strong.refl")
results = detect_blanks.blank_counts_analysis(refl,
expts[0].scan,
phi_step=5,
fractional_loss=0.1)
assert set(results.keys()) == {"data", "layout", "blank_regions"}
assert results["data"][0]["x"] == [
2.5,
7.5,
12.5,
17.5,
22.5,
27.5,
32.5,
37.5,
42.5,
]
assert results["data"][0]["y"] == [
2827,
2589,
2502,
2464,
2515,
2490,
2441,
2500,
2505,
]
assert not any(results["data"][0]["blank"])
assert results["blank_regions"] == []
z = refl["xyzobs.px.value"].parts()[2]
refl_subset = refl.select((z < 10) | (z > 20))
results = detect_blanks.blank_counts_analysis(refl_subset,
expts[0].scan,
phi_step=2,
fractional_loss=0.1)
assert results["data"][0]["blank"].count(True) == 5
assert results["blank_regions"] == [(10, 21)]
def test_connected_components_centred_cell(dials_data):
experiment = ExperimentList.from_file(
dials_data("insulin_processed").join("scaled.expt").strpath, check_format=False
)[0]
experiment.scan.set_image_range((1, 10))
predict = ScanStaticReflectionPredictor(experiment, dmin=3, margin=1)
refl = predict.for_ub(experiment.crystal.get_A())
miller_set = miller.set(
experiment.crystal.get_crystal_symmetry(),
refl["miller_index"],
anomalous_flag=False,
)
miller_array = miller_set.d_spacings().resolution_filter(d_min=3)
complete_set, unique_ms = missing_reflections.connected_components(miller_array)
assert [ms.size() for ms in unique_ms] == [581, 32, 29, 6, 3, 3, 3, 2]
# Verify that all the indices reported missing are actually missing from the input
for ms in unique_ms:
assert ms.common_set(miller_array.map_to_asu()).size() == 0
assert complete_set.completeness() == 1
def test_load_reference_geometries(dials_data):
"""
Test `xia2.Schema.load_reference_geometries`.
Test the function that finds the set of unique instrument models from a list
of experiment list files.
There are eight input instrument models, of which only two are unique.
"""
files = ["scaled_20_25.expt", "scaled_30.expt", "scaled_35.expt"]
files = [(dials_data("l_cysteine_4_sweeps_scaled") / f).strpath for f in files]
files.append((dials_data("l_cysteine_dials_output") / "indexed.expt").strpath)
num_input = sum(len(ExperimentList.from_file(f, check_format=False)) for f in files)
assert num_input == 8, "Expected to find eight experiments, one for each sweep."
unique_geometries = load_reference_geometries(files)
assert len(unique_geometries) == 2, "Expected to find two unique instrument models."
detectors = (geom["detector"] for geom in unique_geometries)
assert not compare_geometries(*detectors), "Unique detectors cannot be equivalent."
def test_few_reflections(dials_data, run_in_tmpdir):
u"""
Test that dials.symmetry does something sensible if given few reflections.
Use some example integrated data generated from a ten-image 1° sweep. These
contain a few dozen integrated reflections.
Args:
dials_data: DIALS custom Pytest fixture for access to test data.
run_in_tmpdir: DIALS custom Pytest fixture to run this test in a temporary
directory.
"""
# Get and use the default parameters for dials.symmetry.
params = symmetry.phil_scope.fetch(source=parse("")).extract()
# Use the integrated data from the first ten images of the first sweep.
data_dir = dials_data("l_cysteine_dials_output")
experiments = ExperimentList.from_file(data_dir / "11_integrated.expt")
reflections = [flex.reflection_table.from_file(data_dir / "11_integrated.refl")]
# Run dials.symmetry on the above data files.
symmetry.symmetry(experiments, reflections, params)
def _create_profile_model(self): # type: () -> bool
"""
Run `dials.create_profile_model` on indexed reflections.
The indexed experiment list will be overwritten with a copy that includes
the profile model but is otherwise identical.
Returns:
Boolean value indicating whether it was possible to determine a profile
model from the data.
"""
info("\nCreating profile model...")
command = [
"dials.create_profile_model",
self.params.dials_index.output.experiments,
self.params.dials_index.output.reflections,
"output = %s" % self.params.dials_index.output.experiments,
]
result = procrunner.run(command,
print_stdout=False,
debug=procrunner_debug)
debug("result = %s", screen19.prettyprint_dictionary(result))
self._sigma_m = None
if result["exitcode"] == 0:
db = ExperimentList.from_file(
self.params.dials_index.output.experiments)[0]
self._oscillation = db.imageset.get_scan().get_oscillation()[1]
self._sigma_m = db.profile.sigma_m()
info(
u"%d images, %s° oscillation, σ_m=%.3f°",
db.imageset.get_scan().get_num_images(),
str(self._oscillation),
self._sigma_m,
)
info("Successfully completed (%.1f sec)", result["runtime"])
return True
warning("Failed with exit code %d", result["exitcode"])
return False
def test_connected_components(dials_data):
experiment = ExperimentList.from_file(
dials_data("centroid_test_data").join("experiments.json").strpath
)[0]
image_ranges = [(1, 9), (1, 100), (1, 1000)]
expected_ms_sizes = [[755], [242, 14, 10, 5, 2, 2, 2], []]
for image_range, expected_sizes in zip(image_ranges, expected_ms_sizes):
experiment.scan.set_image_range(image_range)
predict = ScanStaticReflectionPredictor(experiment, dmin=3, margin=1)
refl = predict.for_ub(experiment.crystal.get_A())
miller_set = miller.set(
experiment.crystal.get_crystal_symmetry(),
refl["miller_index"],
anomalous_flag=False,
)
miller_array = miller_set.d_spacings().resolution_filter(d_min=3)
complete_set, unique_ms = missing_reflections.connected_components(miller_array)
assert len(unique_ms) == len(expected_sizes)
assert [ms.size() for ms in unique_ms] == expected_sizes
# Verify that all the indices reported missing are actually missing from the input
for ms in unique_ms:
assert ms.common_set(miller_array.map_to_asu()).size() == 0
assert complete_set.completeness() == 1
def test_forward_no_model(dials_data):
expt = ExperimentList.from_file(
dials_data("centroid_test_data").join(
"imported_experiments.json").strpath)[0]
# Get the models
beam = expt.beam
detector = expt.detector
gonio = expt.goniometer
scan = expt.scan
scan.set_image_range((0, 1000))
# Set some parameters
sigma_divergence = 0.00101229
mosaicity = 0.157 * math.pi / 180
n_sigma = 3
grid_size = 20
delta_divergence = n_sigma * sigma_divergence
step_size = delta_divergence / grid_size
delta_divergence2 = delta_divergence + step_size * 0.5
delta_mosaicity = n_sigma * mosaicity
# Create the bounding box calculator
calculate_bbox = BBoxCalculator3D(beam, detector, gonio, scan,
delta_divergence2, delta_mosaicity)
# Initialise the transform
spec = transform.TransformSpec(beam, detector, gonio, scan,
sigma_divergence, mosaicity, n_sigma + 1,
grid_size)
# tst_conservation_of_counts(self):
random.seed(0)
assert len(detector) == 1
s0 = beam.get_s0()
m2 = gonio.get_rotation_axis()
s0_length = matrix.col(beam.get_s0()).length()
# Create an s1 map
s1_map = transform.beam_vector_map(detector[0], beam, True)
for i in range(100):
# Get random x, y, z
x = random.uniform(300, 1800)
y = random.uniform(300, 1800)
z = random.uniform(500, 600)
# Get random s1, phi, panel
s1 = matrix.col(detector[0].get_pixel_lab_coord(
(x, y))).normalize() * s0_length
phi = scan.get_angle_from_array_index(z, deg=False)
panel = 0
# Calculate the bounding box
bbox = calculate_bbox(s1, z, panel)
x0, x1, y0, y1, z0, z1 = bbox
# Create the coordinate system
cs = CoordinateSystem(m2, s0, s1, phi)
if abs(cs.zeta()) < 0.1:
continue
# The grid index generator
step_size = delta_divergence / grid_size
grid_index = transform.GridIndexGenerator(cs, x0, y0,
(step_size, step_size),
grid_size, s1_map)
# Create the image
# image = flex.double(flex.grid(z1 - z0, y1 - y0, x1 - x0), 1)
image = gaussian((z1 - z0, y1 - y0, x1 - x0), 10.0,
(z - z0, y - y0, x - x0), (2.0, 2.0, 2.0))
mask = flex.bool(flex.grid(image.all()), False)
for j in range(y1 - y0):
for i in range(x1 - x0):
inside = False
gx00, gy00 = grid_index(j, i)
gx01, gy01 = grid_index(j, i + 1)
gx10, gy10 = grid_index(j + 1, i)
gx11, gy11 = grid_index(j + 1, i + 1)
mingx = min([gx00, gx01, gx10, gx11])
maxgx = max([gx00, gx01, gx10, gx11])
mingy = min([gy00, gy01, gy10, gy11])
maxgy = max([gy00, gy01, gy10, gy11])
if (mingx >= 0 and maxgx < 2 * grid_size + 1 and mingy >= 0
and maxgy < 2 * grid_size + 1):
inside = True
for k in range(1, z1 - z0 - 1):
mask[k, j, i] = inside
# Transform the image to the grid
transformed = transform.TransformForwardNoModel(
spec, cs, bbox, 0, image.as_double(), mask)
grid = transformed.profile()
# Get the sums and ensure they're the same
eps = 1e-7
sum_grid = flex.sum(grid)
sum_image = flex.sum(flex.double(flex.select(image, flags=mask)))
assert abs(sum_grid - sum_image) <= eps
mask = flex.bool(flex.grid(image.all()), True)
transformed = transform.TransformForwardNoModel(
spec, cs, bbox, 0, image.as_double(), mask)
grid = transformed.profile()
# Boost the bbox to make sure all intensity is included
x0, x1, y0, y1, z0, z1 = bbox
bbox2 = (x0 - 10, x1 + 10, y0 - 10, y1 + 10, z0 - 10, z1 + 10)
# Do the reverse transform
transformed = transform.TransformReverseNoModel(
spec, cs, bbox2, 0, grid)
image2 = transformed.profile()
# Check the sum of pixels are the same
sum_grid = flex.sum(grid)
sum_image = flex.sum(image2)
assert abs(sum_grid - sum_image) <= eps
# Do the reverse transform
transformed = transform.TransformReverseNoModel(
spec, cs, bbox, 0, grid)
image2 = transformed.profile()
from dials.algorithms.statistics import pearson_correlation_coefficient
cc = pearson_correlation_coefficient(image.as_1d().as_double(),
image2.as_1d())
assert cc >= 0.99
def _refine(self):
for idxr in set(self._refinr_indexers.values()):
experiments = idxr.get_indexer_experiment_list()
indexed_experiments = idxr.get_indexer_payload(
"experiments_filename")
indexed_reflections = idxr.get_indexer_payload("indexed_filename")
# If multiple sweeps but not doing joint refinement, get only the
# relevant reflections.
multi_sweep = PhilIndex.params.xia2.settings.multi_sweep_refinement
if len(experiments) > 1 and not multi_sweep:
xsweeps = idxr._indxr_sweeps
assert len(xsweeps) == len(experiments)
# Don't do joint refinement
assert len(self._refinr_sweeps) == 1
xsweep = self._refinr_sweeps[0]
i = xsweeps.index(xsweep)
experiments = experiments[i:i + 1]
# Extract and output experiment and reflections for current sweep
indexed_experiments = os.path.join(
self.get_working_directory(),
"%s_indexed.expt" % xsweep.get_name())
indexed_reflections = os.path.join(
self.get_working_directory(),
"%s_indexed.refl" % xsweep.get_name())
experiments.as_file(indexed_experiments)
reflections = flex.reflection_table.from_file(
idxr.get_indexer_payload("indexed_filename"))
sel = reflections["id"] == i
assert sel.count(True) > 0
imageset_id = reflections["imageset_id"].select(sel)
assert imageset_id.all_eq(imageset_id[0])
sel = reflections["imageset_id"] == imageset_id[0]
reflections = reflections.select(sel)
# set indexed reflections to id == 0 and imageset_id == 0
reflections["id"].set_selected(reflections["id"] == i, 0)
reflections["imageset_id"] = flex.int(len(reflections), 0)
reflections.as_file(indexed_reflections)
# currently only handle one lattice/refiner
assert len(experiments.crystals()) == 1
scan_static = PhilIndex.params.dials.refine.scan_static
# Avoid doing scan-varying refinement on narrow wedges.
scan_oscillation_ranges = []
for experiment in experiments:
start, end = experiment.scan.get_oscillation_range()
scan_oscillation_ranges.append(end - start)
min_oscillation_range = min(scan_oscillation_ranges)
if (PhilIndex.params.dials.refine.scan_varying
and min_oscillation_range > 5
and not PhilIndex.params.dials.fast_mode):
scan_varying = PhilIndex.params.dials.refine.scan_varying
else:
scan_varying = False
if scan_static:
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(indexed_reflections)
refiner.set_scan_varying(False)
refiner.run()
self._refinr_experiments_filename = (
refiner.get_refined_experiments_filename())
self._refinr_indexed_filename = refiner.get_refined_filename()
else:
self._refinr_experiments_filename = indexed_experiments
self._refinr_indexed_filename = indexed_reflections
if scan_varying:
refiner = self.Refine()
refiner.set_experiments_filename(
self._refinr_experiments_filename)
refiner.set_indexed_filename(self._refinr_indexed_filename)
if min_oscillation_range < 36:
refiner.set_interval_width_degrees(min_oscillation_range /
2)
refiner.run()
self._refinr_experiments_filename = (
refiner.get_refined_experiments_filename())
self._refinr_indexed_filename = refiner.get_refined_filename()
if scan_static or scan_varying:
FileHandler.record_log_file(
"%s REFINE" % idxr.get_indexer_full_name(),
refiner.get_log_file())
report = self.Report()
report.set_experiments_filename(
self._refinr_experiments_filename)
report.set_reflections_filename(self._refinr_indexed_filename)
html_filename = os.path.join(
self.get_working_directory(),
"%i_dials.refine.report.html" % report.get_xpid(),
)
report.set_html_filename(html_filename)
report.run()
FileHandler.record_html_file(
"%s REFINE" % idxr.get_indexer_full_name(), html_filename)
experiments = ExperimentList.from_file(
self._refinr_experiments_filename)
self.set_refiner_payload("models.expt",
self._refinr_experiments_filename)
self.set_refiner_payload("observations.refl",
self._refinr_indexed_filename)
# this is the result of the cell refinement
self._refinr_cell = experiments.crystals()[0].get_unit_cell(
).parameters()
def test_forward(dials_data):
expt = ExperimentList.from_file(
dials_data("centroid_test_data").join(
"imported_experiments.json").strpath)[0]
# Get the models
beam = expt.beam
detector = expt.detector
gonio = expt.goniometer
scan = expt.scan
# Set some parameters
sigma_divergence = 0.00101229
mosaicity = 0.157 * math.pi / 180
n_sigma = 3
grid_size = 7
delta_divergence = n_sigma * sigma_divergence
step_size = delta_divergence / grid_size
delta_divergence2 = delta_divergence + step_size * 0.5
delta_mosaicity = n_sigma * mosaicity
# Create the bounding box calculator
calculate_bbox = BBoxCalculator3D(beam, detector, gonio, scan,
delta_divergence2, delta_mosaicity)
# Initialise the transform
spec = transform.TransformSpec(beam, detector, gonio, scan,
sigma_divergence, mosaicity, n_sigma + 1,
grid_size)
# tst_conservation_of_counts(self):
assert len(detector) == 1
s0 = beam.get_s0()
m2 = gonio.get_rotation_axis()
s0_length = matrix.col(beam.get_s0()).length()
# Create an s1 map
s1_map = transform.beam_vector_map(detector[0], beam, True)
for i in range(100):
# Get random x, y, z
x = random.uniform(300, 1800)
y = random.uniform(300, 1800)
z = random.uniform(0, 9)
# Get random s1, phi, panel
s1 = matrix.col(detector[0].get_pixel_lab_coord(
(x, y))).normalize() * s0_length
phi = scan.get_angle_from_array_index(z, deg=False)
panel = 0
# Calculate the bounding box
bbox = calculate_bbox(s1, z, panel)
x0, x1, y0, y1, z0, z1 = bbox
# Create the coordinate system
cs = CoordinateSystem(m2, s0, s1, phi)
# The grid index generator
step_size = delta_divergence / grid_size
grid_index = transform.GridIndexGenerator(cs, x0, y0,
(step_size, step_size),
grid_size, s1_map)
# Create the image
# image = flex.double(flex.grid(z1 - z0, y1 - y0, x1 - x0), 1)
image = gaussian((z1 - z0, y1 - y0, x1 - x0), 10.0,
(z - z0, y - y0, x - x0), (2.0, 2.0, 2.0))
mask = flex.bool(flex.grid(image.all()), False)
for j in range(y1 - y0):
for i in range(x1 - x0):
inside = False
gx00, gy00 = grid_index(j, i)
gx01, gy01 = grid_index(j, i + 1)
gx10, gy10 = grid_index(j + 1, i)
gx11, gy11 = grid_index(j + 1, i + 1)
mingx = min([gx00, gx01, gx10, gx11])
maxgx = max([gx00, gx01, gx10, gx11])
mingy = min([gy00, gy01, gy10, gy11])
maxgy = max([gy00, gy01, gy10, gy11])
if (mingx >= 0 and maxgx < 2 * grid_size + 1 and mingy >= 0
and maxgy < 2 * grid_size + 1):
inside = True
for k in range(1, z1 - z0 - 1):
mask[k, j, i] = inside
# Transform the image to the grid
transformed = transform.TransformForward(spec, cs, bbox, 0,
image.as_double(), mask)
grid = transformed.profile()
# Get the sums and ensure they're the same
eps = 1e-7
sum_grid = flex.sum(grid)
sum_image = flex.sum(flex.double(flex.select(image, flags=mask)))
assert abs(sum_grid - sum_image) <= eps
# Test passed
# tst_transform_with_background(self):
assert len(detector) == 1
s0 = beam.get_s0()
m2 = gonio.get_rotation_axis()
s0_length = matrix.col(beam.get_s0()).length()
# Create an s1 map
s1_map = transform.beam_vector_map(detector[0], beam, True)
for i in range(100):
# Get random x, y, z
x = random.uniform(300, 1800)
y = random.uniform(300, 1800)
z = random.uniform(0, 9)
# Get random s1, phi, panel
s1 = matrix.col(detector[0].get_pixel_lab_coord(
(x, y))).normalize() * s0_length
phi = scan.get_angle_from_array_index(z, deg=False)
panel = 0
# Calculate the bounding box
bbox = calculate_bbox(s1, z, panel)
x0, x1, y0, y1, z0, z1 = bbox
# Create the coordinate system
cs = CoordinateSystem(m2, s0, s1, phi)
# The grid index generator
step_size = delta_divergence / grid_size
grid_index = transform.GridIndexGenerator(cs, x0, y0,
(step_size, step_size),
grid_size, s1_map)
# Create the image
# image = flex.double(flex.grid(z1 - z0, y1 - y0, x1 - x0), 1)
image = gaussian((z1 - z0, y1 - y0, x1 - x0), 10.0,
(z - z0, y - y0, x - x0), (2.0, 2.0, 2.0))
background = flex.random_double(len(image))
background.resize(image.accessor())
mask = flex.bool(flex.grid(image.all()), False)
for j in range(y1 - y0):
for i in range(x1 - x0):
inside = False
gx00, gy00 = grid_index(j, i)
gx01, gy01 = grid_index(j, i + 1)
gx10, gy10 = grid_index(j + 1, i)
gx11, gy11 = grid_index(j + 1, i + 1)
mingx = min([gx00, gx01, gx10, gx11])
maxgx = max([gx00, gx01, gx10, gx11])
mingy = min([gy00, gy01, gy10, gy11])
maxgy = max([gy00, gy01, gy10, gy11])
if (mingx >= 0 and maxgx <= 2 * grid_size + 1 and mingy >= 0
and maxgy <= 2 * grid_size + 1):
inside = True
for k in range(1, z1 - z0 - 1):
mask[k, j, i] = inside
# Transform the image to the grid
transformed = transform.TransformForward(spec, cs, bbox, 0,
image.as_double(),
background.as_double(), mask)
igrid = transformed.profile()
bgrid = transformed.background()
# Get the sums and ensure they're the same
eps = 1e-7
sum_igrid = flex.sum(igrid)
sum_bgrid = flex.sum(bgrid)
sum_image = flex.sum(flex.double(flex.select(image, flags=mask)))
sum_bkgrd = flex.sum(flex.double(flex.select(background, flags=mask)))
try:
assert abs(sum_igrid - sum_image) <= eps
assert abs(sum_bgrid - sum_bkgrd) <= eps
except Exception:
print("Failed for: ", (x, y, z))
raise
locations = ax.xaxis.get_major_locator().tick_values(images.min(), images.max())
# Replace the tick at the 0th frame (which is meaningless) with one at the 1st.
locations = np.where(locations == 0, 1, locations)
ax.xaxis.set_major_locator(ticker.FixedLocator(locations))
# Format the rotation magnitude (tumbling angle) axis tick labels in degrees.
ax.yaxis.set_major_formatter(ticker.PercentFormatter(symbol=u"°", decimals=1))
fig.savefig("tumbling_angle")
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"experiment_list",
metavar="refined.expt",
type=str,
help="A refined experiment list from DIALS, such as 'refined.expt'.",
)
if __name__ == "__main__":
experiment_list = ExperimentList.from_file(parser.parse_args().experiment_list)
if len(experiment_list) > 1:
print(
"Warning: You have supplied an input file containing multiple "
"experiment models. Only the first will be used."
)
print("Saving a plot of tumbling angle versus image number as tumbling_angle.png.")
plot_angles(*get_angles(experiment_list[0].crystal))
def experiments_masks(request, dials_data):
filename = (dials_data(request.param["directory"]) /
request.param["filename"]).strpath
return ExperimentList.from_file(filename), request.param["masks"]
"Will not use cuda, but will use openmp, control with env var OMP_NUM_THREADS"
)
imgs, expt = forward_models.model_spots_from_pandas(
df,
mtz_file=Fname,
mtz_col=Fcol,
spectrum_override=spec,
cuda=cuda,
device_Id=dev,
force_no_detector_thickness=no_det_thick,
use_db=True)
if args.compareData:
print("Extracting data from experiment list %s..." %
df.exp_name.values[0])
El = ExperimentList.from_file(df.exp_name.values[0])
iset = El[0].imageset
# NOTE : assumes a multi-panel detector model, otherwise get_raw_data should have no arg, e.g. iset.get_raw_data()
data = np.array([a.as_numpy_array() for a in iset.get_raw_data(0)])
# divide the data by the adu factor
data /= M.params.refiner.adu_per_photon
# extract the background:
print("Extracting background from data ...")
bg = extract_background_from_data(data, args.j,
(args.filtsz, args.filtsz))
imgs_w_bg = imgs + bg # this image is the extracted background and the optimized forward Bragg model
# this image inclues data except for those pixels that were modeled during stage 1
imgs2 = data.copy()
P = M.all_pid
F = M.all_fast
import numpy as np from cxid9114.parameters import ENERGY_CONV defaultF = 1e3 mos_spread = 0 mos_doms = 1 Ncells_abc = 7, 7, 7 profile = "gauss" beamsize = 0.1 exposure_s = 1 total_flux = 1e12 xtal_size = 0.0005 exper_name = "/Users/dermen/pinks/derek/refined_varying/stills2.expt" El = ExperimentList.from_file(exper_name) E = El[0] crystal = E.crystal DET = E.detector BEAM = E.beam symbol = "P212121" sgi = sgtbx.space_group_info(symbol) # TODO: allow override of ucell symm = symmetry(unit_cell=crystal.get_unit_cell(), space_group_info=sgi) miller_set = symm.build_miller_set(anomalous_flag=True, d_min=1.5, d_max=999) # NOTE does build_miller_set automatically expand to p1 ? Does it obey systematic absences ? # Note how to handle sys absences here ? Famp = flex.double(np.ones(len(miller_set.indices())) * defaultF) mil_ar = miller.array(miller_set=miller_set, data=Famp).set_observation_type_xray_amplitude()
def test_blank_integrated_analysis(dials_data):
expts = ExperimentList.from_file(dials_data("insulin_processed") /
"integrated.expt",
check_format=False)
refl = flex.reflection_table.from_file(
dials_data("insulin_processed") / "integrated.refl")
results = detect_blanks.blank_integrated_analysis(refl,
expts[0].scan,
phi_step=5,
fractional_loss=0.1)
assert results["data"][0]["x"] == [
2.5,
7.5,
12.5,
17.5,
22.5,
27.5,
32.5,
37.5,
42.5,
]
assert results["data"][0]["y"] == pytest.approx([
27.903266149430973,
25.832527090455052,
26.9236206883069,
26.50234804728626,
26.41019377727383,
25.810676090828185,
24.844906790823064,
25.89992001081651,
25.580718362291474,
])
assert not any(results["data"][0]["blank"])
assert results["blank_regions"] == []
# Now with some "blank" regions - make some of the reflections weak
z = refl["xyzobs.px.value"].parts()[2]
refl["intensity.prf.value"].set_selected(
z < 10, refl["intensity.prf.value"] * 0.05)
results = detect_blanks.blank_integrated_analysis(refl,
expts[0].scan,
phi_step=5,
fractional_loss=0.1)
assert results["data"][0]["y"] == pytest.approx([
1.3951633074715482,
1.2916263545227527,
26.9236206883069,
26.50234804728626,
26.41019377727383,
25.810676090828185,
24.844906790823064,
25.89992001081651,
25.580718362291474,
])
assert results["data"][0]["blank"] == [
True,
True,
False,
False,
False,
False,
False,
False,
False,
]
assert results["blank_regions"] == [(0, 10)]
# Unset the integrated_prf flags, so the analysis should instead use the umodified
# intensity.sum.value instead
refl.unset_flags(flex.bool(len(refl), True), refl.flags.integrated_prf)
results = detect_blanks.blank_integrated_analysis(refl,
expts[0].scan,
phi_step=5,
fractional_loss=0.1)
assert not any(results["data"][0]["blank"])
assert results["blank_regions"] == []
def load_detector_from_expt(expt_file, exp_id=0):
detector = ExperimentList.from_file(expt_file).detectors()[exp_id]
return detector
def test_forward_panel_edge(dials_data):
expt = ExperimentList.from_file(
dials_data("centroid_test_data").join(
"imported_experiments.json").strpath)[0]
# Get the models
beam = expt.beam
detector = expt.detector
gonio = expt.goniometer
scan = expt.scan
# Set some parameters
sigma_divergence = 0.00101229
mosaicity = 0.157 * math.pi / 180
n_sigma = 3
grid_size = 7
delta_divergence = n_sigma * sigma_divergence
step_size = delta_divergence / grid_size
delta_divergence2 = delta_divergence + step_size * 0.5
delta_mosaicity = n_sigma * mosaicity
# Create the bounding box calculator
calculate_bbox = BBoxCalculator3D(beam, detector, gonio, scan,
delta_divergence2, delta_mosaicity)
# Initialise the transform
spec = transform.TransformSpec(beam, detector, gonio, scan,
sigma_divergence, mosaicity, n_sigma + 1,
grid_size)
assert len(detector) == 1
s0 = beam.get_s0()
m2 = gonio.get_rotation_axis()
s0_length = matrix.col(beam.get_s0()).length()
image_size = detector[0].get_image_size()
refl_xy = [
(0, 0),
(2, 3),
(4, 1000),
(1000, 5),
(image_size[0] - 1, image_size[1] - 1),
(image_size[0] - 2, 1),
(1, image_size[1] - 5),
(1000, image_size[1] - 4),
(image_size[0] - 3, 1000),
]
for x, y in refl_xy:
z = random.uniform(0, 9)
# Get random s1, phi, panel
s1 = matrix.col(detector[0].get_pixel_lab_coord(
(x, y))).normalize() * s0_length
phi = scan.get_angle_from_array_index(z, deg=False)
panel = 0
# Calculate the bounding box
bbox = calculate_bbox(s1, z, panel)
x0, x1, y0, y1, z0, z1 = bbox
# Create the coordinate system
cs = CoordinateSystem(m2, s0, s1, phi)
# Create the image
image = gaussian((z1 - z0, y1 - y0, x1 - x0), 10.0,
(z - z0, y - y0, x - x0), (2.0, 2.0, 2.0))
# Mask for the foreground pixels
refl_mask = image > 1e-3
bg = flex.double(image.accessor())
# Shoebox mask, i.e. mask out pixels that are outside the panel bounds
shoebox_mask = flex.bool(image.accessor(), False)
for j in range(y1 - y0):
for i in range(x1 - x0):
if (j + y0 >= 0 and j + y0 < image_size[1] and i + x0 >= 0
and i + x0 < image_size[0]):
for k in range(z1 - z0):
shoebox_mask[k, j, i] = True
mask = refl_mask & shoebox_mask
# from matplotlib import pyplot as plt
# fig, axes = plt.subplots(ncols=refl_mask.focus()[0], nrows=4)
# for i in range(refl_mask.focus()[0]):
# axes[0, i].imshow(image.as_numpy_array()[i])
# axes[1, i].imshow(refl_mask.as_numpy_array()[i])
# axes[2, i].imshow(shoebox_mask.as_numpy_array()[i])
# axes[3, i].imshow(mask.as_numpy_array()[i])
# plt.show()
# Transform the image to the grid
transformed = transform.TransformForward(spec, cs, bbox, 0,
image.as_double(), bg,
refl_mask)
grid = transformed.profile()
mask = refl_mask & shoebox_mask
# assert only pixels within the panel were transformed
assert flex.sum(grid) == pytest.approx(flex.sum(
image.select(mask.as_1d())),
rel=0.01)
# The total transformed counts should be less than the (unmasked) image counts
assert flex.sum(grid) < flex.sum(image)
# Transform the image to the grid, this time without a background
transformed = transform.TransformForward(spec, cs, bbox, 0,
image.as_double(), refl_mask)
grid = transformed.profile()
mask = refl_mask & shoebox_mask
# assert only pixels within the panel were transformed
assert flex.sum(grid) == pytest.approx(flex.sum(
image.select(mask.as_1d())),
rel=0.01)
# The total transformed counts should be less than the (unmasked) image counts
assert flex.sum(grid) < flex.sum(image)
