def test_experiment_identifiers():
from dials.array_family import flex
from dxtbx.model import ExperimentList, Experiment
table = flex.reflection_table()
table['id'] = flex.int([0,1,2,3])
assert table.are_experiment_identifiers_consistent() == True
identifiers = table.experiment_identifiers()
identifiers[0] = 'abcd'
identifiers[1] = 'efgh'
identifiers[2] = 'ijkl'
identifiers[3] = 'mnop'
assert identifiers[0] == 'abcd'
assert identifiers[1] == 'efgh'
assert identifiers[2] == 'ijkl'
assert identifiers[3] == 'mnop'
for k, v in identifiers:
if k == 0:
assert v == 'abcd'
if k == 1:
assert v == 'efgh'
if k == 2:
assert v == 'ijkl'
if k == 3:
assert v == 'mnop'
assert tuple(identifiers.keys()) == (0, 1, 2, 3)
assert tuple(identifiers.values()) == ("abcd", "efgh", "ijkl", "mnop")
assert table.are_experiment_identifiers_consistent() == True
experiments = ExperimentList()
experiments.append(Experiment(identifier="abcd"))
experiments.append(Experiment(identifier="efgh"))
experiments.append(Experiment(identifier="ijkl"))
experiments.append(Experiment(identifier="mnop"))
assert table.are_experiment_identifiers_consistent(experiments) == True
experiments = ExperimentList()
experiments.append(Experiment(identifier="abcd"))
experiments.append(Experiment(identifier="efgh"))
experiments.append(Experiment(identifier="ijkl"))
experiments.append(Experiment(identifier="mnop"))
experiments[3].identifier = "ijkl"
assert table.are_experiment_identifiers_consistent(experiments) == False
identifiers = table.experiment_identifiers()
identifiers[0] = 'abcd'
identifiers[1] = 'efgh'
identifiers[2] = 'ijkl'
identifiers[3] = 'ijkl'
assert table.are_experiment_identifiers_consistent() == False
identifiers[4] = 'mnop'
import six.moves.cPickle as pickle
pickled = pickle.dumps(table)
table2 = pickle.loads(pickled)
id1 = table.experiment_identifiers()
id2 = table2.experiment_identifiers()
for i in id1.keys():
assert id1[i] == id2[i]
other_table = flex.reflection_table()
other_table['id'] = flex.int([3, 4])
assert other_table.are_experiment_identifiers_consistent() == True
identifiers = other_table.experiment_identifiers()
identifiers[3] = 'mnop'
identifiers[4] = 'qrst'
table.extend(other_table)
assert len(table.experiment_identifiers()) == 5
assert table.experiment_identifiers()[0] == 'abcd'
assert table.experiment_identifiers()[1] == 'efgh'
assert table.experiment_identifiers()[2] == 'ijkl'
assert table.experiment_identifiers()[3] == 'mnop'
assert table.experiment_identifiers()[4] == 'qrst'
def test_forward(dials_data):
expt = ExperimentList.from_file(
dials_data("centroid_test_data").join(
"imported_experiments.json").strpath)[0]
# Get the models
beam = expt.beam
detector = expt.detector
gonio = expt.goniometer
scan = expt.scan
# Set some parameters
sigma_divergence = 0.00101229
mosaicity = 0.157 * math.pi / 180
n_sigma = 3
grid_size = 7
delta_divergence = n_sigma * sigma_divergence
step_size = delta_divergence / grid_size
delta_divergence2 = delta_divergence + step_size * 0.5
delta_mosaicity = n_sigma * mosaicity
# Create the bounding box calculator
calculate_bbox = BBoxCalculator3D(beam, detector, gonio, scan,
delta_divergence2, delta_mosaicity)
# Initialise the transform
spec = transform.TransformSpec(beam, detector, gonio, scan,
sigma_divergence, mosaicity, n_sigma + 1,
grid_size)
# tst_conservation_of_counts(self):
assert len(detector) == 1
s0 = beam.get_s0()
m2 = gonio.get_rotation_axis()
s0_length = matrix.col(beam.get_s0()).length()
# Create an s1 map
s1_map = transform.beam_vector_map(detector[0], beam, True)
for i in range(100):
# Get random x, y, z
x = random.uniform(300, 1800)
y = random.uniform(300, 1800)
z = random.uniform(0, 9)
# Get random s1, phi, panel
s1 = matrix.col(detector[0].get_pixel_lab_coord(
(x, y))).normalize() * s0_length
phi = scan.get_angle_from_array_index(z, deg=False)
panel = 0
# Calculate the bounding box
bbox = calculate_bbox(s1, z, panel)
x0, x1, y0, y1, z0, z1 = bbox
# Create the coordinate system
cs = CoordinateSystem(m2, s0, s1, phi)
# The grid index generator
step_size = delta_divergence / grid_size
grid_index = transform.GridIndexGenerator(cs, x0, y0,
(step_size, step_size),
grid_size, s1_map)
# Create the image
# image = flex.double(flex.grid(z1 - z0, y1 - y0, x1 - x0), 1)
image = gaussian((z1 - z0, y1 - y0, x1 - x0), 10.0,
(z - z0, y - y0, x - x0), (2.0, 2.0, 2.0))
mask = flex.bool(flex.grid(image.all()), False)
for j in range(y1 - y0):
for i in range(x1 - x0):
inside = False
gx00, gy00 = grid_index(j, i)
gx01, gy01 = grid_index(j, i + 1)
gx10, gy10 = grid_index(j + 1, i)
gx11, gy11 = grid_index(j + 1, i + 1)
mingx = min([gx00, gx01, gx10, gx11])
maxgx = max([gx00, gx01, gx10, gx11])
mingy = min([gy00, gy01, gy10, gy11])
maxgy = max([gy00, gy01, gy10, gy11])
if (mingx >= 0 and maxgx < 2 * grid_size + 1 and mingy >= 0
and maxgy < 2 * grid_size + 1):
inside = True
for k in range(1, z1 - z0 - 1):
mask[k, j, i] = inside
# Transform the image to the grid
transformed = transform.TransformForward(spec, cs, bbox, 0,
image.as_double(), mask)
grid = transformed.profile()
# Get the sums and ensure they're the same
eps = 1e-7
sum_grid = flex.sum(grid)
sum_image = flex.sum(flex.double(flex.select(image, flags=mask)))
assert abs(sum_grid - sum_image) <= eps
# Test passed
# tst_transform_with_background(self):
assert len(detector) == 1
s0 = beam.get_s0()
m2 = gonio.get_rotation_axis()
s0_length = matrix.col(beam.get_s0()).length()
# Create an s1 map
s1_map = transform.beam_vector_map(detector[0], beam, True)
for i in range(100):
# Get random x, y, z
x = random.uniform(300, 1800)
y = random.uniform(300, 1800)
z = random.uniform(0, 9)
# Get random s1, phi, panel
s1 = matrix.col(detector[0].get_pixel_lab_coord(
(x, y))).normalize() * s0_length
phi = scan.get_angle_from_array_index(z, deg=False)
panel = 0
# Calculate the bounding box
bbox = calculate_bbox(s1, z, panel)
x0, x1, y0, y1, z0, z1 = bbox
# Create the coordinate system
cs = CoordinateSystem(m2, s0, s1, phi)
# The grid index generator
step_size = delta_divergence / grid_size
grid_index = transform.GridIndexGenerator(cs, x0, y0,
(step_size, step_size),
grid_size, s1_map)
# Create the image
# image = flex.double(flex.grid(z1 - z0, y1 - y0, x1 - x0), 1)
image = gaussian((z1 - z0, y1 - y0, x1 - x0), 10.0,
(z - z0, y - y0, x - x0), (2.0, 2.0, 2.0))
background = flex.random_double(len(image))
background.resize(image.accessor())
mask = flex.bool(flex.grid(image.all()), False)
for j in range(y1 - y0):
for i in range(x1 - x0):
inside = False
gx00, gy00 = grid_index(j, i)
gx01, gy01 = grid_index(j, i + 1)
gx10, gy10 = grid_index(j + 1, i)
gx11, gy11 = grid_index(j + 1, i + 1)
mingx = min([gx00, gx01, gx10, gx11])
maxgx = max([gx00, gx01, gx10, gx11])
mingy = min([gy00, gy01, gy10, gy11])
maxgy = max([gy00, gy01, gy10, gy11])
if (mingx >= 0 and maxgx <= 2 * grid_size + 1 and mingy >= 0
and maxgy <= 2 * grid_size + 1):
inside = True
for k in range(1, z1 - z0 - 1):
mask[k, j, i] = inside
# Transform the image to the grid
transformed = transform.TransformForward(spec, cs, bbox, 0,
image.as_double(),
background.as_double(), mask)
igrid = transformed.profile()
bgrid = transformed.background()
# Get the sums and ensure they're the same
eps = 1e-7
sum_igrid = flex.sum(igrid)
sum_bgrid = flex.sum(bgrid)
sum_image = flex.sum(flex.double(flex.select(image, flags=mask)))
sum_bkgrd = flex.sum(flex.double(flex.select(background, flags=mask)))
try:
assert abs(sum_igrid - sum_image) <= eps
assert abs(sum_bgrid - sum_bkgrd) <= eps
except Exception:
print("Failed for: ", (x, y, z))
raise
def test_map_to_minimum_cell():
# Input and expected output
input_ucs = [
(39.7413, 183.767, 140.649, 90, 90, 90),
(40.16, 142.899, 92.4167, 90, 102.48, 90),
(180.613, 40.1558, 142.737, 90, 90.0174, 90),
]
input_sgs = ["C 2 2 21", "P 1 2 1", "C 1 2 1"]
input_hkl = [
[(1, -75, -71), (1, -73, -70), (1, -71, -69)],
[(14, -37, -36), (-2, -35, -46), (-3, -34, -47)],
[(-31, -5, -3), (-25, -3, -3), (-42, -8, -2)],
]
expected_ucs = [
(39.7413, 94.00755450320204, 140.649, 90.0, 90.0, 77.79717980856927),
(40.16, 92.46399390642911, 142.899, 90.0, 90.0, 77.3882749092846),
(
40.1558,
92.51154528306184,
142.73699999999997,
89.9830147351441,
90.0,
77.46527404307477,
),
]
expected_output_hkl = [
[(-1, 37, -71), (-1, 36, -70), (-1, 35, -69)],
[(-14, 22, 37), (2, 48, 35), (3, 50, 34)],
[(-5, 13, -3), (-3, 11, -3), (-8, 17, -2)],
]
# Setup the input experiments and reflection tables
expts = ExperimentList()
reflections = []
for uc, sg, hkl in zip(input_ucs, input_sgs, input_hkl):
uc = uctbx.unit_cell(uc)
sg = sgtbx.space_group_info(sg).group()
B = scitbx.matrix.sqr(uc.fractionalization_matrix()).transpose()
expts.append(Experiment(crystal=Crystal(B, space_group=sg, reciprocal=True)))
refl = flex.reflection_table()
refl["miller_index"] = flex.miller_index(hkl)
reflections.append(refl)
# Actually run the method we are testing
cb_ops = change_of_basis_ops_to_minimum_cell(
expts, max_delta=5, relative_length_tolerance=0.05, absolute_angle_tolerance=2
)
cb_ops_as_xyz = [cb_op.as_xyz() for cb_op in cb_ops]
# Actual cb_ops are machine dependent (sigh)
assert cb_ops_as_xyz == [
"-x+y,-2*y,z",
"-x+z,-z,-y",
"x+y,-2*x,z",
] or cb_ops_as_xyz == ["x-y,2*y,z", "x-z,z,-y", "-x-y,2*x,z"]
expts_min, reflections = apply_change_of_basis_ops(expts, reflections, cb_ops)
# Verify that the unit cells have been transformed as expected
for expt, uc in zip(expts, expected_ucs):
assert expt.crystal.get_unit_cell().parameters() == pytest.approx(uc, abs=4e-2)
# Space group should be set to P1
assert [expt.crystal.get_space_group().type().number() for expt in expts_min] == [
1,
1,
1,
]
# Verify that the reflections have been reindexed as expected
# Because the exact choice of minimum cell can be platform-dependent,
# compare the magnitude, but not the sign of the output hkl values
for refl, expected_hkl in zip(reflections, expected_output_hkl):
for hkl, e_hkl in zip(refl["miller_index"], expected_hkl):
assert [abs(h) for h in hkl] == [abs(eh) for eh in e_hkl]
def run(self):
''' Parse the options. '''
from dials.util.options import flatten_experiments, flatten_reflections
from dxtbx.model import ExperimentList
from scitbx.math import five_number_summary
# Parse the command line arguments
params, options = self.parser.parse_args(show_diff_phil=True)
self.params = params
experiments = flatten_experiments(params.input.experiments)
reflections = flatten_reflections(params.input.reflections)
assert len(reflections) == 1
reflections = reflections[0]
print("Found", len(reflections), "reflections", "and",
len(experiments), "experiments")
filtered_reflections = flex.reflection_table()
filtered_experiments = ExperimentList()
skipped_reflections = flex.reflection_table()
skipped_experiments = ExperimentList()
if params.detector is not None:
culled_reflections = flex.reflection_table()
culled_experiments = ExperimentList()
detector = experiments.detectors()[params.detector]
for expt_id, experiment in enumerate(experiments):
refls = reflections.select(reflections['id'] == expt_id)
if experiment.detector is detector:
culled_experiments.append(experiment)
refls['id'] = flex.int(len(refls),
len(culled_experiments) - 1)
culled_reflections.extend(refls)
else:
skipped_experiments.append(experiment)
refls['id'] = flex.int(len(refls),
len(skipped_experiments) - 1)
skipped_reflections.extend(refls)
print(
"RMSD filtering %d experiments using detector %d, out of %d" %
(len(culled_experiments), params.detector, len(experiments)))
reflections = culled_reflections
experiments = culled_experiments
difference_vector_norms = (reflections['xyzcal.mm'] -
reflections['xyzobs.mm.value']).norms()
if params.max_delta is not None:
sel = difference_vector_norms <= params.max_delta
reflections = reflections.select(sel)
difference_vector_norms = difference_vector_norms.select(sel)
data = flex.double()
counts = flex.double()
for i in range(len(experiments)):
dvns = difference_vector_norms.select(reflections['id'] == i)
counts.append(len(dvns))
if len(dvns) == 0:
data.append(0)
continue
rmsd = math.sqrt(flex.sum_sq(dvns) / len(dvns))
data.append(rmsd)
data *= 1000
subset = data.select(counts > 0)
print(len(subset), "experiments with > 0 reflections")
if params.show_plots:
h = flex.histogram(subset, n_slots=40)
fig = plt.figure()
ax = fig.add_subplot('111')
ax.plot(h.slot_centers().as_numpy_array(),
h.slots().as_numpy_array(), '-')
plt.title("Histogram of %d image RMSDs" % len(subset))
fig = plt.figure()
plt.boxplot(subset, vert=False)
plt.title("Boxplot of %d image RMSDs" % len(subset))
plt.show()
outliers = counts == 0
min_x, q1_x, med_x, q3_x, max_x = five_number_summary(subset)
print(
"Five number summary of RMSDs (microns): min %.1f, q1 %.1f, med %.1f, q3 %.1f, max %.1f"
% (min_x, q1_x, med_x, q3_x, max_x))
iqr_x = q3_x - q1_x
cut_x = params.iqr_multiplier * iqr_x
outliers.set_selected(data > q3_x + cut_x, True)
#outliers.set_selected(col < q1_x - cut_x, True) # Don't throw away the images that are outliers in the 'good' direction!
for i in range(len(experiments)):
if outliers[i]:
continue
refls = reflections.select(reflections['id'] == i)
refls['id'] = flex.int(len(refls), len(filtered_experiments))
filtered_reflections.extend(refls)
filtered_experiments.append(experiments[i])
zeroes = counts == 0
n_zero = len(counts.select(zeroes))
print(
"Removed %d bad experiments and %d experiments with zero reflections, out of %d (%%%.1f)"
% (len(experiments) - len(filtered_experiments) - n_zero, n_zero,
len(experiments), 100 *
((len(experiments) - len(filtered_experiments)) /
len(experiments))))
if params.detector is not None:
crystals = filtered_experiments.crystals()
for expt_id, experiment in enumerate(skipped_experiments):
if experiment.crystal in crystals:
filtered_experiments.append(experiment)
refls = skipped_reflections.select(
skipped_reflections['id'] == expt_id)
refls['id'] = flex.int(len(refls),
len(filtered_experiments) - 1)
filtered_reflections.extend(refls)
if params.delta_psi_filter is not None:
delta_psi = filtered_reflections['delpsical.rad'] * 180 / math.pi
sel = (delta_psi <= params.delta_psi_filter) & (
delta_psi >= -params.delta_psi_filter)
l = len(filtered_reflections)
filtered_reflections = filtered_reflections.select(sel)
print("Filtering by delta psi, removing %d out of %d reflections" %
(l - len(filtered_reflections), l))
print("Final experiment count", len(filtered_experiments))
filtered_experiments.as_file(params.output.filtered_experiments)
filtered_reflections.as_pickle(params.output.filtered_reflections)
def test_forward_no_model(dials_data):
expt = ExperimentList.from_file(
dials_data("centroid_test_data").join(
"imported_experiments.json").strpath)[0]
# Get the models
beam = expt.beam
detector = expt.detector
gonio = expt.goniometer
scan = expt.scan
scan.set_image_range((0, 1000))
# Set some parameters
sigma_divergence = 0.00101229
mosaicity = 0.157 * math.pi / 180
n_sigma = 3
grid_size = 20
delta_divergence = n_sigma * sigma_divergence
step_size = delta_divergence / grid_size
delta_divergence2 = delta_divergence + step_size * 0.5
delta_mosaicity = n_sigma * mosaicity
# Create the bounding box calculator
calculate_bbox = BBoxCalculator3D(beam, detector, gonio, scan,
delta_divergence2, delta_mosaicity)
# Initialise the transform
spec = transform.TransformSpec(beam, detector, gonio, scan,
sigma_divergence, mosaicity, n_sigma + 1,
grid_size)
# tst_conservation_of_counts(self):
random.seed(0)
assert len(detector) == 1
s0 = beam.get_s0()
m2 = gonio.get_rotation_axis()
s0_length = matrix.col(beam.get_s0()).length()
# Create an s1 map
s1_map = transform.beam_vector_map(detector[0], beam, True)
for i in range(100):
# Get random x, y, z
x = random.uniform(300, 1800)
y = random.uniform(300, 1800)
z = random.uniform(500, 600)
# Get random s1, phi, panel
s1 = matrix.col(detector[0].get_pixel_lab_coord(
(x, y))).normalize() * s0_length
phi = scan.get_angle_from_array_index(z, deg=False)
panel = 0
# Calculate the bounding box
bbox = calculate_bbox(s1, z, panel)
x0, x1, y0, y1, z0, z1 = bbox
# Create the coordinate system
cs = CoordinateSystem(m2, s0, s1, phi)
if abs(cs.zeta()) < 0.1:
continue
# The grid index generator
step_size = delta_divergence / grid_size
grid_index = transform.GridIndexGenerator(cs, x0, y0,
(step_size, step_size),
grid_size, s1_map)
# Create the image
# image = flex.double(flex.grid(z1 - z0, y1 - y0, x1 - x0), 1)
image = gaussian((z1 - z0, y1 - y0, x1 - x0), 10.0,
(z - z0, y - y0, x - x0), (2.0, 2.0, 2.0))
mask = flex.bool(flex.grid(image.all()), False)
for j in range(y1 - y0):
for i in range(x1 - x0):
inside = False
gx00, gy00 = grid_index(j, i)
gx01, gy01 = grid_index(j, i + 1)
gx10, gy10 = grid_index(j + 1, i)
gx11, gy11 = grid_index(j + 1, i + 1)
mingx = min([gx00, gx01, gx10, gx11])
maxgx = max([gx00, gx01, gx10, gx11])
mingy = min([gy00, gy01, gy10, gy11])
maxgy = max([gy00, gy01, gy10, gy11])
if (mingx >= 0 and maxgx < 2 * grid_size + 1 and mingy >= 0
and maxgy < 2 * grid_size + 1):
inside = True
for k in range(1, z1 - z0 - 1):
mask[k, j, i] = inside
# Transform the image to the grid
transformed = transform.TransformForwardNoModel(
spec, cs, bbox, 0, image.as_double(), mask)
grid = transformed.profile()
# Get the sums and ensure they're the same
eps = 1e-7
sum_grid = flex.sum(grid)
sum_image = flex.sum(flex.double(flex.select(image, flags=mask)))
assert abs(sum_grid - sum_image) <= eps
mask = flex.bool(flex.grid(image.all()), True)
transformed = transform.TransformForwardNoModel(
spec, cs, bbox, 0, image.as_double(), mask)
grid = transformed.profile()
# Boost the bbox to make sure all intensity is included
x0, x1, y0, y1, z0, z1 = bbox
bbox2 = (x0 - 10, x1 + 10, y0 - 10, y1 + 10, z0 - 10, z1 + 10)
# Do the reverse transform
transformed = transform.TransformReverseNoModel(
spec, cs, bbox2, 0, grid)
image2 = transformed.profile()
# Check the sum of pixels are the same
sum_grid = flex.sum(grid)
sum_image = flex.sum(image2)
assert abs(sum_grid - sum_image) <= eps
# Do the reverse transform
transformed = transform.TransformReverseNoModel(
spec, cs, bbox, 0, grid)
image2 = transformed.profile()
from dials.algorithms.statistics import pearson_correlation_coefficient
cc = pearson_correlation_coefficient(image.as_1d().as_double(),
image2.as_1d())
assert cc >= 0.99
def run(args):
usage = "dev.dials.csv [options] imported.expt strong.refl output.csv=rl.csv"
parser = OptionParser(
usage=usage,
phil=phil_scope,
read_experiments=True,
read_reflections=True,
check_format=False,
)
params, options = parser.parse_args(show_diff_phil=False)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
if not experiments or not reflections:
parser.print_help()
exit(0)
spots = []
for reflection in reflections:
unique_ids = set(reflection["id"])
for unique_id in sorted(unique_ids):
spots.append(reflection.select(reflection["id"] == unique_id))
if not reflection: # If there are no reflections then export an empty list
spots.append(reflection)
assert len(experiments) == len(spots)
if params.output.compress:
fout = gzip.GzipFile(params.output.csv, "w")
if six.PY3:
# GzipFile() always provides binary access only.
# Replace the file object with one that allows writing text:
fout = io.TextIOWrapper(fout)
# Rely on garbage collection to close the underlying GzipFile.
else:
fout = open(params.output.csv, "w")
fout.write("# x,y,z,experiment_id,imageset_id\n")
dp = params.output.dp
if dp <= 0:
fmt = "%f,%f,%f,%d,%d\n"
else:
fmt = "%%.%df,%%.%df,%%.%df,%%d,%%d\n" % (dp, dp, dp)
print("Using format:", fmt.strip())
for k, (expt, refl) in enumerate(zip(experiments, spots)):
if "imageset_id" not in refl:
refl["imageset_id"] = refl["id"]
refl.centroid_px_to_mm(ExperimentList([expt]))
refl.map_centroids_to_reciprocal_space(ExperimentList([expt]))
rlp = refl["rlp"]
for _rlp in rlp:
fout.write(fmt % (_rlp[0], _rlp[1], _rlp[2], k, k))
print("Appended %d spots to %s" % (len(rlp), params.output.csv))
fout.close()
def test_stills_indexer_multi_lattice_bug_MosaicSauter2014(
dials_regression, tmpdir):
""" Problem: In stills_indexer, before calling the refine function, the experiment list contains a list of
dxtbx crystal models (that are not MosaicSauter2014 models). The conversion to MosaicSauter2014 is made
during the refine step when functions from nave_parameters is called. If the experiment list contains
more than 1 experiment, for eg. multiple lattices, only the first crystal gets assigned mosaicity. In
actuality, all crystal models should be assigned mosaicity. This test only compares whether or not all crystal models
have been assigned a MosaicSauter2014 model. """
from dxtbx.model.experiment_list import ExperimentListFactory
from dxtbx.model.experiment_list import Experiment, ExperimentList
from dials.array_family import flex
from dxtbx.model import Crystal
from dials.algorithms.indexing.stills_indexer import StillsIndexer
from dials.command_line.stills_process import (
phil_scope as stills_process_phil_scope, )
import dxtbx_model_ext # needed for comparison of types
experiment_data = os.path.join(
dials_regression,
"refinement_test_data",
"cspad_refinement",
"cspad_refined_experiments_step6_level2_300.json",
)
reflection_data = os.path.join(
dials_regression,
"refinement_test_data",
"cspad_refinement",
"cspad_reflections_step7_300.pickle",
)
refl = flex.reflection_table.from_file(reflection_data)
explist = ExperimentListFactory.from_json_file(experiment_data,
check_format=False)[0:2]
reflist = refl.select(
refl["id"] < 2) # Only use the first 2 for convenience
# Construct crystal models that don't have mosaicity. These A,B,C values are the same
# as read in from the dials_regression folder
# Crystal-0
cs0 = Crystal(explist[0].crystal)
exp0 = Experiment(
imageset=explist[0].imageset,
beam=explist[0].beam,
detector=explist[0].detector,
goniometer=None,
scan=None,
crystal=cs0,
)
# Crystal-1
cs1 = Crystal(explist[1].crystal)
exp1 = Experiment(
imageset=explist[1].imageset,
beam=explist[1].beam,
detector=explist[1].detector,
goniometer=None,
scan=None,
crystal=cs1,
)
# Construct a new experiment_list that will be passed on for refinement
unrefined_explist = ExperimentList([exp0, exp1])
# Get default params from stills_process and construct StillsIndexer, then run refinement
params = stills_process_phil_scope.extract()
SI = StillsIndexer(reflist, unrefined_explist, params=params)
refined_explist, new_reflist = SI.refine(unrefined_explist, reflist)
# Now check whether the models have mosaicity after stills_indexer refinement
# Also check that mosaicity values are within expected limits
for ii, crys in enumerate(refined_explist.crystals()):
assert isinstance(crys, dxtbx_model_ext.MosaicCrystalSauter2014)
if ii == 0:
assert crys.get_domain_size_ang() == pytest.approx(2242.0, rel=0.1)
if ii == 1:
assert crys.get_domain_size_ang() == pytest.approx(2689.0, rel=0.1)
writer.add_image(noisy)
writer.add_image(sim_data)
writer.close_file()
shot_data += list(zip([i_shot]*len(cents), cents , cents_mm, i_refs ) )
print("Rank %d: shot %d / %d has %d/%d refls TOok %f seconds to simulate %d panels, Ncells=%d %d %d "
% (rank, i_shot, Nexper, len(i_refs), len(exper_refls_strong), tsim, len(panels_with_spots), Ncells_abc[0], Ncells_abc[1], Ncells_abc[2]), flush=True)
if has_mpi:
shot_data = comm.reduce(shot_data)
if rank==0:
print("Saving all data")
exper_ids, cents, cents_mm, i_refs = zip(*shot_data)
El_out = ExperimentList()
Rmaster_out = flex.reflection_table()
Rmaster_orig = flex.reflection_table()
df = pandas.DataFrame({"exper_ids":exper_ids, "cents": cents, "cents_mm": cents_mm, "i_refs": i_refs})
unique_exper_ids = df.exper_ids.unique()
print("%d unique experiments with %d total reflections" % (len(unique_exper_ids), len(cents) ))
from copy import deepcopy
for new_exper_id, exper_id in enumerate(unique_exper_ids):
exper_id = int(exper_id)
df_id = df.query("exper_ids==%d" % exper_id)
r = Rmaster.select(Rmaster["id"]==exper_id)
r_orig = deepcopy(r)
sel = flex.bool([i in df_id.i_refs.values for i in range(len(r))])
cents = [val for val in df_id.cents.values]
cents_mm = [val for val in df_id.cents_mm.values]
for ii, i_ref in enumerate(df_id.i_refs.values):
def _refine(self):
for idxr in set(self._refinr_indexers.values()):
experiments = idxr.get_indexer_experiment_list()
indexed_experiments = idxr.get_indexer_payload(
"experiments_filename")
indexed_reflections = idxr.get_indexer_payload("indexed_filename")
# If multiple sweeps but not doing joint refinement, get only the
# relevant reflections.
multi_sweep = PhilIndex.params.xia2.settings.multi_sweep_refinement
if len(experiments) > 1 and not multi_sweep:
xsweeps = idxr._indxr_sweeps
assert len(xsweeps) == len(experiments)
# Don't do joint refinement
assert len(self._refinr_sweeps) == 1
xsweep = self._refinr_sweeps[0]
i = xsweeps.index(xsweep)
experiments = experiments[i:i + 1]
# Extract and output experiment and reflections for current sweep
indexed_experiments = os.path.join(
self.get_working_directory(),
"%s_indexed.expt" % xsweep.get_name())
indexed_reflections = os.path.join(
self.get_working_directory(),
"%s_indexed.refl" % xsweep.get_name())
experiments.as_file(indexed_experiments)
reflections = flex.reflection_table.from_file(
idxr.get_indexer_payload("indexed_filename"))
sel = reflections["id"] == i
assert sel.count(True) > 0
imageset_id = reflections["imageset_id"].select(sel)
assert imageset_id.all_eq(imageset_id[0])
sel = reflections["imageset_id"] == imageset_id[0]
reflections = reflections.select(sel)
# set indexed reflections to id == 0 and imageset_id == 0
reflections["id"].set_selected(reflections["id"] == i, 0)
reflections["imageset_id"] = flex.int(len(reflections), 0)
reflections.as_file(indexed_reflections)
# currently only handle one lattice/refiner
assert len(experiments.crystals()) == 1
scan_static = PhilIndex.params.dials.refine.scan_static
# Avoid doing scan-varying refinement on narrow wedges.
scan_oscillation_ranges = []
for experiment in experiments:
start, end = experiment.scan.get_oscillation_range()
scan_oscillation_ranges.append(end - start)
min_oscillation_range = min(scan_oscillation_ranges)
if (PhilIndex.params.dials.refine.scan_varying
and min_oscillation_range > 5
and not PhilIndex.params.dials.fast_mode):
scan_varying = PhilIndex.params.dials.refine.scan_varying
else:
scan_varying = False
if scan_static:
refiner = self.Refine()
refiner.set_experiments_filename(indexed_experiments)
refiner.set_indexed_filename(indexed_reflections)
refiner.set_scan_varying(False)
refiner.run()
self._refinr_experiments_filename = (
refiner.get_refined_experiments_filename())
self._refinr_indexed_filename = refiner.get_refined_filename()
else:
self._refinr_experiments_filename = indexed_experiments
self._refinr_indexed_filename = indexed_reflections
if scan_varying:
refiner = self.Refine()
refiner.set_experiments_filename(
self._refinr_experiments_filename)
refiner.set_indexed_filename(self._refinr_indexed_filename)
if min_oscillation_range < 36:
refiner.set_interval_width_degrees(min_oscillation_range /
2)
refiner.run()
self._refinr_experiments_filename = (
refiner.get_refined_experiments_filename())
self._refinr_indexed_filename = refiner.get_refined_filename()
if scan_static or scan_varying:
FileHandler.record_log_file(
"%s REFINE" % idxr.get_indexer_full_name(),
refiner.get_log_file())
report = self.Report()
report.set_experiments_filename(
self._refinr_experiments_filename)
report.set_reflections_filename(self._refinr_indexed_filename)
html_filename = os.path.join(
self.get_working_directory(),
"%i_dials.refine.report.html" % report.get_xpid(),
)
report.set_html_filename(html_filename)
report.run()
FileHandler.record_html_file(
"%s REFINE" % idxr.get_indexer_full_name(), html_filename)
experiments = ExperimentList.from_file(
self._refinr_experiments_filename)
self.set_refiner_payload("models.expt",
self._refinr_experiments_filename)
self.set_refiner_payload("observations.refl",
self._refinr_indexed_filename)
# this is the result of the cell refinement
self._refinr_cell = experiments.crystals()[0].get_unit_cell(
).parameters()
def __init__(self, params):
from six.moves import cPickle as pickle
from dxtbx.model import BeamFactory
from dxtbx.model import DetectorFactory
from dxtbx.model.crystal import CrystalFactory
from cctbx.crystal_orientation import crystal_orientation, basis_type
from dxtbx.model import Experiment, ExperimentList
from scitbx import matrix
self.experiments = ExperimentList()
self.unique_file_names = []
self.params = params
data = pickle.load(
open(self.params.output.prefix + "_frame.pickle", "rb"))
frames_text = data.split("\n")
for item in frames_text:
tokens = item.split(' ')
wavelength = float(tokens[order_dict["wavelength"]])
beam = BeamFactory.simple(wavelength=wavelength)
detector = DetectorFactory.simple(
sensor=DetectorFactory.sensor(
"PAD"), # XXX shouldn't hard code for XFEL
distance=float(tokens[order_dict["distance"]]),
beam_centre=[
float(tokens[order_dict["beam_x"]]),
float(tokens[order_dict["beam_y"]])
],
fast_direction="+x",
slow_direction="+y",
pixel_size=[self.params.pixel_size, self.params.pixel_size],
image_size=[1795,
1795], # XXX obviously need to figure this out
)
reciprocal_matrix = matrix.sqr([
float(tokens[order_dict[k]]) for k in [
'res_ori_1', 'res_ori_2', 'res_ori_3', 'res_ori_4',
'res_ori_5', 'res_ori_6', 'res_ori_7', 'res_ori_8',
'res_ori_9'
]
])
ORI = crystal_orientation(reciprocal_matrix, basis_type.reciprocal)
direct = matrix.sqr(ORI.direct_matrix())
transfer_dict = dict(
__id__="crystal",
ML_half_mosaicity_deg=float(
tokens[order_dict["half_mosaicity_deg"]]),
ML_domain_size_ang=float(
tokens[order_dict["domain_size_ang"]]),
real_space_a=matrix.row(direct[0:3]),
real_space_b=matrix.row(direct[3:6]),
real_space_c=matrix.row(direct[6:9]),
space_group_hall_symbol=self.params.target_space_group.type().
hall_symbol(),
)
crystal = CrystalFactory.from_dict(transfer_dict)
""" old code reflects python-based crystal model
crystal = Crystal(
real_space_a = matrix.row(direct[0:3]),
real_space_b = matrix.row(direct[3:6]),
real_space_c = matrix.row(direct[6:9]),
space_group_symbol = self.params.target_space_group.type().lookup_symbol(),
mosaicity = float(tokens[order_dict["half_mosaicity_deg"]]),
)
crystal.domain_size = float(tokens[order_dict["domain_size_ang"]])
"""
#if isoform is not None:
# newB = matrix.sqr(isoform.fractionalization_matrix()).transpose()
# crystal.set_B(newB)
self.experiments.append(
Experiment(
beam=beam,
detector=None, #dummy for now
crystal=crystal))
self.unique_file_names.append(
tokens[order_dict["unique_file_name"]])
self.show_summary()
def mock_exp_obj(id_=0):
"""Make a mock experiments file object."""
exp = Mock()
exp.data = ExperimentList()
exp.data.append(Experiment(identifier=str(id_)))
return exp
class read_experiments(object):
def __init__(self, params):
from six.moves import cPickle as pickle
from dxtbx.model import BeamFactory
from dxtbx.model import DetectorFactory
from dxtbx.model.crystal import CrystalFactory
from cctbx.crystal_orientation import crystal_orientation, basis_type
from dxtbx.model import Experiment, ExperimentList
from scitbx import matrix
self.experiments = ExperimentList()
self.unique_file_names = []
self.params = params
data = pickle.load(
open(self.params.output.prefix + "_frame.pickle", "rb"))
frames_text = data.split("\n")
for item in frames_text:
tokens = item.split(' ')
wavelength = float(tokens[order_dict["wavelength"]])
beam = BeamFactory.simple(wavelength=wavelength)
detector = DetectorFactory.simple(
sensor=DetectorFactory.sensor(
"PAD"), # XXX shouldn't hard code for XFEL
distance=float(tokens[order_dict["distance"]]),
beam_centre=[
float(tokens[order_dict["beam_x"]]),
float(tokens[order_dict["beam_y"]])
],
fast_direction="+x",
slow_direction="+y",
pixel_size=[self.params.pixel_size, self.params.pixel_size],
image_size=[1795,
1795], # XXX obviously need to figure this out
)
reciprocal_matrix = matrix.sqr([
float(tokens[order_dict[k]]) for k in [
'res_ori_1', 'res_ori_2', 'res_ori_3', 'res_ori_4',
'res_ori_5', 'res_ori_6', 'res_ori_7', 'res_ori_8',
'res_ori_9'
]
])
ORI = crystal_orientation(reciprocal_matrix, basis_type.reciprocal)
direct = matrix.sqr(ORI.direct_matrix())
transfer_dict = dict(
__id__="crystal",
ML_half_mosaicity_deg=float(
tokens[order_dict["half_mosaicity_deg"]]),
ML_domain_size_ang=float(
tokens[order_dict["domain_size_ang"]]),
real_space_a=matrix.row(direct[0:3]),
real_space_b=matrix.row(direct[3:6]),
real_space_c=matrix.row(direct[6:9]),
space_group_hall_symbol=self.params.target_space_group.type().
hall_symbol(),
)
crystal = CrystalFactory.from_dict(transfer_dict)
""" old code reflects python-based crystal model
crystal = Crystal(
real_space_a = matrix.row(direct[0:3]),
real_space_b = matrix.row(direct[3:6]),
real_space_c = matrix.row(direct[6:9]),
space_group_symbol = self.params.target_space_group.type().lookup_symbol(),
mosaicity = float(tokens[order_dict["half_mosaicity_deg"]]),
)
crystal.domain_size = float(tokens[order_dict["domain_size_ang"]])
"""
#if isoform is not None:
# newB = matrix.sqr(isoform.fractionalization_matrix()).transpose()
# crystal.set_B(newB)
self.experiments.append(
Experiment(
beam=beam,
detector=None, #dummy for now
crystal=crystal))
self.unique_file_names.append(
tokens[order_dict["unique_file_name"]])
self.show_summary()
def get_experiments(self):
return self.experiments
def get_files(self):
return self.unique_file_names
def show_summary(self):
w = flex.double([e.beam.get_wavelength() for e in self.experiments])
stats = flex.mean_and_variance(w)
print "Wavelength mean and standard deviation:", stats.mean(
), stats.unweighted_sample_standard_deviation()
uc = [e.crystal.get_unit_cell().parameters() for e in self.experiments]
a = flex.double([u[0] for u in uc])
stats = flex.mean_and_variance(a)
print "Unit cell a mean and standard deviation:", stats.mean(
), stats.unweighted_sample_standard_deviation()
b = flex.double([u[1] for u in uc])
stats = flex.mean_and_variance(b)
print "Unit cell b mean and standard deviation:", stats.mean(
), stats.unweighted_sample_standard_deviation()
c = flex.double([u[2] for u in uc])
stats = flex.mean_and_variance(c)
print "Unit cell c mean and standard deviation:", stats.mean(
), stats.unweighted_sample_standard_deviation()
d = flex.double(
[e.crystal.get_domain_size_ang() for e in self.experiments])
stats = flex.mean_and_variance(d)
# NOTE XXX FIXME: cxi.index seems to record the half-domain size; report here the full domain size
print "Domain size mean and standard deviation:", 2. * stats.mean(
), 2. * stats.unweighted_sample_standard_deviation()
def merge_data_to_mtz(params, experiments, reflections):
"""Merge data (at each wavelength) and write to an mtz file object."""
wavelengths = match_wavelengths(experiments) # wavelengths is an ordered dict
mtz_datasets = [
MTZDataClass(wavelength=w, project_name=params.output.project_name)
for w in wavelengths.keys()
]
dataset_names = params.output.dataset_names
crystal_names = params.output.crystal_names
# check if best_unit_cell is set.
best_unit_cell = params.best_unit_cell
if not best_unit_cell:
best_unit_cell = determine_best_unit_cell(experiments)
reflections[0]["d"] = best_unit_cell.d(reflections[0]["miller_index"])
for expt in experiments:
expt.crystal.unit_cell = best_unit_cell
if len(wavelengths) > 1:
logger.info(
"Multiple wavelengths found: \n%s",
"\n".join(
" Wavlength: %.5f, experiment numbers: %s "
% (k, ",".join(map(str, v)))
for k, v in wavelengths.items()
),
)
if not dataset_names or len(dataset_names) != len(wavelengths):
logger.info(
"Unequal number of dataset names and wavelengths, using default naming."
)
dataset_names = [None] * len(wavelengths)
if not crystal_names or len(crystal_names) != len(wavelengths):
logger.info(
"Unequal number of crystal names and wavelengths, using default naming."
)
crystal_names = [None] * len(wavelengths)
experiments_subsets = []
reflections_subsets = []
for dataset, dname, cname in zip(mtz_datasets, dataset_names, crystal_names):
dataset.dataset_name = dname
dataset.crystal_name = cname
for exp_nos in wavelengths.values():
expids = [experiments[i].identifier for i in exp_nos]
experiments_subsets.append(
ExperimentList([experiments[i] for i in exp_nos])
)
reflections_subsets.append(
reflections[0].select_on_experiment_identifiers(expids)
)
else:
mtz_datasets[0].dataset_name = dataset_names[0]
mtz_datasets[0].crystal_name = crystal_names[0]
experiments_subsets = [experiments]
reflections_subsets = reflections
for experimentlist, reflection_table, mtz_dataset in zip(
experiments_subsets, reflections_subsets, mtz_datasets
):
# merge and truncate the data
merged_array, merged_anomalous_array, stats_summary = merge(
experimentlist,
reflection_table,
d_min=params.d_min,
d_max=params.d_max,
combine_partials=params.combine_partials,
partiality_threshold=params.partiality_threshold,
best_unit_cell=best_unit_cell,
anomalous=params.anomalous,
assess_space_group=params.assess_space_group,
n_bins=params.merging.n_bins,
use_internal_variance=params.merging.use_internal_variance,
)
mtz_dataset.merged_array = merged_array
mtz_dataset.merged_anomalous_array = merged_anomalous_array
if params.anomalous:
merged_intensities = merged_anomalous_array
else:
merged_intensities = merged_array
if params.truncate:
amplitudes, anomalous_amplitudes = truncate(merged_intensities)
mtz_dataset.amplitudes = amplitudes
mtz_dataset.anomalous_amplitudes = anomalous_amplitudes
show_wilson_scaling_analysis(merged_intensities)
if stats_summary:
logger.info(stats_summary)
return make_merged_mtz_file(mtz_datasets)
def _index(self):
'''Actually index the diffraction pattern. Note well that
this is not going to compute the matrix...'''
# acknowledge this program
Citations.cite('labelit')
Citations.cite('distl')
#self.reset()
_images = []
for i in self._indxr_images:
for j in i:
if not j in _images:
_images.append(j)
_images.sort()
images_str = '%d' % _images[0]
for i in _images[1:]:
images_str += ', %d' % i
cell_str = None
if self._indxr_input_cell:
cell_str = '%.2f %.2f %.2f %.2f %.2f %.2f' % \
self._indxr_input_cell
if self._indxr_sweep_name:
# then this is a proper autoindexing run - describe this
# to the journal entry
#if len(self._fp_directory) <= 50:
#dirname = self._fp_directory
#else:
#dirname = '...%s' % self._fp_directory[-46:]
dirname = os.path.dirname(self.get_imageset().get_template())
Journal.block(
'autoindexing', self._indxr_sweep_name, 'labelit',
{'images':images_str,
'target cell':cell_str,
'target lattice':self._indxr_input_lattice,
'template':self.get_imageset().get_template(),
'directory':dirname})
if len(_images) > 4:
raise RuntimeError('cannot use more than 4 images')
from xia2.Wrappers.Labelit.LabelitIndex import LabelitIndex
index = LabelitIndex()
index.set_working_directory(self.get_working_directory())
auto_logfiler(index)
#task = 'Autoindex from images:'
#for i in _images:
#task += ' %s' % self.get_image_name(i)
#self.set_task(task)
Debug.write('Indexing from images:')
for i in _images:
index.add_image(self.get_image_name(i))
Debug.write('%s' % self.get_image_name(i))
xsweep = self.get_indexer_sweep()
if xsweep is not None:
if xsweep.get_distance() is not None:
index.set_distance(xsweep.get_distance())
#if self.get_wavelength_prov() == 'user':
#index.set_wavelength(self.get_wavelength())
if xsweep.get_beam_centre() is not None:
index.set_beam_centre(xsweep.get_beam_centre())
if self._refine_beam is False:
index.set_refine_beam(False)
else:
index.set_refine_beam(True)
index.set_beam_search_scope(self._beam_search_scope)
if ((math.fabs(self.get_wavelength() - 1.54) < 0.01) or
(math.fabs(self.get_wavelength() - 2.29) < 0.01)):
index.set_Cu_KA_or_Cr_KA(True)
#sweep = self.get_indexer_sweep_name()
#FileHandler.record_log_file(
#'%s INDEX' % (sweep), self.get_log_file())
try:
index.run()
except RuntimeError as e:
if self._refine_beam is False:
raise e
# can we improve the situation?
if self._beam_search_scope < 4.0:
self._beam_search_scope += 4.0
# try repeating the indexing!
self.set_indexer_done(False)
return 'failed'
# otherwise this is beyond redemption
raise e
self._solutions = index.get_solutions()
# FIXME this needs to check the smilie status e.g.
# ":)" or ";(" or " ".
# FIXME need to check the value of the RMSD and raise an
# exception if the P1 solution has an RMSD > 1.0...
# Change 27/FEB/08 to support user assigned spacegroups
# (euugh!) have to "ignore" solutions with higher symmetry
# otherwise the rest of xia will override us. Bummer.
for i, solution in self._solutions.iteritems():
if self._indxr_user_input_lattice:
if (lattice_to_spacegroup(solution['lattice']) >
lattice_to_spacegroup(self._indxr_input_lattice)):
Debug.write('Ignoring solution: %s' % solution['lattice'])
del self._solutions[i]
# check the RMSD from the triclinic unit cell
if self._solutions[1]['rmsd'] > 1.0 and False:
# don't know when this is useful - but I know when it is not!
raise RuntimeError('high RMSD for triclinic solution')
# configure the "right" solution
self._solution = self.get_solution()
# now store also all of the other solutions... keyed by the
# lattice - however these should only be added if they
# have a smiley in the appropriate record, perhaps?
for solution in self._solutions.keys():
lattice = self._solutions[solution]['lattice']
if lattice in self._indxr_other_lattice_cell:
if self._indxr_other_lattice_cell[lattice]['goodness'] < \
self._solutions[solution]['metric']:
continue
self._indxr_other_lattice_cell[lattice] = {
'goodness':self._solutions[solution]['metric'],
'cell':self._solutions[solution]['cell']}
self._indxr_lattice = self._solution['lattice']
self._indxr_cell = tuple(self._solution['cell'])
self._indxr_mosaic = self._solution['mosaic']
lms = LabelitMosflmScript()
lms.set_working_directory(self.get_working_directory())
lms.set_solution(self._solution['number'])
self._indxr_payload['mosflm_orientation_matrix'] = lms.calculate()
# get the beam centre from the mosflm script - mosflm
# may have inverted the beam centre and labelit will know
# this!
mosflm_beam_centre = lms.get_mosflm_beam()
if mosflm_beam_centre:
self._indxr_payload['mosflm_beam_centre'] = tuple(mosflm_beam_centre)
import copy
detector = copy.deepcopy(self.get_detector())
beam = copy.deepcopy(self.get_beam())
from dxtbx.model.detector_helpers import set_mosflm_beam_centre
set_mosflm_beam_centre(detector, beam, mosflm_beam_centre)
from xia2.Experts.SymmetryExpert import lattice_to_spacegroup_number
from scitbx import matrix
from cctbx import sgtbx, uctbx
from dxtbx.model import CrystalFactory
mosflm_matrix = matrix.sqr(
[float(i) for line in lms.calculate()
for i in line.replace("-", " -").split() ][:9])
space_group = sgtbx.space_group_info(lattice_to_spacegroup_number(
self._solution['lattice'])).group()
crystal_model = CrystalFactory.from_mosflm_matrix(
mosflm_matrix,
unit_cell=uctbx.unit_cell(
tuple(self._solution['cell'])),
space_group=space_group)
from dxtbx.model import Experiment, ExperimentList
experiment = Experiment(beam=beam,
detector=detector,
goniometer=self.get_goniometer(),
scan=self.get_scan(),
crystal=crystal_model,
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# also get an estimate of the resolution limit from the
# labelit.stats_distl output... FIXME the name is wrong!
lsd = LabelitStats_distl()
lsd.set_working_directory(self.get_working_directory())
lsd.stats_distl()
resolution = 1.0e6
for i in _images:
stats = lsd.get_statistics(self.get_image_name(i))
resol = 0.5 * (stats['resol_one'] + stats['resol_two'])
if resol < resolution:
resolution = resol
self._indxr_resolution_estimate = resolution
return 'ok'
def test_experiment_identifiers():
from dxtbx.model import Experiment, ExperimentList
table = flex.reflection_table()
table["id"] = flex.int([0, 1, 2, 3])
table.assert_experiment_identifiers_are_consistent()
identifiers = table.experiment_identifiers()
identifiers[0] = "abcd"
identifiers[1] = "efgh"
identifiers[2] = "ijkl"
identifiers[3] = "mnop"
assert identifiers[0] == "abcd"
assert identifiers[1] == "efgh"
assert identifiers[2] == "ijkl"
assert identifiers[3] == "mnop"
for k, v in identifiers:
if k == 0:
assert v == "abcd"
if k == 1:
assert v == "efgh"
if k == 2:
assert v == "ijkl"
if k == 3:
assert v == "mnop"
assert tuple(identifiers.keys()) == (0, 1, 2, 3)
assert tuple(identifiers.values()) == ("abcd", "efgh", "ijkl", "mnop")
table.assert_experiment_identifiers_are_consistent()
experiments = ExperimentList()
experiments.append(Experiment(identifier="abcd"))
experiments.append(Experiment(identifier="efgh"))
experiments.append(Experiment(identifier="ijkl"))
experiments.append(Experiment(identifier="mnop"))
table.assert_experiment_identifiers_are_consistent()
experiments = ExperimentList()
experiments.append(Experiment(identifier="abcd"))
experiments.append(Experiment(identifier="efgh"))
experiments.append(Experiment(identifier="ijkl"))
experiments.append(Experiment(identifier="mnop"))
experiments[3].identifier = "ijkl"
with pytest.raises(AssertionError):
table.assert_experiment_identifiers_are_consistent(experiments)
experiments[2].identifier = "mnop"
table.assert_experiment_identifiers_are_consistent(experiments)
identifiers = table.experiment_identifiers()
identifiers[0] = "abcd"
identifiers[1] = "efgh"
identifiers[2] = "ijkl"
identifiers[3] = "ijkl"
with pytest.raises(AssertionError):
table.assert_experiment_identifiers_are_consistent()
identifiers[3] = "mnop"
pickled = pickle.dumps(table)
table2 = pickle.loads(pickled)
id1 = table.experiment_identifiers()
id2 = table2.experiment_identifiers()
for i in id1.keys():
assert id1[i] == id2[i]
other_table = flex.reflection_table()
other_table["id"] = flex.int([3, 4])
table.assert_experiment_identifiers_are_consistent()
packed = table.as_msgpack()
table2 = table.from_msgpack(packed)
id1 = table.experiment_identifiers()
id2 = table2.experiment_identifiers()
for i in id1.keys():
assert id1[i] == id2[i]
other_table = flex.reflection_table()
other_table["id"] = flex.int([3, 4])
table.assert_experiment_identifiers_are_consistent()
identifiers = other_table.experiment_identifiers()
identifiers[3] = "mnop"
identifiers[4] = "qrst"
table.extend(other_table)
assert len(table.experiment_identifiers()) == 5
assert table.experiment_identifiers()[0] == "abcd"
assert table.experiment_identifiers()[1] == "efgh"
assert table.experiment_identifiers()[2] == "ijkl"
assert table.experiment_identifiers()[3] == "mnop"
assert table.experiment_identifiers()[4] == "qrst"
assert len(table.experiment_identifiers()) == 5
assert table.experiment_identifiers()[0] == "abcd"
assert table.experiment_identifiers()[1] == "efgh"
assert table.experiment_identifiers()[2] == "ijkl"
assert table.experiment_identifiers()[3] == "mnop"
assert table.experiment_identifiers()[4] == "qrst"
def discover_better_experimental_model(imagesets,
spot_lists,
params,
dps_params,
nproc=1,
wide_search_binning=1):
assert len(imagesets) == len(spot_lists)
assert len(imagesets) > 0
# XXX should check that all the detector and beam objects are the same
spot_lists_mm = []
max_cell_list = []
detector = imagesets[0].get_detector()
beam = imagesets[0].get_beam()
beam_panel = detector.get_panel_intersection(beam.get_s0())
if beam_panel == -1:
raise Sorry("input beam does not intersect detector")
for imageset, spots in zip(imagesets, spot_lists):
spots_mm = copy.deepcopy(spots)
spots_mm["imageset_id"] = flex.int(len(spots), 0)
expts = ExperimentList([
Experiment(
detector=imageset.get_detector(),
beam=imageset.get_beam(),
goniometer=imageset.get_goniometer(),
scan=imageset.get_scan(),
)
])
spots_mm.centroid_px_to_mm(expts)
spots_mm.map_centroids_to_reciprocal_space(expts)
if dps_params.d_min is not None:
d_spacings = 1 / spots_mm["rlp"].norms()
sel = d_spacings > dps_params.d_min
spots_mm = spots_mm.select(sel)
# derive a max_cell from mm spots
if params.max_cell is None:
max_cell = find_max_cell(spots_mm,
max_cell_multiplier=1.3,
step_size=45).max_cell
max_cell_list.append(max_cell)
if (params.max_reflections is not None
and spots_mm.size() > params.max_reflections):
logger.info("Selecting subset of %i reflections for analysis" %
params.max_reflections)
perm = flex.random_permutation(spots_mm.size())
sel = perm[:params.max_reflections]
spots_mm = spots_mm.select(sel)
spot_lists_mm.append(spots_mm)
if params.max_cell is None:
max_cell = flex.median(flex.double(max_cell_list))
else:
max_cell = params.max_cell
args = [(imageset, spots, max_cell, dps_params)
for imageset, spots in zip(imagesets, spot_lists_mm)]
results = easy_mp.parallel_map(
func=run_dps,
iterable=args,
processes=nproc,
method="multiprocessing",
preserve_order=True,
asynchronous=True,
preserve_exception_message=True,
)
solution_lists = [r["solutions"] for r in results]
amax_list = [r["amax"] for r in results]
assert len(solution_lists) > 0
detector = imagesets[0].get_detector()
beam = imagesets[0].get_beam()
# perform calculation
if dps_params.indexing.improve_local_scope == "origin_offset":
discoverer = better_experimental_model_discovery(
imagesets,
spot_lists_mm,
solution_lists,
amax_list,
dps_params,
wide_search_binning=wide_search_binning,
)
new_detector = discoverer.optimize_origin_offset_local_scope()
old_panel, old_beam_centre = detector.get_ray_intersection(
beam.get_s0())
new_panel, new_beam_centre = new_detector.get_ray_intersection(
beam.get_s0())
old_beam_centre_px = detector[old_panel].millimeter_to_pixel(
old_beam_centre)
new_beam_centre_px = new_detector[new_panel].millimeter_to_pixel(
new_beam_centre)
logger.info("Old beam centre: %.2f, %.2f mm" % old_beam_centre +
" (%.1f, %.1f px)" % old_beam_centre_px)
logger.info("New beam centre: %.2f, %.2f mm" % new_beam_centre +
" (%.1f, %.1f px)" % new_beam_centre_px)
logger.info(
"Shift: %.2f, %.2f mm" %
(matrix.col(old_beam_centre) - matrix.col(new_beam_centre)).elems +
" (%.1f, %.1f px)" % (matrix.col(old_beam_centre_px) -
matrix.col(new_beam_centre_px)).elems)
return new_detector, beam
elif dps_params.indexing.improve_local_scope == "S0_vector":
raise NotImplementedError()
def merge_data_to_mtz(params, experiments, reflections):
"""Merge data (at each wavelength) and write to an mtz file object."""
wavelengths = match_wavelengths(
experiments,
absolute_tolerance=params.wavelength_tolerance,
) # wavelengths is an ordered dict
mtz_datasets = [
MTZDataClass(wavelength=w, project_name=params.output.project_name)
for w in wavelengths.keys()
]
dataset_names = params.output.dataset_names
crystal_names = params.output.crystal_names
# check if best_unit_cell is set.
best_unit_cell = params.best_unit_cell
if not best_unit_cell:
best_unit_cell = determine_best_unit_cell(experiments)
reflections[0]["d"] = best_unit_cell.d(reflections[0]["miller_index"])
for expt in experiments:
expt.crystal.unit_cell = best_unit_cell
if len(wavelengths) > 1:
logger.info(
"Multiple wavelengths found: \n%s",
"\n".join(" Wavlength: %.5f, experiment numbers: %s " %
(k, ",".join(map(str, v)))
for k, v in wavelengths.items()),
)
if not dataset_names or len(dataset_names) != len(wavelengths):
logger.info(
"Unequal number of dataset names and wavelengths, using default naming."
)
dataset_names = [None] * len(wavelengths)
if not crystal_names or len(crystal_names) != len(wavelengths):
logger.info(
"Unequal number of crystal names and wavelengths, using default naming."
)
crystal_names = [None] * len(wavelengths)
experiments_subsets = []
reflections_subsets = []
for dataset, dname, cname in zip(mtz_datasets, dataset_names,
crystal_names):
dataset.dataset_name = dname
dataset.crystal_name = cname
for exp_nos in wavelengths.values():
expids = [experiments[i].identifier for i in exp_nos]
experiments_subsets.append(
ExperimentList([experiments[i] for i in exp_nos]))
reflections_subsets.append(
reflections[0].select_on_experiment_identifiers(expids))
else:
mtz_datasets[0].dataset_name = dataset_names[0]
mtz_datasets[0].crystal_name = crystal_names[0]
experiments_subsets = [experiments]
reflections_subsets = reflections
# merge and truncate the data for each wavelength group
for experimentlist, reflection_table, mtz_dataset in zip(
experiments_subsets, reflections_subsets, mtz_datasets):
# First generate two merge_equivalents objects, collect merging stats
merged, merged_anomalous, stats_summary = merge(
experimentlist,
reflection_table,
d_min=params.d_min,
d_max=params.d_max,
combine_partials=params.combine_partials,
partiality_threshold=params.partiality_threshold,
best_unit_cell=best_unit_cell,
anomalous=params.anomalous,
assess_space_group=params.assess_space_group,
n_bins=params.merging.n_bins,
use_internal_variance=params.merging.use_internal_variance,
)
merged_array = merged.array()
# Save the relevant data in the mtz_dataset dataclass
# This will add the data for IMEAN/SIGIMEAN
mtz_dataset.merged_array = merged_array
if merged_anomalous:
merged_anomalous_array = merged_anomalous.array()
# This will add the data for I(+), I(-), SIGI(+), SIGI(-), N(+), N(-)
mtz_dataset.merged_anomalous_array = merged_anomalous_array
mtz_dataset.multiplicities = merged_anomalous.redundancies()
else:
merged_anomalous_array = None
# This will add the data for N
mtz_dataset.multiplicities = merged.redundancies()
if params.anomalous:
merged_intensities = merged_anomalous_array
else:
merged_intensities = merged_array
anom_amplitudes = None
if params.truncate:
amplitudes, anom_amplitudes, dano = truncate(merged_intensities)
# This will add the data for F, SIGF
mtz_dataset.amplitudes = amplitudes
# This will add the data for F(+), F(-), SIGF(+), SIGF(-)
mtz_dataset.anomalous_amplitudes = anom_amplitudes
# This will add the data for DANO, SIGDANO
mtz_dataset.dano = dano
# print out analysis statistics
show_wilson_scaling_analysis(merged_intensities)
if stats_summary:
logger.info(stats_summary)
if anom_amplitudes:
logger.info(make_dano_table(anom_amplitudes))
# pass the dataclasses to an MTZ writer to generate the mtz file and return.
return make_merged_mtz_file(mtz_datasets)
parser.add_argument('in_refl', help='Input reflection file.')
parser.add_argument(
'output',
help=
'Template for output file. {output}.expt, {output}.refl will be produced.')
args = parser.parse_args()
# Set parameters
expt_filename = args.in_expt
refl_filename = args.in_refl
new_expt_filename = args.output + '.expt'
new_refl_filename = args.output + '.refl'
# Get experiments
expts = ExperimentListFactory.from_json_file(expt_filename)
new_expts = ExperimentList()
# Initialize flex tables for refl files
refl_input = flex.reflection_table().from_file(refl_filename)
refl_output = refl_input.copy()
refl_output["id"] = flex.int([-1] * len(refl_output))
# Initialize data frame
dials_df = rs.DataSet({
'Wavelength': refl_input['Wavelength'],
'ID': refl_input['id'],
'new_ID': [-1] * len(refl_input)
}) #.infer_mtz_dtypes()
# Generate beams per reflection
print(f'Number of rows: {len(dials_df)}')
def run_indexing(
reflections,
experiment,
working_directory,
extra_args,
expected_unit_cell,
expected_rmsds,
expected_hall_symbol,
n_expected_lattices=1,
relative_length_tolerance=0.005,
absolute_angle_tolerance=0.5,
):
commands = ["dials.index"]
if isinstance(reflections, list):
commands.extend(reflections)
else:
commands.append(reflections)
if isinstance(experiment, list):
commands.extend(experiment)
else:
commands.append(experiment)
commands.extend(extra_args)
result = procrunner.run(commands, working_directory=working_directory)
assert not result.returncode and not result.stderr
out_expts = working_directory.join("indexed.expt")
out_refls = working_directory.join("indexed.refl")
assert out_expts.check()
assert out_refls.check()
experiments_list = load.experiment_list(out_expts.strpath,
check_format=False)
assert len(experiments_list.crystals()) == n_expected_lattices
indexed_reflections = flex.reflection_table.from_file(out_refls.strpath)
indexed_reflections.assert_experiment_identifiers_are_consistent(
experiments_list)
rmsds = None
for i, experiment in enumerate(experiments_list):
assert unit_cells_are_similar(
experiment.crystal.get_unit_cell(),
expected_unit_cell,
relative_length_tolerance=relative_length_tolerance,
absolute_angle_tolerance=absolute_angle_tolerance,
), (
experiment.crystal.get_unit_cell().parameters(),
expected_unit_cell.parameters(),
)
sg = experiment.crystal.get_space_group()
assert sg.type().hall_symbol() == expected_hall_symbol, (
sg.type().hall_symbol(),
expected_hall_symbol,
)
reflections = indexed_reflections.select(
indexed_reflections["id"] == i)
mi = reflections["miller_index"]
assert (mi != (0, 0, 0)).count(False) == 0
reflections = reflections.select(mi != (0, 0, 0))
reflections = reflections.select(
reflections.get_flags(reflections.flags.used_in_refinement))
assert len(reflections) > 0
obs_x, obs_y, obs_z = reflections["xyzobs.mm.value"].parts()
calc_x, calc_y, calc_z = reflections["xyzcal.mm"].parts()
rmsd_x = flex.mean(flex.pow2(obs_x - calc_x))**0.5
rmsd_y = flex.mean(flex.pow2(obs_y - calc_y))**0.5
rmsd_z = flex.mean(flex.pow2(obs_z - calc_z))**0.5
rmsds = (rmsd_x, rmsd_y, rmsd_z)
for actual, expected in zip(rmsds, expected_rmsds):
assert actual <= expected, "%s %s" % (rmsds, expected_rmsds)
assert experiment.identifier != ""
expt = ExperimentList()
expt.append(experiment)
reflections.assert_experiment_identifiers_are_consistent(expt)
return _indexing_result(indexed_reflections, experiments_list, rmsds)
class DataManager(object):
def __init__(self, experiments, reflections):
self._input_experiments = experiments
self._input_reflections = reflections
self._experiments = copy.deepcopy(experiments)
self._reflections = copy.deepcopy(reflections)
self._set_batches()
def _set_batches(self):
max_batches = max(e.scan.get_image_range()[1]
for e in self._experiments)
max_batches += 10 # allow some head room
n = int(math.ceil(math.log10(max_batches)))
for i, expt in enumerate(self._experiments):
expt.scan.set_batch_offset(i * 10**n)
logger.debug(
"%s %s" %
(expt.scan.get_batch_offset(), expt.scan.get_batch_range()))
@property
def experiments(self):
return self._experiments
@experiments.setter
def experiments(self, experiments):
self._experiments = experiments
@property
def reflections(self):
return self._reflections
@reflections.setter
def reflections(self, reflections):
self._reflections = reflections
def select(self, experiment_identifiers):
self._experiments = ExperimentList([
expt for expt in self._experiments
if expt.identifier in experiment_identifiers
])
experiment_identifiers = self._experiments.identifiers()
sel = flex.bool(len(self._reflections), False)
for i_expt, identifier in enumerate(experiment_identifiers):
sel_expt = self._reflections['identifier'] == identifier
sel.set_selected(sel_expt, True)
self._reflections['id'].set_selected(sel_expt, i_expt)
self._reflections = self._reflections.select(sel)
assert self.reflections.are_experiment_identifiers_consistent(
self._experiments)
def reflections_as_miller_arrays(self,
intensity_key='intensity.sum.value'):
from cctbx import crystal, miller
variance_key = intensity_key.replace('.value', '.variance')
assert intensity_key in self._reflections
assert variance_key in self._reflections
miller_arrays = []
for expt in self._experiments:
crystal_symmetry = crystal.symmetry(
unit_cell=expt.crystal.get_unit_cell(),
space_group=expt.crystal.get_space_group())
sel = ((self._reflections.get_flags(
self._reflections.flags.integrated_sum)
& (self._reflections['identifier'] == expt.identifier)))
assert sel.count(True) > 0
refl = self._reflections.select(sel)
data = refl[intensity_key]
variances = refl[variance_key]
# FIXME probably need to do some filtering of intensities similar to that
# done in export_mtz
miller_indices = refl['miller_index']
assert variances.all_gt(0)
sigmas = flex.sqrt(variances)
miller_set = miller.set(crystal_symmetry,
miller_indices,
anomalous_flag=False)
intensities = miller.array(miller_set, data=data, sigmas=sigmas)
intensities.set_observation_type_xray_intensity()
intensities.set_info(
miller.array_info(source='DIALS', source_type='pickle'))
miller_arrays.append(intensities)
return miller_arrays
def reindex(self, cb_op=None, cb_ops=None, space_group=None):
assert [cb_op, cb_ops].count(None) == 1
if cb_op is not None:
logger.info('Reindexing: %s' % cb_op)
self._reflections['miller_index'] = cb_op.apply(
self._reflections['miller_index'])
for expt in self._experiments:
cryst_reindexed = expt.crystal.change_basis(cb_op)
if space_group is not None:
cryst_reindexed.set_space_group(space_group)
expt.crystal.update(cryst_reindexed)
elif isinstance(cb_ops, dict):
for cb_op, dataset_ids in cb_ops.iteritems():
cb_op = sgtbx.change_of_basis_op(cb_op)
for dataset_id in dataset_ids:
expt = self._experiments[dataset_id]
logger.info('Reindexing experiment %s: %s' %
(expt.identifier, cb_op.as_xyz()))
cryst_reindexed = expt.crystal.change_basis(cb_op)
if space_group is not None:
cryst_reindexed.set_space_group(space_group)
expt.crystal.update(cryst_reindexed)
sel = self._reflections['identifier'] == expt.identifier
self._reflections['miller_index'].set_selected(
sel,
cb_op.apply(
self._reflections['miller_index'].select(sel)))
else:
assert len(cb_ops) == len(self._experiments)
for cb_op, expt in zip(cb_ops, self._experiments):
logger.info('Reindexing experiment %s: %s' %
(expt.identifier, cb_op.as_xyz()))
cryst_reindexed = expt.crystal.change_basis(cb_op)
if space_group is not None:
cryst_reindexed.set_space_group(space_group)
expt.crystal.update(cryst_reindexed)
sel = self._reflections['identifier'] == expt.identifier
self._reflections['miller_index'].set_selected(
sel,
cb_op.apply(self._reflections['miller_index'].select(sel)))
def export_reflections(self, filename):
self._reflections.as_pickle(filename)
def export_experiments(self, filename):
dump.experiment_list(self._experiments, filename)
def export_mtz(self, filename=None, params=None):
if params is None:
params = export_phil_scope.extract()
if filename is not None:
params.mtz.hklout = filename
m = export_mtz(
self._reflections,
self._experiments,
params.mtz.hklout,
include_partials=params.mtz.include_partials,
keep_partials=params.mtz.keep_partials,
scale_partials=params.mtz.scale_partials,
min_isigi=params.mtz.min_isigi,
force_static_model=params.mtz.force_static_model,
filter_ice_rings=params.mtz.filter_ice_rings,
ignore_profile_fitting=params.mtz.ignore_profile_fitting,
apply_scales=params.mtz.apply_scales)
m.show_summary()
b1 = set(b.num() for b in m.batches())
b2 = set(m.get_column('BATCH').extract_values().as_double().iround())
assert len(b2.difference(b1)) == 0
return params.mtz.hklout
def update_all_data(reflections_path=None, experiments_path=None):
dat = InfoData()
if (reflections_path != None):
try:
refl_tabl = flex.reflection_table.from_pickle(reflections_path)
dat.n_strng = refl_tabl.get_flags(
refl_tabl.flags.strong).count(True)
print "dat.n_strng =", dat.n_strng
dat.n_index = refl_tabl.get_flags(
refl_tabl.flags.indexed).count(True)
print "dat.n_index =", dat.n_index
dat.n_refnd = refl_tabl.get_flags(
refl_tabl.flags.used_in_refinement).count(True)
print "dat.n_refnd =", dat.n_refnd
dat.n_integ_sum = refl_tabl.get_flags(
refl_tabl.flags.integrated_sum).count(True)
print "dat.n_integ_sum =", dat.n_integ_sum
dat.n_integ_prf = refl_tabl.get_flags(
refl_tabl.flags.integrated_prf).count(True)
print "dat.n_integ_prf =", dat.n_integ_prf
except:
print "failed to find reflections"
if (experiments_path != None):
print "trying experiments"
try:
experiments = ExperimentListFactory.from_json_file(
experiments_path, check_format=False)
except:
try:
# FIXME here only take the first datablock. What if there are more?
datablock = DataBlockFactory.from_serialized_format(
experiments_path, check_format=False)[0]
# FIXME here only take the first model from each
beam = datablock.unique_beams()[0]
detector = datablock.unique_detectors()[0]
scan = datablock.unique_scans()[0]
# build a pseudo ExperimentList (with empty crystals)
experiments = ExperimentList()
experiments.append(
Experiment(beam=beam, detector=detector, scan=scan))
except ValueError:
print "failed to read json file"
return dat
print "len(experiments)", len(experiments)
# FIXME take just the first experiment. What if there are more?
exp = experiments[0]
# Get crystal data
if exp.crystal is not None:
unit_cell = exp.crystal.get_unit_cell()
dat.a, dat.b, dat.c, dat.alpha, dat.beta, dat.gamma = unit_cell.parameters(
)
exp_crystal = exp.crystal
#print "exp_crystal = ", exp_crystal
b_mat = exp.crystal.get_B()
dat.b11 = b_mat[0]
dat.b12 = b_mat[1]
dat.b13 = b_mat[2]
dat.b21 = b_mat[3]
dat.b22 = b_mat[4]
dat.b23 = b_mat[5]
dat.b31 = b_mat[6]
dat.b32 = b_mat[7]
dat.b33 = b_mat[8]
sg = str(exp.crystal.get_space_group().info())
print "spgr = ", sg
dat.spg_group = sg
from scitbx import matrix
u_mat = matrix.sqr(exp.crystal.get_U())
dat.u11 = b_mat[0]
dat.u12 = b_mat[1]
dat.u13 = b_mat[2]
dat.u21 = b_mat[3]
dat.u22 = b_mat[4]
dat.u23 = b_mat[5]
dat.u31 = b_mat[6]
dat.u32 = b_mat[7]
dat.u33 = b_mat[8]
rot_angs = u_mat.r3_rotation_matrix_as_x_y_z_angles(deg=True)
print "u_mat =", u_mat
print "rot_angs =", rot_angs
dat.r1, dat.r2, dat.r3 = rot_angs
# Get beam data
dat.w_lambda = exp.beam.get_wavelength()
# Get detector data
# assume details for the panel the beam intersects are the same for the whole detector
pnl_beam_intersects, (beam_x, beam_y) = \
exp.detector.get_ray_intersection(exp.beam.get_s0())
pnl = exp.detector[pnl_beam_intersects]
print "beam_x, beam_y =", beam_x, beam_y
dat.xb = beam_x
dat.yb = beam_y
dist = pnl.get_distance()
print "pnl_beam_intersects ", pnl_beam_intersects
print "dist ", dist
dat.dd = dist
dat.img_ran1, dat.img_ran2 = exp.scan.get_image_range()
dat.oscil1, dat.oscil2 = exp.scan.get_oscillation()
# is the next line right? check what dials.show does
dat.e_time = max(exp.scan.get_exposure_times())
#print set(exp.scan.get_exposure_times())
dat.n_pans = len(exp.detector)
dat.x_px_size, dat.y_px_size = pnl.get_pixel_size()
dat.gain = pnl.get_gain()
dat.max_res = exp.detector.get_max_resolution(exp.beam.get_s0())
# manually finding template from experiments_path
try:
with open(experiments_path) as infile:
json_info = json.load(infile)
if (type(json_info) is dict):
print "found Dictionary"
imageset = json_info['imageset']
elif (type(json_info) is list):
print "found List"
imageset = json_info[0]['imageset']
dat.tmpl_str = imageset[0]['template']
print "dat.tmpl_str =", dat.tmpl_str
except:
print "failed to find template in JSON file"
return dat
from IPython import embed
# Load DIALS files
expt_file = "dials_temp_files/mega_ultra_refined.expt"
refl_file = "dials_temp_files/mega_ultra_refined.refl"
# Get data
print('Loading DIALS files.')
elist = ExperimentListFactory.from_json_file(expt_file, check_format=False)
refls = reflection_table.from_file(refl_file)
# Remove outliers
print('Removing outliers')
idx = refls.get_flags(refls.flags.used_in_refinement).as_numpy_array()
idy = np.arange(len(elist))[idx].tolist()
elist = ExperimentList([elist[i] for i in idy])
refls = refls.select(flex.bool(idx))
# Get experiment data from experiment objects
print('Getting experiment data.')
img_num = 0
i = 0
img = elist.imagesets()[img_num]
experiment = elist[0]
while (True): # Get first expt for this image
experiment = elist[i]
if (experiment.imageset == img):
break
i = i + 1
cryst = experiment.crystal
spacegroup = gemmi.SpaceGroup(
def experiments_masks(request, dials_data):
filename = (dials_data(request.param["directory"]) /
request.param["filename"]).strpath
return ExperimentList.from_file(filename), request.param["masks"]
def test_filtered_arrays_from_experiments_reflections():
"""Test the creating of a miller array from crystal and reflection table."""
refl = generate_integrated_test_reflections()
refl["miller_index"] = flex.miller_index([(1, 0, 0), (2, 0, 0), (3, 0, 0),
(4, 0, 0), (5, 0, 0), (6, 0, 0)])
experiments = ExperimentList()
exp_dict = {
"__id__": "crystal",
"real_space_a": [1.0, 0.0, 0.0],
"real_space_b": [0.0, 1.0, 0.0],
"real_space_c": [0.0, 0.0, 2.0],
"space_group_hall_symbol": " C 2y",
}
crystal = Crystal.from_dict(exp_dict)
experiments.append(Experiment(crystal=crystal))
miller_set = filtered_arrays_from_experiments_reflections(
experiments, [refl])[0]
assert isinstance(miller_set, miller.set)
assert list(miller_set.data()) == [4.6, 2.4, 2.5] # same as calling filter
# for export on scale intensity reducer.
# now try for prf
del refl["intensity.scale.value"]
miller_set = filtered_arrays_from_experiments_reflections(
experiments, [refl])[0]
assert isinstance(miller_set, miller.set)
assert list(miller_set.data()) == [1.0, 2.0, 3.0] # same as calling filter
# for export on prf + sum intensity reducer.
# now just for sum
del refl["intensity.prf.value"]
miller_set = filtered_arrays_from_experiments_reflections(
experiments, [refl])[0]
assert isinstance(miller_set, miller.set)
assert list(miller_set.data()) == [11.0, 12.0, 13.0,
14.0] # same as calling
# filter for export on prf intensity reducer.
# Now try with a bad dataset - should be filtered.
refl = generate_integrated_test_reflections()
refl["miller_index"] = flex.miller_index([(1, 0, 0), (2, 0, 0), (3, 0, 0),
(4, 0, 0), (5, 0, 0), (6, 0, 0)])
# Trigger filtering on prf/sum, but when prf is bad - should proceed with sum
refl.unset_flags(flex.bool(6, True), refl.flags.integrated_prf)
del refl["intensity.scale.value"]
refl2 = generate_integrated_test_reflections()
refl2["partiality"] = flex.double(6, 0.0)
experiments = ExperimentList()
experiments.append(Experiment(crystal=crystal))
experiments.append(Experiment(crystal=crystal))
miller_sets = filtered_arrays_from_experiments_reflections(
experiments, [refl, refl2], outlier_rejection_after_filter=True)
assert len(miller_sets) == 1
experiments = ExperimentList()
experiments.append(Experiment(crystal=crystal))
experiments.append(Experiment(crystal=crystal))
refl2 = generate_integrated_test_reflections()
refl2["partiality"] = flex.double(6, 0.0)
with pytest.raises(ValueError):
refl["partiality"] = flex.double(6, 0.0)
_ = filtered_arrays_from_experiments_reflections(
experiments, [refl, refl2])
def test_forward_panel_edge(dials_data):
expt = ExperimentList.from_file(
dials_data("centroid_test_data").join(
"imported_experiments.json").strpath)[0]
# Get the models
beam = expt.beam
detector = expt.detector
gonio = expt.goniometer
scan = expt.scan
# Set some parameters
sigma_divergence = 0.00101229
mosaicity = 0.157 * math.pi / 180
n_sigma = 3
grid_size = 7
delta_divergence = n_sigma * sigma_divergence
step_size = delta_divergence / grid_size
delta_divergence2 = delta_divergence + step_size * 0.5
delta_mosaicity = n_sigma * mosaicity
# Create the bounding box calculator
calculate_bbox = BBoxCalculator3D(beam, detector, gonio, scan,
delta_divergence2, delta_mosaicity)
# Initialise the transform
spec = transform.TransformSpec(beam, detector, gonio, scan,
sigma_divergence, mosaicity, n_sigma + 1,
grid_size)
assert len(detector) == 1
s0 = beam.get_s0()
m2 = gonio.get_rotation_axis()
s0_length = matrix.col(beam.get_s0()).length()
image_size = detector[0].get_image_size()
refl_xy = [
(0, 0),
(2, 3),
(4, 1000),
(1000, 5),
(image_size[0] - 1, image_size[1] - 1),
(image_size[0] - 2, 1),
(1, image_size[1] - 5),
(1000, image_size[1] - 4),
(image_size[0] - 3, 1000),
]
for x, y in refl_xy:
z = random.uniform(0, 9)
# Get random s1, phi, panel
s1 = matrix.col(detector[0].get_pixel_lab_coord(
(x, y))).normalize() * s0_length
phi = scan.get_angle_from_array_index(z, deg=False)
panel = 0
# Calculate the bounding box
bbox = calculate_bbox(s1, z, panel)
x0, x1, y0, y1, z0, z1 = bbox
# Create the coordinate system
cs = CoordinateSystem(m2, s0, s1, phi)
# Create the image
image = gaussian((z1 - z0, y1 - y0, x1 - x0), 10.0,
(z - z0, y - y0, x - x0), (2.0, 2.0, 2.0))
# Mask for the foreground pixels
refl_mask = image > 1e-3
bg = flex.double(image.accessor())
# Shoebox mask, i.e. mask out pixels that are outside the panel bounds
shoebox_mask = flex.bool(image.accessor(), False)
for j in range(y1 - y0):
for i in range(x1 - x0):
if (j + y0 >= 0 and j + y0 < image_size[1] and i + x0 >= 0
and i + x0 < image_size[0]):
for k in range(z1 - z0):
shoebox_mask[k, j, i] = True
mask = refl_mask & shoebox_mask
# from matplotlib import pyplot as plt
# fig, axes = plt.subplots(ncols=refl_mask.focus()[0], nrows=4)
# for i in range(refl_mask.focus()[0]):
# axes[0, i].imshow(image.as_numpy_array()[i])
# axes[1, i].imshow(refl_mask.as_numpy_array()[i])
# axes[2, i].imshow(shoebox_mask.as_numpy_array()[i])
# axes[3, i].imshow(mask.as_numpy_array()[i])
# plt.show()
# Transform the image to the grid
transformed = transform.TransformForward(spec, cs, bbox, 0,
image.as_double(), bg,
refl_mask)
grid = transformed.profile()
mask = refl_mask & shoebox_mask
# assert only pixels within the panel were transformed
assert flex.sum(grid) == pytest.approx(flex.sum(
image.select(mask.as_1d())),
rel=0.01)
# The total transformed counts should be less than the (unmasked) image counts
assert flex.sum(grid) < flex.sum(image)
# Transform the image to the grid, this time without a background
transformed = transform.TransformForward(spec, cs, bbox, 0,
image.as_double(), refl_mask)
grid = transformed.profile()
mask = refl_mask & shoebox_mask
# assert only pixels within the panel were transformed
assert flex.sum(grid) == pytest.approx(flex.sum(
image.select(mask.as_1d())),
rel=0.01)
# The total transformed counts should be less than the (unmasked) image counts
assert flex.sum(grid) < flex.sum(image)
def test_split_by_wavelength(tmpdir):
"""Test the split_by_wavelength option of dials.split_experiments"""
experiments = ExperimentList()
exp = generate_exp(wavelength=1.0)
exp.identifier = "0"
experiments.append(exp)
exp = generate_exp(wavelength=0.5)
exp.identifier = "1"
experiments.append(exp)
reflections = flex.reflection_table()
reflections["id"] = flex.int([0, 1])
reflections["intensity"] = flex.double([100.0, 200.0])
reflections.experiment_identifiers()[0] = "0"
reflections.experiment_identifiers()[1] = "1"
experiments.as_json(tmpdir.join("tmp.expt").strpath)
reflections.as_file(tmpdir.join("tmp.refl").strpath)
result = procrunner.run(
[
"dials.split_experiments", "tmp.expt", "tmp.refl",
"by_wavelength=True"
],
working_directory=tmpdir,
)
assert not result.returncode and not result.stderr
for i, (wl, ids,
intensity) in enumerate(zip([0.5, 1.0], ["1", "0"],
[200.0, 100.0])):
assert tmpdir.join("split_%d.expt" % i).check()
assert tmpdir.join("split_%d.refl" % i).check()
exp_single = load.experiment_list(tmpdir.join("split_%d.expt" %
i).strpath,
check_format=False)
ref_single = flex.reflection_table.from_file(
tmpdir.join("split_%d.refl" % i).strpath)
assert exp_single[0].beam.get_wavelength() == wl
assert exp_single[0].identifier == ids
id_ = ref_single["id"][0]
assert ref_single.experiment_identifiers()[id_] == ids
assert list(ref_single["intensity"]) == [intensity]
# Now test for successful error handling if no identifiers set.
experiments[0].identifier = ""
experiments[1].identifier = ""
experiments.as_json(tmpdir.join("tmp.expt").strpath)
result = procrunner.run(
[
"dials.split_experiments", "tmp.expt", "tmp.refl",
"by_wavelength=True"
],
working_directory=tmpdir,
)
assert result.returncode == 1
assert result.stderr.startswith(b"Sorry")
experiments[0].identifier = "0"
experiments[1].identifier = "1"
del reflections.experiment_identifiers()[0]
del reflections.experiment_identifiers()[1]
experiments.as_json(tmpdir.join("tmp.expt").strpath)
reflections.as_file(tmpdir.join("tmp.refl").strpath)
result = procrunner.run(
[
"dials.split_experiments", "tmp.expt", "tmp.refl",
"by_wavelength=True"
],
working_directory=tmpdir,
)
assert result.returncode == 1
assert result.stderr.startswith(b"Sorry")
def index(self):
experiments = ExperimentList()
had_refinement_error = False
have_similar_crystal_models = False
while True:
if had_refinement_error or have_similar_crystal_models:
break
max_lattices = self.params.multiple_lattice_search.max_lattices
if max_lattices is not None and len(experiments) >= max_lattices:
break
if len(experiments) > 0:
cutoff_fraction = (self.params.multiple_lattice_search.
recycle_unindexed_reflections_cutoff)
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_indexed = flex.min(
d_spacings.select(self.indexed_reflections))
min_reflections_for_indexing = cutoff_fraction * len(
self.reflections.select(d_spacings > d_min_indexed))
crystal_ids = self.reflections.select(
d_spacings > d_min_indexed)["id"]
if (crystal_ids
== -1).count(True) < min_reflections_for_indexing:
logger.info(
"Finish searching for more lattices: %i unindexed reflections remaining."
% ((crystal_ids == -1).count(True)))
break
n_lattices_previous_cycle = len(experiments)
if self.d_min is None:
self.d_min = self.params.refinement_protocol.d_min_start
if len(experiments) == 0:
experiments.extend(self.find_lattices())
else:
try:
new = self.find_lattices()
experiments.extend(new)
except Sorry:
logger.info("Indexing remaining reflections failed")
if self.params.refinement_protocol.d_min_step is libtbx.Auto:
n_cycles = self.params.refinement_protocol.n_macro_cycles
if self.d_min is None or n_cycles == 1:
self.params.refinement_protocol.d_min_step = 0
else:
d_spacings = 1 / self.reflections["rlp"].norms()
d_min_all = flex.min(d_spacings)
self.params.refinement_protocol.d_min_step = (
self.d_min - d_min_all) / (n_cycles - 1)
logger.info("Using d_min_step %.1f" %
self.params.refinement_protocol.d_min_step)
if len(experiments) == 0:
raise DialsIndexError("No suitable lattice could be found.")
elif len(experiments) == n_lattices_previous_cycle:
# no more lattices found
break
for i_cycle in range(
self.params.refinement_protocol.n_macro_cycles):
if (i_cycle > 0 and self.d_min is not None
and self.params.refinement_protocol.d_min_step > 0):
d_min = self.d_min - self.params.refinement_protocol.d_min_step
d_min = max(d_min, 0)
if self.params.refinement_protocol.d_min_final is not None:
d_min = max(
d_min, self.params.refinement_protocol.d_min_final)
if d_min >= 0:
self.d_min = d_min
logger.info("Increasing resolution to %.2f Angstrom" %
d_min)
# reset reflection lattice flags
# the lattice a given reflection belongs to: a value of -1 indicates
# that a reflection doesn't belong to any lattice so far
self.reflections["id"] = flex.int(len(self.reflections), -1)
self.index_reflections(experiments, self.reflections)
if i_cycle == 0 and self.params.known_symmetry.space_group is not None:
self._apply_symmetry_post_indexing(
experiments, self.reflections,
n_lattices_previous_cycle)
logger.info("\nIndexed crystal models:")
self.show_experiments(experiments,
self.reflections,
d_min=self.d_min)
if self._check_have_similar_crystal_models(experiments):
have_similar_crystal_models = True
break
logger.info("")
logger.info("#" * 80)
logger.info("Starting refinement (macro-cycle %i)" %
(i_cycle + 1))
logger.info("#" * 80)
logger.info("")
self.indexed_reflections = self.reflections["id"] > -1
sel = flex.bool(len(self.reflections), False)
lengths = 1 / self.reflections["rlp"].norms()
if self.d_min is not None:
isel = (lengths <= self.d_min).iselection()
sel.set_selected(isel, True)
sel.set_selected(self.reflections["id"] == -1, True)
self.reflections.unset_flags(sel,
self.reflections.flags.indexed)
self.unindexed_reflections = self.reflections.select(sel)
reflections_for_refinement = self.reflections.select(
self.indexed_reflections)
if self.params.refinement_protocol.mode == "repredict_only":
refined_experiments, refined_reflections = (
experiments,
reflections_for_refinement,
)
from dials.algorithms.refinement.prediction.managed_predictors import (
ExperimentsPredictorFactory, )
ref_predictor = ExperimentsPredictorFactory.from_experiments(
experiments,
spherical_relp=self.all_params.refinement.
parameterisation.spherical_relp_model,
)
ref_predictor(refined_reflections)
else:
try:
refined_experiments, refined_reflections = self.refine(
experiments, reflections_for_refinement)
except (DialsRefineConfigError,
DialsRefineRuntimeError) as e:
if len(experiments) == 1:
raise DialsIndexRefineError(e.message)
had_refinement_error = True
logger.info("Refinement failed:")
logger.info(e)
del experiments[-1]
break
self._unit_cell_volume_sanity_check(experiments,
refined_experiments)
self.refined_reflections = refined_reflections
self.refined_reflections.unset_flags(
self.refined_reflections["id"] < 0,
self.refined_reflections.flags.indexed,
)
for i, expt in enumerate(self.experiments):
ref_sel = self.refined_reflections.select(
self.refined_reflections["imageset_id"] == i)
ref_sel = ref_sel.select(ref_sel["id"] >= 0)
for i_expt in set(ref_sel["id"]):
refined_expt = refined_experiments[i_expt]
expt.detector = refined_expt.detector
expt.beam = refined_expt.beam
expt.goniometer = refined_expt.goniometer
expt.scan = refined_expt.scan
refined_expt.imageset = expt.imageset
if not (self.all_params.refinement.parameterisation.beam.fix
== "all" and self.all_params.refinement.
parameterisation.detector.fix == "all"):
# Experimental geometry may have changed - re-map centroids to
# reciprocal space
self.reflections = self._map_centroids_to_reciprocal_space(
self.experiments, self.reflections)
# update for next cycle
experiments = refined_experiments
self.refined_experiments = refined_experiments
logger.info("\nRefined crystal models:")
self.show_experiments(self.refined_experiments,
self.reflections,
d_min=self.d_min)
if (i_cycle >= 2 and self.d_min
== self.params.refinement_protocol.d_min_final):
logger.info(
"Target d_min_final reached: finished with refinement")
break
if self.refined_experiments is None:
raise DialsIndexRefineError(
"None of the experiments could refine.")
if len(self.refined_experiments) > 1:
from dials.algorithms.indexing.compare_orientation_matrices import (
rotation_matrix_differences, )
logger.info(
rotation_matrix_differences(
self.refined_experiments.crystals()))
self._xyzcal_mm_to_px(self.experiments, self.refined_reflections)
def _index(self):
"""Actually do the autoindexing using the data prepared by the
previous method."""
idxref = self.Idxref()
self._index_remove_masked_regions()
for file in ["SPOT.XDS"]:
idxref.set_input_data_file(file, self._indxr_payload[file])
# edit SPOT.XDS to remove reflections in untrusted regions of the detector
idxref.set_data_range(self._indxr_images[0][0],
self._indxr_images[0][1])
idxref.set_background_range(self._indxr_images[0][0],
self._indxr_images[0][1])
# set the phi start etc correctly
for block in self._indxr_images[:1]:
starting_frame = block[0]
starting_angle = self.get_scan().get_angle_from_image_index(
starting_frame)
idxref.set_starting_frame(starting_frame)
idxref.set_starting_angle(starting_angle)
idxref.add_spot_range(block[0], block[1])
for block in self._indxr_images[1:]:
idxref.add_spot_range(block[0], block[1])
if self._indxr_user_input_lattice:
idxref.set_indexer_user_input_lattice(True)
if self._indxr_input_lattice and self._indxr_input_cell:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
idxref.set_indexer_input_cell(self._indxr_input_cell)
logger.debug("Set lattice: %s", self._indxr_input_lattice)
logger.debug("Set cell: %f %f %f %f %f %f" %
self._indxr_input_cell)
original_cell = self._indxr_input_cell
elif self._indxr_input_lattice:
idxref.set_indexer_input_lattice(self._indxr_input_lattice)
original_cell = None
else:
original_cell = None
converter = to_xds(self.get_imageset())
xds_beam_centre = converter.detector_origin
idxref.set_beam_centre(xds_beam_centre[0], xds_beam_centre[1])
# fixme need to check if the lattice, cell have been set already,
# and if they have, pass these in as input to the indexing job.
done = False
while not done:
try:
done = idxref.run()
# N.B. in here if the IDXREF step was being run in the first
# pass done is FALSE however there should be a refined
# P1 orientation matrix etc. available - so keep it!
except XDSException as e:
# inspect this - if we have complaints about not
# enough reflections indexed, and we have a target
# unit cell, and they are the same, well ignore it
if "solution is inaccurate" in str(e):
logger.debug(
"XDS complains solution inaccurate - ignoring")
done = idxref.continue_from_error()
elif ("insufficient percentage (< 70%)" in str(e)
or "insufficient percentage (< 50%)"
in str(e)) and original_cell:
done = idxref.continue_from_error()
lattice, cell, mosaic = idxref.get_indexing_solution()
# compare solutions FIXME should use xds_cell_deviation
check = PhilIndex.params.xia2.settings.xds_check_cell_deviation
for j in range(3):
# allow two percent variation in unit cell length
if (math.fabs(
(cell[j] - original_cell[j]) / original_cell[j]) >
0.02 and check):
logger.debug("XDS unhappy and solution wrong")
raise e
# and two degree difference in angle
if (math.fabs(cell[j + 3] - original_cell[j + 3]) > 2.0
and check):
logger.debug("XDS unhappy and solution wrong")
raise e
logger.debug("XDS unhappy but solution ok")
elif "insufficient percentage (< 70%)" in str(
e) or "insufficient percentage (< 50%)" in str(e):
logger.debug("XDS unhappy but solution probably ok")
done = idxref.continue_from_error()
else:
raise e
FileHandler.record_log_file(
"%s INDEX" % self.get_indexer_full_name(),
os.path.join(self.get_working_directory(), "IDXREF.LP"),
)
for file in ["SPOT.XDS", "XPARM.XDS"]:
self._indxr_payload[file] = idxref.get_output_data_file(file)
# need to get the indexing solutions out somehow...
self._indxr_other_lattice_cell = idxref.get_indexing_solutions()
(
self._indxr_lattice,
self._indxr_cell,
self._indxr_mosaic,
) = idxref.get_indexing_solution()
xparm_file = os.path.join(self.get_working_directory(), "XPARM.XDS")
models = dxtbx.load(xparm_file)
crystal_model = to_crystal(xparm_file)
# this information gets lost when re-creating the models from the
# XDS results - however is not refined so can simply copy from the
# input - https://github.com/xia2/xia2/issues/372
models.get_detector()[0].set_thickness(
converter.get_detector()[0].get_thickness())
experiment = Experiment(
beam=models.get_beam(),
detector=models.get_detector(),
goniometer=models.get_goniometer(),
scan=models.get_scan(),
crystal=crystal_model,
# imageset=self.get_imageset(),
)
experiment_list = ExperimentList([experiment])
self.set_indexer_experiment_list(experiment_list)
# I will want this later on to check that the lattice was ok
self._idxref_subtree_problem = idxref.get_index_tree_problem()
def test_change_of_basis_ops_to_minimum_cell_1037(mocker):
# See https://github.com/dials/dials/issues/1037
input_ucs = [
(
4.805202948916906,
12.808064769657364,
16.544899201125446,
106.45808502003258,
90.0065567098825,
100.77735674275475,
),
(
4.808011343212577,
12.821894835790472,
16.557339561965573,
106.48431244651402,
90.0252848479048,
100.77252933676507,
),
(
4.8096632137789985,
12.815648858527567,
16.55931712239122,
106.48990701341536,
90.01703141314147,
100.80397887485773,
),
(
4.807294085194974,
12.822386757910516,
16.560411742466663,
106.43185845358086,
90.02067929544215,
100.79522302759383,
),
]
# Setup the input experiments and reflection tables
expts = ExperimentList()
for uc in input_ucs:
uc = uctbx.unit_cell(uc)
sg = sgtbx.space_group_info("P1").group()
B = scitbx.matrix.sqr(uc.fractionalization_matrix()).transpose()
expts.append(Experiment(crystal=Crystal(B, space_group=sg, reciprocal=True)))
# We want to spy on the return value of this function
mocker.spy(symmetry, "unit_cells_are_similar_to")
# Actually run the method we are testing
cb_ops = change_of_basis_ops_to_minimum_cell(
expts, max_delta=5, relative_length_tolerance=0.05, absolute_angle_tolerance=2
)
import pytest_mock
if getattr(pytest_mock, "version", "").startswith("1."):
assert symmetry.unit_cells_are_similar_to.return_value is True
else:
assert symmetry.unit_cells_are_similar_to.spy_return is True
cb_ops_as_xyz = [cb_op.as_xyz() for cb_op in cb_ops]
assert len(set(cb_ops_as_xyz)) == 1
# Actual cb_ops are machine dependent (sigh)
assert cb_ops_as_xyz[0] in ("x,y,z", "-x,y,-z", "x-y,-y,-z")
def test_blank_integrated_analysis(dials_data):
expts = ExperimentList.from_file(dials_data("insulin_processed") /
"integrated.expt",
check_format=False)
refl = flex.reflection_table.from_file(
dials_data("insulin_processed") / "integrated.refl")
results = detect_blanks.blank_integrated_analysis(refl,
expts[0].scan,
phi_step=5,
fractional_loss=0.1)
assert results["data"][0]["x"] == [
2.5,
7.5,
12.5,
17.5,
22.5,
27.5,
32.5,
37.5,
42.5,
]
assert results["data"][0]["y"] == pytest.approx([
27.903266149430973,
25.832527090455052,
26.9236206883069,
26.50234804728626,
26.41019377727383,
25.810676090828185,
24.844906790823064,
25.89992001081651,
25.580718362291474,
])
assert not any(results["data"][0]["blank"])
assert results["blank_regions"] == []
# Now with some "blank" regions - make some of the reflections weak
z = refl["xyzobs.px.value"].parts()[2]
refl["intensity.prf.value"].set_selected(
z < 10, refl["intensity.prf.value"] * 0.05)
results = detect_blanks.blank_integrated_analysis(refl,
expts[0].scan,
phi_step=5,
fractional_loss=0.1)
assert results["data"][0]["y"] == pytest.approx([
1.3951633074715482,
1.2916263545227527,
26.9236206883069,
26.50234804728626,
26.41019377727383,
25.810676090828185,
24.844906790823064,
25.89992001081651,
25.580718362291474,
])
assert results["data"][0]["blank"] == [
True,
True,
False,
False,
False,
False,
False,
False,
False,
]
assert results["blank_regions"] == [(0, 10)]
# Unset the integrated_prf flags, so the analysis should instead use the umodified
# intensity.sum.value instead
refl.unset_flags(flex.bool(len(refl), True), refl.flags.integrated_prf)
results = detect_blanks.blank_integrated_analysis(refl,
expts[0].scan,
phi_step=5,
fractional_loss=0.1)
assert not any(results["data"][0]["blank"])
assert results["blank_regions"] == []
