def update(experiments: ExperimentList,
new_params: libtbx.phil.scope_extract) -> ExperimentList:
"""
Modify detector, beam, goniometer and scan in experiments with the values in new_params
"""
update_geometry = ManualGeometryUpdater(new_params)
imagesets = experiments.imagesets()
for imageset in imagesets:
imageset_new = update_geometry(imageset)
imageset.set_detector(imageset_new.get_detector())
imageset.set_beam(imageset_new.get_beam())
imageset.set_goniometer(imageset_new.get_goniometer())
imageset.set_scan(imageset_new.get_scan())
return experiments
def do_boilerplate(
experiments: ExperimentList,
reflections: flex.reflection_table,
params: libtbx.phil.scope_extract,
) -> (ExperimentList, flex.reflection_table):
"""
Write the behaviour of the program as functions and classes outside run().
Don't include file output here, remember that this function may be re-used
elsewhere by someone who doesn't need the output written immediately to file.
It can be especially helpful to document any expected exceptions that might be
raised, in order to keep track of what needs to be handled in any code that
re-uses this function.
Args:
experiments: An experiment list.
reflections: A reflection table.
params: Some parameters, in the form of a scope_extract object,
which is the usable form of a parsed PHIL scope.
Raises:
RuntimeError: if someone says 'goose'.
"""
logger.info("Hello world!")
# Here's an example of an error that might be raised, as documented above.
if "goose" in ["duck", "duck", "duck"]:
raise RuntimeError("Quick, run!")
logger.info("The input reflection table contains %d reflections.",
len(reflections))
logger.info("The input experiment list contains %d imagesets.",
len(experiments.imagesets()))
logger.info("integer_parameter: %i", params.integer_parameter)
logger.info("bool_parameter: %s", params.bool_parameter)
return experiments, reflections
class Screen19(object):
"""Encapsulates the screening script."""
def __init__(self):
# Throughout the pipeline, retain the state of the processing.
self.expts = ExperimentList([])
self.refls = flex.reflection_table()
# Get some default parameters. These must be extracted from the 'fetched'
# PHIL scope, rather than the 'definition' phil scope returned by
# iotbx.phil.parse. Confused? Blame PHIL.
self.params = phil_scope.fetch(iotbx.phil.parse("")).extract()
def _quick_import(self, files): # type: (List[str]) -> bool
"""
Generate xia2-style templates from file names and attempt a quick import.
From each given filename, generate a filename template by substituting a hash
character (#) for each numeral in the last contiguous group of numerals
before the file extension. For example, the filename `example_01_0001.cbf`
becomes `example_01_####.cbf`.
Contiguous image ranges are recorded by associating the start and end image
number of the range with the relevant filename template.
dials.import is then run with options to extrapolate header information from
the first image file, thereby running more quickly than reading each image
header individually.
Args:
files: List of image filenames.
Returns:
Boolean flag indicating whether the quick import has succeeded.
"""
if len(files) == 1:
# No point in quick-importing a single file
return False
debug("Attempting quick import...")
files.sort()
templates = {} # type: Dict[str, List[Optional[List[int]]]]
for f in files:
template, image = screen19.make_template(f)
if template not in templates:
image_range = [image, image] if image else []
templates.update({template: [image_range]})
elif image == templates[template][-1][-1] + 1:
templates[template][-1][-1] = image
elif image == templates[template][-1][-1]:
# We have a duplicate input file name. Do nothing.
pass
else:
templates[template].append([image, image])
# Return tuple of template and image range for each unique image range
templates = [(t, tuple(r)) for t, ranges in templates.items()
for r in ranges] # type: Templates
return self._quick_import_templates(templates)
def _quick_import_templates(self, templates): # type: (Templates) -> bool
"""
Take image file templates and frame number ranges and try to run dials.import.
dials.import is run with options to extrapolate header information from
the first image file, thereby running more quickly than reading each image
header individually.
Args:
templates: A list of tuples, each tuple containing a xia2-style filename
template and the start and end image numbers of the associated
sweep.
Returns:
Boolean flag indicating whether the quick import has succeeded.
"""
debug("Quick import template summary:\n\t%s", templates)
if len(templates) > 1:
debug("Cannot currently run quick import on multiple templates.")
return False
try:
scan_range = templates[0][1] # type: Tuple[int, int]
if not scan_range:
raise IndexError
except IndexError:
debug(
"Cannot run quick import: could not determine image naming template."
)
return False
info("Running quick import.")
self.params.dials_import.input.template = [templates[0][0]]
self.params.dials_import.geometry.scan.image_range = scan_range
self.params.dials_import.geometry.scan.extrapolate_scan = True
self._run_dials_import()
return True
def _import(self, files): # type: (List[str]) -> None
"""
Try to run a quick call of dials.import. Failing that, run a slow call.
Try initially to construct file name templates contiguous groups of files.
Failing that, pass a full list of the files to the importer (slower).
Args:
files: List of image filenames.
"""
info("\nImporting data...")
if len(files) == 1:
if os.path.isdir(files[0]):
debug("You specified a directory. Importing all CBF files in "
"that directory.")
# TODO Support HDF5.
files = [
os.path.join(files[0], f) for f in os.listdir(files[0])
if f.endswith(".cbf") or f.endswith(".cbf.gz")
or f.endswith(".cbf.bz2")
]
elif len(files[0].split(":")) == 3:
debug("You specified an image range in the xia2 format. "
"Importing all specified files.")
template, start, end = files[0].split(":")
template = screen19.make_template(template)[0]
start, end = int(start), int(end)
if not self._quick_import_templates([(template,
(start, end))]):
warning("Could not import specified image range.")
sys.exit(1)
info("Quick import successful.")
return
elif files[0].endswith(".expt"):
debug("You specified an existing experiment list file. "
"No import necessary.")
try:
self.expts = ExperimentList.from_file(files[0])
except (IOError, PickleError, ValueError):
pass
else:
self.params.dials_import.output.experiments = files[0]
if self.expts:
return
if not files:
warning("No images found matching input.")
sys.exit(1)
# Can the files be quick-imported?
if self._quick_import(files):
info("Quick import successful.")
return
self.params.dials_import.input.experiments = files
self._run_dials_import()
def _run_dials_import(self):
"""
Perform a minimal version of dials.import to get an experiment list.
Use some filleted bits of dials.import and dials.util.options.Importer.
"""
# Get some key data format arguments.
try:
format_kwargs = {
"dynamic_shadowing":
self.params.dials_import.format.dynamic_shadowing,
"multi_panel": self.params.dials_import.format.multi_panel,
}
except AttributeError:
format_kwargs = {}
# If filenames contain wildcards, expand
args = []
for arg in self.params.dials_import.input.experiments:
if "*" in arg:
args.extend(glob(arg))
else:
args.append(arg)
if args:
# Are compare{beam,detector,goniometer} and scan_tolerance necessary?
# They are cargo-culted from the DIALS option parser.
tol_params = self.params.dials_import.input.tolerance
compare_beam = BeamComparison(
wavelength_tolerance=tol_params.beam.wavelength,
direction_tolerance=tol_params.beam.direction,
polarization_normal_tolerance=tol_params.beam.
polarization_normal,
polarization_fraction_tolerance=tol_params.beam.
polarization_fraction,
)
compare_detector = DetectorComparison(
fast_axis_tolerance=tol_params.detector.fast_axis,
slow_axis_tolerance=tol_params.detector.slow_axis,
origin_tolerance=tol_params.detector.origin,
)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=tol_params.goniometer.rotation_axis,
fixed_rotation_tolerance=tol_params.goniometer.fixed_rotation,
setting_rotation_tolerance=tol_params.goniometer.
setting_rotation,
)
scan_tolerance = tol_params.scan.oscillation
# Import an experiment list from image data.
try:
experiments = ExperimentListFactory.from_filenames(
args,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
)
except IOError as e:
warning("%s '%s'", e.strerror, e.filename)
sys.exit(1)
# Record the imported experiments for use elsewhere.
# Quit if there aren't any.
self.expts.extend(experiments)
if not self.expts:
warning("No images found.")
sys.exit(1)
else:
# Use the template importer.
if len(self.params.dials_import.input.template) > 0:
importer = ExperimentListTemplateImporter(
self.params.dials_import.input.template,
format_kwargs=format_kwargs)
# Record the imported experiments for use elsewhere.
# Quit if there aren't any.
self.expts.extend(importer.experiments)
if not self.expts:
warning("No images found matching template %s" %
self.params.dials_import.input.template[0])
sys.exit(1)
# Setup the metadata updater
metadata_updater = MetaDataUpdater(self.params.dials_import)
# Extract the experiments and loop through
self.expts = metadata_updater(self.expts.imagesets())
def _count_processors(self, nproc=None): # type: (Optional[int]) -> None
"""
Determine the number of processors and save it as an instance variable.
The user may specify the number of processors to use. If no value is
given, the number of available processors is returned.
Args:
nproc (optional): Number of processors.
"""
if nproc and nproc is not Auto:
self.nproc = nproc
return
# if environmental variable NSLOTS is set to a number then use that
try:
self.nproc = int(os.environ.get("NSLOTS"))
return
except (ValueError, TypeError):
pass
self.nproc = number_of_processors(return_value_if_unknown=-1)
if self.nproc <= 0:
warning(
"Could not determine number of available processors. Error code %d",
self.nproc,
)
sys.exit(1)
def _count_images(self): # type: () -> int
"""
Attempt to determine the number of diffraction images.
The number of diffraction images is determined from the imported_experiments
JSON file.
Returns:
Number of images.
"""
# FIXME: This exception handling should be redundant. Empty experiment
# lists should get caught at the import stage. Is this so?
try:
return self.expts[0].imageset.size()
except IndexError:
warning("Could not determine number of images in dataset.")
sys.exit(1)
def _check_intensities(self,
mosaicity_correction=True): # type: (bool) -> None
"""
Run xia2.overload and plot a histogram of pixel intensities.
If `mosaicity_correction` is true, the pixel intensities are approximately
adjusted to take account of a systematic defect in the detector count rate
correction. See https://github.com/xia2/screen19/wiki#mosaicity-correction
Args:
mosaicity_correction (optional): default is `True`.
"""
info("\nTesting pixel intensities...")
command = ["xia2.overload", "nproc=%s" % self.nproc, "indexed.expt"]
debug("running %s", command)
result = procrunner.run(command,
print_stdout=False,
debug=procrunner_debug)
debug("result = %s", screen19.prettyprint_dictionary(result))
info("Successfully completed (%.1f sec)", result["runtime"])
if result["exitcode"] != 0:
warning("Failed with exit code %d", result["exitcode"])
sys.exit(1)
with open("overload.json") as fh:
overload_data = json.load(fh)
info("Pixel intensity distribution:")
count_sum = 0
hist = {}
if "bins" in overload_data:
for b in range(overload_data["bin_count"]):
if overload_data["bins"][b] > 0:
hist[b] = overload_data["bins"][b]
count_sum += b * overload_data["bins"][b]
else:
hist = {
int(k): v
for k, v in overload_data["counts"].items() if int(k) > 0
}
count_sum = sum([k * v for k, v in hist.items()])
average_to_peak = 1
if mosaicity_correction:
# Adjust for the detector count rate correction
if self._sigma_m:
delta_z = self._oscillation / self._sigma_m / math.sqrt(2)
average_to_peak = (
math.sqrt(math.pi) * delta_z * math.erf(delta_z) +
math.exp(-(delta_z**2)) - 1) / delta_z**2
info("Average-to-peak intensity ratio: %f", average_to_peak)
scale = 100 * overload_data["scale_factor"] / average_to_peak
info("Determined scale factor for intensities as %f", scale)
debug(
"intensity histogram: { %s }",
", ".join(["%d:%d" % (k, hist[k]) for k in sorted(hist)]),
)
max_count = max(hist.keys())
hist_max = max_count * scale
hist_granularity, hist_format = 1, "%.0f"
if hist_max < 50:
hist_granularity, hist_format = 2, "%.1f"
if hist_max < 15:
hist_granularity, hist_format = 10, "%.1f"
rescaled_hist = {}
for x in hist.keys():
rescaled = round(x * scale * hist_granularity)
if rescaled > 0:
rescaled_hist[rescaled] = hist[x] + rescaled_hist.get(
rescaled, 0)
hist = rescaled_hist
debug(
"rescaled histogram: { %s }",
", ".join([(hist_format + ":%d") % (k / hist_granularity, hist[k])
for k in sorted(hist)]),
)
screen19.plot_intensities(hist,
1 / hist_granularity,
procrunner_debug=procrunner_debug)
linear_response_limit = 100 * self.params.maximum_flux.trusted_range_correction
marginal_limit = max(70, linear_response_limit)
text = "".join((
"Strongest pixel (%d counts) " % max_count,
"reaches %.1f%% " % hist_max,
"of the detector count rate limit",
))
if hist_max > 100:
warning("Warning: %s!", text)
else:
info(text)
if ("overload_limit" in overload_data
and max_count >= overload_data["overload_limit"]):
warning(
"Warning: THE DATA CONTAIN REGULAR OVERLOADS!\n"
" The photon incidence rate is outside the specified "
"limits of the detector.\n"
" The built-in detector count rate correction cannot "
"adjust for this.\n"
" You should aim for count rates below {:.0%} of the "
"detector limit.".format(
self.params.maximum_flux.trusted_range_correction))
elif hist_max > marginal_limit:
warning(
"Warning: The photon incidence rate is well outside the "
"linear response region of the detector (<{:.0%}).\n"
" The built-in detector count rate correction may not be "
"able to adjust for this.".format(
self.params.maximum_flux.trusted_range_correction))
elif hist_max > linear_response_limit:
info("The photon incidence rate is outside the linear response "
"region of the detector (<{:.0%}).\n"
" The built-in detector count rate correction may be able "
"to adjust for this.".format(
self.params.maximum_flux.trusted_range_correction))
if not mosaicity_correction:
warning(
"Warning: Not enough data for proper profile estimation."
" The spot intensities are not corrected for mosaicity.\n"
" The true photon incidence rate will be higher than the "
"given estimate.")
info("Total sum of counts in dataset: %d", count_sum)
def _find_spots(self, args=None): # type: (Optional[List[str]]) -> None
"""
Call `dials.find_spots` on the imported experiment list.
Args:
args (optional): List of any additional PHIL parameters to be used by
dials.import.
"""
info("\nFinding spots...")
dials_start = timeit.default_timer()
# Use some choice fillets from dials.find_spots
# Ignore `args`, use `self.params`
# Loop through all the imagesets and find the strong spots
self.refls = flex.reflection_table.from_observations(
self.expts, self.params.dials_find_spots)
# Add n_signal column - before deleting shoeboxes
good = MaskCode.Foreground | MaskCode.Valid
self.refls["n_signal"] = self.refls["shoebox"].count_mask_values(good)
# Delete the shoeboxes
if not self.params.dials_find_spots.output.shoeboxes:
del self.refls["shoebox"]
info(
60 * "-" + "\n%s\n" + 60 * "-" +
"\nSuccessfully completed (%.1f sec)",
spot_counts_per_image_plot(self.refls),
timeit.default_timer() - dials_start,
)
def _index(self): # type: () -> bool
"""
Call `dials.index` on the output of spot finding.
Returns:
Boolean value indicating whether indexing was successful.
"""
dials_start = timeit.default_timer()
# Prepare max_cell constraint strategies.
max_cell = self.params.dials_index.indexing.max_cell
# By default, try unconstrained max_cell followed by max_cell=20.
# If the user has already specified a max_cell < 20, do not relax to 20Å.
cell_constraints = [([], max_cell)]
if not max_cell or max_cell is Auto or max_cell > 20:
cell_constraints += [(["max_cell constraint"], 20)]
# Prepare indexing methods, preferring the real_space_grid_search if a
# known unit cell has been specified, otherwise using 3D FFT, then 1D FFT.
methods = ([
(["real space grid search"], "real_space_grid_search")
] if self.params.dials_index.indexing.known_symmetry.unit_cell else [])
methods += [(["3D FFT"], "fft3d"), (["1D FFT"], "fft1d")]
# Cycle through the indexing methods for each of the max_cell constraint
# strategies until an indexing solution is found.
for i, (max_cell_msg, max_cell) in enumerate(cell_constraints):
# Set the max_cell constraint strategy.
self.params.dials_index.indexing.max_cell = max_cell
for j, (method_msg, method) in enumerate(methods):
# Set the indexing method.
self.params.dials_index.indexing.method = method
# Log a handy message to the user.
msg = ("Retrying with " +
" and ".join(method_msg + max_cell_msg) if i +
j else "Indexing")
info("\n%s...", msg)
try:
# If indexing is successful, break out of the inner loop.
self.expts, self.refls = index(self.expts, [self.refls],
self.params.dials_index)
break
except (DialsIndexError, ValueError) as e:
# If indexing is unsuccessful, try again with the next
# strategy.
warning("Failed: %s", str(e))
continue
else:
# When all the indexing methods are unsuccessful, move onto
# the next max_cell constraint strategy and try again.
continue
# We should only get here if successfully indexed. Break out of the loop
break
else:
# Indexing completely unsuccessful.
return False
sg_type = self.expts[0].crystal.get_crystal_symmetry().space_group(
).type()
symb = sg_type.universal_hermann_mauguin_symbol()
unit_cell = self.expts[0].crystal.get_unit_cell()
self.refls.as_file(self.params.dials_index.output.reflections)
self.expts.as_file(self.params.dials_index.output.experiments)
self.refls.as_file(self.params.dials_index.output.reflections)
info(
"Found primitive solution: %s %s using %s reflections\n"
"Indexed experiments and reflections saved as %s, %s\n"
"Successfully completed (%.1f sec)",
symb,
unit_cell,
self.refls["id"].count(0),
self.params.dials_index.output.experiments,
self.params.dials_index.output.reflections,
timeit.default_timer() - dials_start,
)
# Report the indexing successful.
return True
def _wilson_calculation(self): # type: () -> None
"""
Run `screen19.minimum_exposure` on an experiment list and reflection table.
For best results, the reflections and experiment list should contain the
results of integration or scaling. If only strong spots are used, the Wilson
plot fit may be poor.
"""
dials_start = timeit.default_timer()
info("\nEstimating lower exposure bound...")
suggest_minimum_exposure(self.expts, self.refls,
self.params.minimum_exposure)
info("Successfully completed (%.1f sec)",
timeit.default_timer() - dials_start)
def _refine(self): # type: () -> None
"""
Run `dials.refine` on the results of indexing.
"""
dials_start = timeit.default_timer()
info("\nRefining...")
try:
self.expts, self.refls, _, _ = run_dials_refine(
self.expts, self.refls, self.params.dials_refine)
except Sorry as e:
warning("dials.refine failed: %d\nGiving up.\n", e)
sys.exit(1)
info("Successfully refined (%.1f sec)",
timeit.default_timer() - dials_start)
def _create_profile_model(self): # type: () -> bool
"""
Run `dials.create_profile_model` on indexed reflections.
The indexed experiment list will be overwritten with a copy that includes
the profile model but is otherwise identical.
Returns:
Boolean value indicating whether it was possible to determine a profile
model from the data.
"""
info("\nCreating profile model...")
command = [
"dials.create_profile_model",
self.params.dials_index.output.experiments,
self.params.dials_index.output.reflections,
"output = %s" % self.params.dials_index.output.experiments,
]
result = procrunner.run(command,
print_stdout=False,
debug=procrunner_debug)
debug("result = %s", screen19.prettyprint_dictionary(result))
self._sigma_m = None
if result["exitcode"] == 0:
db = ExperimentList.from_file(
self.params.dials_index.output.experiments)[0]
self._oscillation = db.imageset.get_scan().get_oscillation()[1]
self._sigma_m = db.profile.sigma_m()
info(
u"%d images, %s° oscillation, σ_m=%.3f°",
db.imageset.get_scan().get_num_images(),
str(self._oscillation),
self._sigma_m,
)
info("Successfully completed (%.1f sec)", result["runtime"])
return True
warning("Failed with exit code %d", result["exitcode"])
return False
def _integrate(self): # type: () -> None
"""Run `dials.integrate` to integrate reflection intensities."""
dials_start = timeit.default_timer()
info("\nIntegrating...")
# Don't waste time recreating the profile model
self.params.dials_integrate.create_profile_model = False
# Get the dials.integrate PHIL scope, populated with parsed input parameters
integrate_scope = phil_scope.get("dials_integrate").objects[0]
integrate_scope.name = ""
integrate_scope = integrate_scope.format(self.params.dials_integrate)
try:
integrated_experiments, integrated_reflections = _run_integration(
integrate_scope,
self.params.dials_index.output.experiments,
self.params.dials_index.output.reflections,
)
# Save the output to files
integrated_reflections.as_file(
self.params.dials_integrate.output.reflections)
integrated_experiments.as_file(
self.params.dials_integrate.output.experiments)
# ... and also store the output internally
self.expts, self.refls = integrated_experiments, integrated_reflections
info(
"Successfully completed (%.1f sec)",
timeit.default_timer() - dials_start,
)
except SystemExit as e:
if e.code:
warning("dials.integrate failed with exit code %d\nGiving up.",
e.code)
sys.exit(1)
# This is a hacky check but should work for as long as DIALS 2.0 is supported.
if version.dials_version() < "DIALS 2.1":
def _refine_bravais(self, experiments, reflections):
# type: (ExperimentList, flex.reflection_table) -> None
"""
Run `dials.refine_bravais_settings` on an experiments and reflections.
Args:
experiments: An experiment list..
reflections: The corresponding reflection table.
"""
info("\nRefining Bravais settings...")
command = [
"dials.refine_bravais_settings", experiments, reflections
]
result = procrunner.run(command,
print_stdout=False,
debug=procrunner_debug)
debug("result = %s", screen19.prettyprint_dictionary(result))
if result["exitcode"] == 0:
m = re.search(
r"[-+]{3,}\n[^\n]*\n[-+|]{3,}\n(.*\n)*[-+]{3,}",
result["stdout"].decode("utf-8"),
)
if m:
info(m.group(0))
else:
info(
"Could not interpret dials.refine_bravais_settings output, "
"please check dials.refine_bravais_settings.log")
info("Successfully completed (%.1f sec)", result["runtime"])
else:
warning("Failed with exit code %d", result["exitcode"])
sys.exit(1)
else:
def _refine_bravais(self): # type: () -> None
"""Run `dials.refine_bravais_settings` to determine the space group."""
dials_start = timeit.default_timer()
info("\nRefining Bravais settings...")
self.refls = eliminate_sys_absent(self.expts, self.refls)
map_to_primitive(self.expts, self.refls)
try:
refined_settings = refined_settings_from_refined_triclinic(
self.expts, self.refls, self.params.dials_refine_bravais)
except RuntimeError as e:
warning("dials.refine_bravais_settings failed.\nGiving up.")
sys.exit(e)
possible_bravais_settings = {
solution["bravais"]
for solution in refined_settings
}
bravais_lattice_to_space_group_table(possible_bravais_settings)
try:
# Old version of dials with as_str() method
logger.info(refined_settings.as_str())
except AttributeError:
# Newer versions of dials (>= 2.2.2) has proper __str__ method
logger.info(refined_settings)
info(
"Successfully completed (%.1f sec)",
timeit.default_timer() - dials_start,
)
def _report(self, experiments, reflections):
# type: (ExperimentList, flex.reflection_table) -> None
"""
Run `dials.report` on an experiment list and reflection table.
Args:
experiments: An experiment list.
reflections: The corresponding reflection table.
"""
info("\nCreating report...")
command = ["dials.report", experiments, reflections]
result = procrunner.run(command,
print_stdout=False,
debug=procrunner_debug)
debug("result = %s", screen19.prettyprint_dictionary(result))
if result["exitcode"] == 0:
info("Successfully completed (%.1f sec)", result["runtime"])
# if sys.stdout.isatty():
# info("Trying to start browser")
# try:
# import subprocess
# d = dict(os.environ)
# d["LD_LIBRARY_PATH"] = ""
# subprocess.Popen(["xdg-open", "dials-report.html"], env=d)
# except Exception as e:
# debug("Could not open browser\n%s", str(e))
else:
warning("Failed with exit code %d", result["exitcode"])
sys.exit(1)
def run(self, args=None, phil=phil_scope, set_up_logging=False):
# type: (Optional[List[str]], scope, bool) -> None
"""
TODO: Docstring.
Args:
args:
phil:
set_up_logging:
Returns:
"""
usage = "%prog [options] image_directory | image_files.cbf | imported.expt"
parser = OptionParser(usage=usage,
epilog=__doc__,
phil=phil,
check_format=False)
self.params, options, unhandled = parser.parse_args(
args=args,
show_diff_phil=True,
return_unhandled=True,
quick_parse=True)
version_information = "screen19 v%s using %s (%s)" % (
screen19.__version__,
dials.util.version.dials_version(),
time.strftime("%Y-%m-%d %H:%M:%S"),
)
start = timeit.default_timer()
if len(unhandled) == 0:
print(__doc__)
print(version_information)
return
if set_up_logging:
# Configure the logging
log.config(verbosity=self.params.verbosity,
logfile=self.params.output.log)
# Unless verbose output has been requested, suppress generation of
# debug and info log records from any child DIALS command, retaining
# those from screen19 itself.
if not self.params.verbosity:
logging.getLogger("dials").setLevel(logging.WARNING)
logging.getLogger("dials.screen19").setLevel(logging.INFO)
info(version_information)
debug("Run with:\n%s\n%s", " ".join(unhandled),
parser.diff_phil.as_str())
self._count_processors(nproc=self.params.nproc)
debug("Using %s processors", self.nproc)
# Set multiprocessing settings for spot-finding, indexing and
# integration to match the top-level specified number of processors
self.params.dials_find_spots.spotfinder.mp.nproc = self.nproc
self.params.dials_index.indexing.nproc = self.nproc
# Setting self.params.dials_refine.refinement.mp.nproc is not helpful
self.params.dials_integrate.integration.mp.nproc = self.nproc
# Set the input and output parameters for the DIALS components
# TODO: Compare to diff_phil and start from later in the pipeline if
# appropriate
self._import(unhandled)
imported_name = self.params.dials_import.output.experiments
self._find_spots()
if not self._index():
info("\nRetrying for stronger spots only...")
strong_refls = self.refls
self.params.dials_find_spots.spotfinder.threshold.dispersion.sigma_strong = (
15)
self._find_spots()
if not self._index():
warning("Giving up.")
self.expts.as_file(imported_name)
strong_refls.as_file("strong.refl")
self.refls.as_file("stronger.refl")
info(
"Could not find an indexing solution. You may want to "
"have a look at the reciprocal space by running:\n\n"
" dials.reciprocal_lattice_viewer %s %s\n\n"
"or, to only include stronger spots:\n\n"
" dials.reciprocal_lattice_viewer %s %s\n",
imported_name,
"strong.refl",
imported_name,
"stronger.refl",
)
sys.exit(1)
if not self._create_profile_model():
info(
"\nRefining model to attempt to increase number of valid spots..."
)
self._refine()
if not self._create_profile_model():
warning("Giving up.")
info(
"The identified indexing solution may not be correct. "
"You may want to have a look at the reciprocal space by "
"running:\n\n"
" dials.reciprocal_lattice_viewer indexed.expt indexed.refl\n"
)
sys.exit(1)
self._check_intensities()
if self.params.minimum_exposure.data == "integrated":
self._integrate()
self._wilson_calculation()
experiments = self.params.dials_integrate.output.experiments
reflections = self.params.dials_integrate.output.reflections
else:
self._wilson_calculation()
experiments = self.params.dials_create_profile.output
reflections = self.params.dials_index.output.reflections
# This is a hacky check but should work for as long as DIALS 2.0 is supported.
if version.dials_version() < "DIALS 2.1":
self._refine_bravais(experiments, reflections)
else:
self._refine_bravais()
self._report(experiments, reflections)
runtime = timeit.default_timer() - start
debug(
"Finished at %s, total runtime: %.1f",
time.strftime("%Y-%m-%d %H:%M:%S"),
runtime,
)
info("screen19 successfully completed (%.1f sec).", runtime)
print('Loading DIALS files.')
elist = ExperimentListFactory.from_json_file(expt_file, check_format=False)
refls = reflection_table.from_file(refl_file)
# Remove outliers
print('Removing outliers')
idx = refls.get_flags(refls.flags.used_in_refinement).as_numpy_array()
idy = np.arange(len(elist))[idx].tolist()
elist = ExperimentList([elist[i] for i in idy])
refls = refls.select(flex.bool(idx))
# Get experiment data from experiment objects
print('Getting experiment data.')
img_num = 0
i = 0
img = elist.imagesets()[img_num]
experiment = elist[0]
while (True): # Get first expt for this image
experiment = elist[i]
if (experiment.imageset == img):
break
i = i + 1
cryst = experiment.crystal
spacegroup = gemmi.SpaceGroup(
cryst.get_space_group().type().universal_hermann_mauguin_symbol())
# Get beam vector
s0 = np.array(experiment.beam.get_s0())
# Get unit cell params
cell_params = cryst.get_unit_cell().parameters()
def update_all_data(reflections_path=None, experiments_path=None):
dat = InfoData()
dat.ref2exp = None
if reflections_path is not None:
try:
refl_tabl = flex.reflection_table.from_file(reflections_path)
dat.n_strng = refl_tabl.get_flags(
refl_tabl.flags.strong).count(True)
dat.n_index = refl_tabl.get_flags(
refl_tabl.flags.indexed).count(True)
dat.n_refnd = refl_tabl.get_flags(
refl_tabl.flags.used_in_refinement).count(True)
dat.n_integ_sum = refl_tabl.get_flags(
refl_tabl.flags.integrated_sum).count(True)
dat.n_integ_prf = refl_tabl.get_flags(
refl_tabl.flags.integrated_prf).count(True)
except BaseException as e:
# We don't want to catch bare exceptions but don't know
# what this was supposed to catch. Log it.
logger.info(" >> Caught unknown exception type:",
type(e).__name__, e, "N###")
logger.info("failed to find reflections")
logger.info("reflections_path = %s", reflections_path)
if experiments_path is not None:
try:
experiments = ExperimentListFactory.from_json_file(
experiments_path, check_format=False)
except BaseException as e:
# We don't want to catch bare exceptions but don't know
# what this was supposed to catch. Log it.
logger.info("\n exception #1 %s: %s", type(e).__name__, e)
# TODO Maybe the next try block is not needed, consider removing all
try:
# FIXME here only take the first datablock. What if there are more?
datablock = DataBlockFactory.from_serialized_format(
experiments_path, check_format=False)[0]
# FIXME here only take the first model from each
beam = datablock.unique_beams()[0]
detector = datablock.unique_detectors()[0]
scan = datablock.unique_scans()[0]
# build a pseudo ExperimentList (with empty crystals)
experiments = ExperimentList()
experiments.append(
Experiment(beam=beam, detector=detector, scan=scan))
except ValueError:
logger.info("failed to read json file")
logger.info("experiments_path = %s", experiments_path)
return dat
try:
imageset_tmp = experiments.imagesets()[0]
mask_file = imageset_tmp.external_lookup.mask.filename
pick_file = open(mask_file, "rb")
mask_tup_obj = pickle.load(pick_file)
pick_file.close()
dat.mask_flex = mask_tup_obj[0]
mask_np_arr = dat.mask_flex.as_numpy_array()
dat.np_mask = mask_np_arr
except IOError:
logger.info("No mask in this node")
dat.np_mask = None
dat.mask_flex = None
# FIXME it takes just the first experiment. What if there are more?
exp = experiments[0]
# Get crystal data
if exp.crystal is not None:
unit_cell = exp.crystal.get_unit_cell()
dat.a, dat.b, dat.c, dat.alpha, dat.beta, dat.gamma = unit_cell.parameters(
)
exp_crystal = exp.crystal
logger.info("exp_crystal = %s", exp_crystal)
b_mat = exp.crystal.get_B()
dat.b11 = b_mat[0]
dat.b12 = b_mat[1]
dat.b13 = b_mat[2]
dat.b21 = b_mat[3]
dat.b22 = b_mat[4]
dat.b23 = b_mat[5]
dat.b31 = b_mat[6]
dat.b32 = b_mat[7]
dat.b33 = b_mat[8]
sg = str(exp.crystal.get_space_group().info())
logger.info("spgr = %s", sg)
dat.spg_group = sg
from scitbx import matrix
u_mat = matrix.sqr(exp.crystal.get_U())
dat.u11 = b_mat[0]
dat.u12 = b_mat[1]
dat.u13 = b_mat[2]
dat.u21 = b_mat[3]
dat.u22 = b_mat[4]
dat.u23 = b_mat[5]
dat.u31 = b_mat[6]
dat.u32 = b_mat[7]
dat.u33 = b_mat[8]
rot_angs = u_mat.r3_rotation_matrix_as_x_y_z_angles(deg=True)
logger.info("u_mat = %s", u_mat)
logger.info("rot_angs = %s", rot_angs)
dat.r1, dat.r2, dat.r3 = rot_angs
# Get beam data
dat.w_lambda = exp.beam.get_wavelength()
# Get detector data
# assume details for the panel the beam intersects are the same
# for the whole detector
# TODO fix it for I23
pnl_beam_intersects, (beam_x,
beam_y) = exp.detector.get_ray_intersection(
exp.beam.get_s0())
pnl = exp.detector[pnl_beam_intersects]
logger.info("beam_x, beam_y = %s %s", beam_x, beam_y)
dat.n_pan_xb_yb = pnl_beam_intersects
dat.xb = beam_x
dat.yb = beam_y
dist = pnl.get_distance()
logger.info("pnl_beam_intersects %s", pnl_beam_intersects)
logger.info("dist %s", dist)
dat.dd = dist
dat.img_ran1, dat.img_ran2 = exp.scan.get_image_range()
dat.oscil1, dat.oscil2 = exp.scan.get_oscillation()
# is the next line right? check what dials.show does
dat.e_time = max(exp.scan.get_exposure_times())
dat.n_pans = len(exp.detector)
dat.x_px_size, dat.y_px_size = pnl.get_pixel_size()
dat.gain = pnl.get_gain()
dat.max_res = exp.detector.get_max_resolution(exp.beam.get_s0())
# manually finding template from experiments_path
try:
with open(experiments_path) as infile:
json_info = json.load(infile)
if type(json_info) is dict:
imageset = json_info["imageset"]
elif type(json_info) is list:
imageset = json_info[0]["imageset"]
dat.tmpl_str = imageset[0]["template"]
logger.info("dat.tmpl_str = %s", dat.tmpl_str)
except (KeyError, IndexError):
logger.info("failed to find template in JSON file")
dat.ref2exp = exp
return dat
def load_imagesets(
template,
directory,
id_image=None,
image_range=None,
use_cache=True,
reversephi=False,
):
global imageset_cache
from dxtbx.model.experiment_list import ExperimentListFactory
from xia2.Applications.xia2setup import known_hdf5_extensions
from dxtbx.imageset import ImageSequence
full_template_path = os.path.join(directory, template)
if full_template_path not in imageset_cache or not use_cache:
from dxtbx.model.experiment_list import BeamComparison
from dxtbx.model.experiment_list import DetectorComparison
from dxtbx.model.experiment_list import GoniometerComparison
params = PhilIndex.params.xia2.settings
compare_beam = BeamComparison(
wavelength_tolerance=params.input.tolerance.beam.wavelength,
direction_tolerance=params.input.tolerance.beam.direction,
polarization_normal_tolerance=params.input.tolerance.beam.
polarization_normal,
polarization_fraction_tolerance=params.input.tolerance.beam.
polarization_fraction,
)
compare_detector = DetectorComparison(
fast_axis_tolerance=params.input.tolerance.detector.fast_axis,
slow_axis_tolerance=params.input.tolerance.detector.slow_axis,
origin_tolerance=params.input.tolerance.detector.origin,
)
compare_goniometer = GoniometerComparison(
rotation_axis_tolerance=params.input.tolerance.goniometer.
rotation_axis,
fixed_rotation_tolerance=params.input.tolerance.goniometer.
fixed_rotation,
setting_rotation_tolerance=params.input.tolerance.goniometer.
setting_rotation,
)
scan_tolerance = params.input.tolerance.scan.oscillation
# If diamond anvil cell data, always use dynamic shadowing
high_pressure = PhilIndex.params.dials.high_pressure.correction
format_kwargs = {
"dynamic_shadowing": params.input.format.dynamic_shadowing
or high_pressure,
"multi_panel": params.input.format.multi_panel,
}
if os.path.splitext(full_template_path)[-1] in known_hdf5_extensions:
# if we are passed the correct file, use this, else look for a master
# file (i.e. something_master.h5)
if os.path.exists(full_template_path) and os.path.isfile(
full_template_path):
master_file = full_template_path
else:
g = glob.glob(os.path.join(directory, "*_master.h5"))
master_file = None
for p in g:
substr = longest_common_substring(template, p)
if substr:
if master_file is None or (len(substr) > len(
longest_common_substring(
template, master_file))):
master_file = p
if master_file is None:
raise RuntimeError("Can't find master file for %s" %
full_template_path)
unhandled = []
experiments = ExperimentListFactory.from_filenames(
[master_file],
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
)
assert len(unhandled) == 0, (
"unhandled image files identified: %s" % unhandled)
else:
from dxtbx.sequence_filenames import locate_files_matching_template_string
params = PhilIndex.get_python_object()
read_all_image_headers = params.xia2.settings.read_all_image_headers
if read_all_image_headers:
paths = sorted(
locate_files_matching_template_string(full_template_path))
unhandled = []
experiments = ExperimentListFactory.from_filenames(
paths,
verbose=False,
unhandled=unhandled,
compare_beam=compare_beam,
compare_detector=compare_detector,
compare_goniometer=compare_goniometer,
scan_tolerance=scan_tolerance,
format_kwargs=format_kwargs,
)
assert len(unhandled) == 0, (
"unhandled image files identified: %s" % unhandled)
else:
from xia2.Handlers.CommandLine import CommandLine
experiments = ExperimentList()
start_ends = CommandLine.get_start_ends(full_template_path)
if not start_ends:
start_ends.append(None)
for start_end in start_ends:
importer = ExperimentListTemplateImporter(
[full_template_path],
format_kwargs=format_kwargs,
image_range=start_end,
)
experiments.extend(importer.experiments)
imagesets = [
iset for iset in experiments.imagesets()
if isinstance(iset, ImageSequence)
]
assert len(imagesets) > 0, "no imageset found"
imageset_cache[full_template_path] = collections.OrderedDict()
if reversephi:
for imageset in imagesets:
goniometer = imageset.get_goniometer()
goniometer.set_rotation_axis(
tuple(-g for g in goniometer.get_rotation_axis()))
reference_geometry = PhilIndex.params.xia2.settings.input.reference_geometry
if reference_geometry is not None and len(reference_geometry) > 0:
update_with_reference_geometry(imagesets, reference_geometry)
# Update the geometry
params = PhilIndex.params.xia2.settings
update_geometry = []
from dials.command_line.dials_import import ManualGeometryUpdater
from dials.util.options import geometry_phil_scope
# Then add manual geometry
work_phil = geometry_phil_scope.format(params.input)
diff_phil = geometry_phil_scope.fetch_diff(source=work_phil)
if diff_phil.as_str() != "":
update_geometry.append(ManualGeometryUpdater(params.input))
imageset_list = []
for imageset in imagesets:
for updater in update_geometry:
imageset = updater(imageset)
imageset_list.append(imageset)
imagesets = imageset_list
for imageset in imagesets:
scan = imageset.get_scan()
exposure_times = scan.get_exposure_times()
epochs = scan.get_epochs()
if exposure_times.all_eq(0) or exposure_times[0] == 0:
exposure_times = flex.double(exposure_times.size(), 1)
scan.set_exposure_times(exposure_times)
elif not exposure_times.all_gt(0):
exposure_times = flex.double(exposure_times.size(),
exposure_times[0])
scan.set_exposure_times(exposure_times)
if epochs.size() > 1 and not epochs.all_gt(0):
if epochs[0] == 0:
epochs[0] = 1
for i in range(1, epochs.size()):
epochs[i] = epochs[i - 1] + exposure_times[i - 1]
scan.set_epochs(epochs)
_id_image = scan.get_image_range()[0]
imageset_cache[full_template_path][_id_image] = imageset
if id_image is not None:
return [imageset_cache[full_template_path][id_image]]
elif image_range is not None:
for imageset in imageset_cache[full_template_path].values():
scan = imageset.get_scan()
scan_image_range = scan.get_image_range()
if (image_range[0] >= scan_image_range[0]
and image_range[1] <= scan_image_range[1]):
b0 = scan.get_batch_offset()
i0 = image_range[0] - scan_image_range[0] + b0
i1 = image_range[1] - scan_image_range[0] + b0
imagesets = [imageset[i0:i1 + 1]]
assert len(
imagesets[0]) == image_range[1] - image_range[0] + 1, len(
imagesets[0])
return imagesets
return list(imageset_cache[full_template_path].values())
