def integrate(self):
# Process reference reflections
self.indexed, _ = self.process_reference(self.indexed)
# Get integrator from input params
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
# Compute the profile model
self.experiments = ProfileModelFactory.create(self.phil,
self.experiments,
self.indexed)
# Predict the reflections
predicted = flex.reflection_table.from_predictions_multi(
self.experiments,
dmin=self.phil.prediction.d_min,
dmax=self.phil.prediction.d_max,
margin=self.phil.prediction.margin,
force_static=self.phil.prediction.force_static)
# Match the predictions with the reference
predicted.match_with_reference(self.indexed)
# Create the integrator
integrator = IntegratorFactory.create(self.phil, self.experiments,
predicted)
# Integrate the reflections
self.integrated = integrator.integrate()
if self.integrated.has_key('intensity.prf.value'):
method = 'prf' # integration by profile fitting
elif self.integrated.has_key('intensity.sum.value'):
method = 'sum' # integration by simple summation
self.integrated = self.integrated.select(
self.integrated['intensity.' + method + '.variance'] >
0) # keep only spots with sigmas above zero
# Save the reflections if selected
if self.phil.output.integrated_filename:
self.save_reflections(self.integrated,
self.phil.output.integrated_filename)
self.write_integration_pickles()
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed)
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated)
crystal_model = self.experiments.crystals()[0]
def predict(experiment, table, params):
elist = ExperimentList([experiment])
predicted = flex.reflection_table.from_predictions_multi(
elist,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
padding=params.prediction.padding,
)
# need to set imageset id?
elist = ProfileModelFactory.create(params, elist, table)
predicted.compute_bbox(elist)
return predicted, elist
def integrate(self):
# Process reference reflections
self.indexed,_ = self.process_reference(self.indexed)
# Get integrator from input params
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
# Compute the profile model
self.experiments = ProfileModelFactory.create(self.phil, self.experiments, self.indexed)
# Predict the reflections
predicted = flex.reflection_table.from_predictions_multi(
self.experiments,
dmin=self.phil.prediction.d_min,
dmax=self.phil.prediction.d_max,
margin=self.phil.prediction.margin,
force_static=self.phil.prediction.force_static)
# Match the predictions with the reference
predicted.match_with_reference(self.indexed)
# Create the integrator
integrator = IntegratorFactory.create(self.phil, self.experiments, predicted)
# Integrate the reflections
self.integrated = integrator.integrate()
if self.integrated.has_key('intensity.prf.value'):
method = 'prf' # integration by profile fitting
elif self.integrated.has_key('intensity.sum.value'):
method = 'sum' # integration by simple summation
self.integrated = self.integrated.select(self.integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero
# Save the reflections if selected
if self.phil.output.integrated_filename:
self.save_reflections(self.integrated, self.phil.output.integrated_filename)
self.write_integration_pickles()
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed)
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated)
crystal_model = self.experiments.crystals()[0]
def integrate(self, experiments, indexed):
from time import time
st = time()
logger.info('*' * 80)
logger.info('Integrating Reflections')
logger.info('*' * 80)
indexed,_ = self.process_reference(indexed)
# Get the integrator from the input parameters
logger.info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
logger.info("")
integrator = IntegratorFactory.create(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# Select only those reflections which were integrated
if 'intensity.prf.variance' in integrated:
selection = integrated.get_flags(
integrated.flags.integrated,
all=True)
else:
selection = integrated.get_flags(
integrated.flags.integrated_sum)
integrated = integrated.select(selection)
len_all = len(integrated)
integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels))
print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all)
# verify sigmas are sensible
if 'intensity.prf.value' in integrated:
if (integrated['intensity.prf.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
if 'intensity.sum.value' in integrated:
if (integrated['intensity.sum.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
# apply detector gain to summation variances
integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain
if 'background.sum.value' in integrated:
if (integrated['background.sum.variance'] < 0).count(True) > 0:
raise Sorry("Found negative variances")
if (integrated['background.sum.variance'] == 0).count(True) > 0:
print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True))
integrated = integrated.select(integrated['background.sum.variance'] > 0)
# apply detector gain to background summation variances
integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated, self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed)
for i in xrange(6):
bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated)
crystal_model = experiments.crystals()[0]
if hasattr(crystal_model, '._ML_domain_size_ang'):
log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
logger.info(log_str)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return integrated
def run(self):
''' Run the script. '''
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.util.command_line import Command
from dials.array_family import flex
from dials.util.options import flatten_reflections, flatten_experiments
from dxtbx.model.experiment.experiment_list import ExperimentListDumper
from libtbx.utils import Sorry
from dials.util import log
log.config()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=True)
reflections = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reflections) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reflections) != 1:
raise Sorry('exactly 1 reflection table must be specified')
if len(experiments) == 0:
raise Sorry('no experiments were specified')
if (not 'background.mean' in reflections[0]) and params.subtract_background:
raise Sorry('for subtract_background need background.mean in reflections')
reflections, _ = self.process_reference(reflections[0], params)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
# Match with predicted
matched, reflections, unmatched = predicted.match_with_reference(reflections)
assert(len(matched) == len(predicted))
assert(matched.count(True) <= len(reflections))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif len(unmatched) != 0:
logger.info('')
logger.info('*' * 80)
logger.info('Warning: %d reference spots were not matched to predictions' % (
len(unmatched)))
logger.info('*' * 80)
logger.info('')
# Create the profile model
experiments = ProfileModelFactory.create(params, experiments, reflections)
for model in experiments:
sigma_b = model.profile.sigma_b(deg=True)
sigma_m = model.profile.sigma_m(deg=True)
if type(sigma_b) == type(1.0):
logger.info('Sigma B: %f' % sigma_b)
logger.info('Sigma M: %f' % sigma_m)
else: # scan varying
mean_sigma_b = sum(sigma_b) / len(sigma_b)
mean_sigma_m = sum(sigma_m) / len(sigma_m)
logger.info('Sigma B: %f' % mean_sigma_b)
logger.info('Sigma M: %f' % mean_sigma_m)
# Wrtie the parameters
Command.start("Writing experiments to %s" % params.output)
dump = ExperimentListDumper(experiments)
with open(params.output, "w") as outfile:
outfile.write(dump.as_json())
Command.end("Wrote experiments to %s" % params.output)
def integrate(self, experiments, indexed):
from time import time
from logging import info
st = time()
info('*' * 80)
info('Integrating Reflections')
info('*' * 80)
indexed,_ = self.process_reference(indexed)
# Get the integrator from the input parameters
info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
info("")
info("=" * 80)
info("")
info("Predicting reflections")
info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
info("")
integrator = IntegratorFactory.create(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
if integrated.has_key('intensity.prf.value'):
method = 'prf' # integration by profile fitting
elif integrated.has_key('intensity.sum.value'):
method = 'sum' # integration by simple summation
integrated = integrated.select(integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated, self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(integrated)
crystal_model = experiments.crystals()[0]
print "Integrated. RMSD indexed,", rmsd_indexed, "RMSD integrated", rmsd_integrated, \
"Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
info('')
info('Time Taken = %f seconds' % (time() - st))
return integrated
def run_integration(params, experiments, reference=None):
"""Perform the integration.
Returns:
experiments: The integrated experiments
reflections: The integrated reflections
report(optional): An integration report.
Raises:
ValueError: For a number of bad inputs
RuntimeError: If the profile model creation fails
"""
predicted = None
rubbish = None
for abs_params in params.absorption_correction:
if abs_params.apply:
if not (params.integration.debug.output
and not params.integration.debug.separate_files):
raise ValueError(
"Shoeboxes must be saved to integration intermediates to apply an absorption correction. "
+
"Set integration.debug.output=True, integration.debug.separate_files=False and "
+
"integration.debug.delete_shoeboxes=True to temporarily store shoeboxes."
)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info("Using external mask: %s", mask.filename)
for tile in mask.data:
logger.info(" Mask has %d pixels masked",
tile.data().count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
summary = "\n".join((
"",
"=" * 80,
"",
"Experiments",
"",
"Models for experiment %d" % i,
"",
str(exp.beam),
str(exp.detector),
))
if exp.goniometer:
summary += str(exp.goniometer) + "\n"
if exp.scan:
summary += str(exp.scan) + "\n"
summary += str(exp.crystal)
logger.info(summary)
logger.info("\n".join(("", "=" * 80, "")))
logger.info(heading("Initialising"))
# Load the data
if reference:
reference, rubbish = process_reference(reference)
# Check pixels don't belong to neighbours
if exp.goniometer is not None and exp.scan is not None:
reference = filter_reference_pixels(reference, experiments)
# Modify experiment list if scan range is set.
experiments, reference = split_for_scan_range(experiments, reference,
params.scan_range)
# Modify experiment list if exclude images is set
if params.exclude_images:
for experiment in experiments:
for index in params.exclude_images:
experiment.imageset.mark_for_rejection(index, True)
# Predict the reflections
logger.info("\n".join(("", "=" * 80, "")))
logger.info(heading("Predicting reflections"))
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
padding=params.prediction.padding,
)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reference)
if matched.count(True) == 0:
raise ValueError("""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
""")
elif unmatched:
msg = (
"Warning: %d reference spots were not matched to predictions" %
unmatched.size())
border = "\n".join(("", "*" * 80, ""))
logger.info("".join((border, msg, border)))
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
old_id = eid
new_id = good_expt_count
tmp = src.select(src["id"] == old_id)
tmp["id"] = flex.int(len(tmp), good_expt_count)
if old_id in tmp.experiment_identifiers():
identifier = tmp.experiment_identifiers()[old_id]
del tmp.experiment_identifiers()[old_id]
tmp.experiment_identifiers()[new_id] = identifier
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference["id"] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info(
"Removing experiment %d: no reference reflections matched to predictions",
expt_id,
)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = sample_predictions(experiments, predicted, params)
# Compute the profile model - either load existing or compute
# can raise RuntimeError
experiments = ProfileModelFactory.create(params, experiments, reference)
for expr in experiments:
if expr.profile is None:
raise ValueError("No profile information in experiment list")
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
integrator = create_integrator(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Remove unintegrated reflections
if not params.output.output_unintegrated_reflections:
keep = reflections.get_flags(reflections.flags.integrated, all=False)
logger.info(
"Removing %d unintegrated reflections of %d total",
keep.count(False),
keep.size(),
)
reflections = reflections.select(keep)
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Correct integrated intensities for absorption correction, if necessary
for abs_params in params.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction, )
experiments, reflections = multi_kapton_correction(
experiments,
reflections,
abs_params.fuller_kapton,
logger=logger)()
if params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter, )
sig_filter = SignificanceFilter(params)
filtered_refls = sig_filter(experiments, reflections)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
logger.info(
"Removed %d reflections out of %d when applying significance filter",
(reflections.size() - filtered_refls.size()),
reflections.size(),
)
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if refls:
accepted_expts.append(expt)
current_id = expt_id
new_id = len(accepted_expts) - 1
refls["id"] = flex.int(len(refls), new_id)
if expt.identifier:
del refls.experiment_identifiers()[current_id]
refls.experiment_identifiers()[new_id] = expt.identifier
accepted_refls.extend(refls)
else:
logger.info(
"Removed experiment %d which has no reflections left after applying significance filter",
expt_id,
)
if not accepted_refls:
raise ValueError(
"No reflections left after applying significance filter")
experiments = accepted_expts
reflections = accepted_refls
# Write a report if requested
report = None
if params.output.report is not None:
report = integrator.report()
return experiments, reflections, report
assert(matched.count(True) <= len(reference))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif matched.count(True) != len(reference):
logger.info('')
logger.info('*' * 80)
logger.info('Warning: %d reference spots were not matched to predictions' % (
len(reference) - matched.count(True)))
logger.info('*' * 80)
logger.info('')
# Compute the profile model
experiments = ProfileModelFactory.create(params, experiments, reference)
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
#print len(reflections)
stats['integrated_intensity'] = flex.sum(reflections['intensity.sum.value'])
except Exception, e:
logger.error(e)
def integrate_coset(self, experiments, indexed):
TRANS = self.params.integration.coset.transformation
# here get a deepcopy that we are not afraid to modify:
experiments_local = copy.deepcopy(experiments)
print("*" * 80)
print("Coset Reflections for modeling or validating the background")
print("*" * 80)
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import create_integrator
# XXX Fixme later implement support for non-primitive lattices NKS
base_set = miller_set( crystal_symmetry = symmetry(
unit_cell = experiments_local[0].crystal.get_unit_cell(),
space_group = experiments_local[0].crystal.get_space_group()),
indices = indexed["miller_index"]
)
triclinic = base_set.customized_copy(
crystal_symmetry=symmetry(unit_cell = experiments_local[0].crystal.get_unit_cell(),space_group="P1"))
# ================
# Compute the profile model
# Predict the reflections
# Create the integrator
# This creates a reference to the experiment, not a copy:
experiments_local = ProfileModelFactory.create(self.params, experiments_local, indexed)
# for debug SLT[TRANS].show_summary()
for e in experiments_local:
e.crystal.set_space_group(triclinic.space_group())
Astar = e.crystal.get_A()
# debug OriAstar = crystal_orientation(Astar,True)
# debug OriAstar.show(legend="old ")
Astarprime = sqr(Astar)* ( sqr(SLT[TRANS]._reindex_N).transpose().inverse() )
e.crystal.set_A(Astarprime)
# debug OriAstarprime = crystal_orientation(Astarprime,True)
# debug OriAstarprime.show(legend="new ")
print("Predicting coset reflections")
print("")
predicted = flex.reflection_table.from_predictions_multi(
experiments_local,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static,
)
print("sublattice total predictions %d"%len(predicted))
# filter the sublattice, keep only the coset indices
miller = predicted["miller_index"]
# coset of modulus 2, wherein there is a binary choice
# see Sauter & Zwart, Acta D (2009) 65:553, Table 1; select the valid coset using eqn(5).
coset_select_algorithm_2 = flex.bool()
M_mat = SLT[TRANS].matS() # the transformation
M_p = M_mat.inverse()
for idx in miller:
H_row = row(idx)
h_orig_setting = H_row * M_p
on_coset=False
for icom in h_orig_setting.elems:
if icom.denominator() > 1: on_coset=True; break
coset_select_algorithm_2.append(on_coset)
predicted = predicted.select(coset_select_algorithm_2)
print("of which %d are in coset %d"%(len(predicted), TRANS))
print("")
integrator = create_integrator(self.params, experiments_local, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated:
del integrated["shoebox"]
if self.params.output.composite_output:
if (
self.params.output.coset_experiments_filename
or self.params.output.coset_filename
):
assert (
self.params.output.coset_experiments_filename is not None
and self.params.output.coset_filename is not None
)
n = len(self.all_coset_experiments)
self.all_coset_experiments.extend(experiments_local)
for i, experiment in enumerate(experiments_local):
refls = integrated.select(integrated["id"] == i)
refls["id"] = flex.int(len(refls), n)
del refls.experiment_identifiers()[i]
refls.experiment_identifiers()[n] = experiment.identifier
self.all_coset_reflections.extend(refls)
n += 1
else:
# Dump experiments to disk
if self.params.output.coset_experiments_filename:
experiments_local.as_json(self.params.output.coset_experiments_filename)
if self.params.output.coset_filename:
# Save the reflections
self.save_reflections(
integrated, self.params.output.coset_filename
)
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "coset RMSD indexed (px): %f\n" % (rmsd_indexed)
log_str += "integrated %d\n"%len(integrated)
for i in range(6):
bright_integrated = integrated.select(
(
integrated["intensity.sum.value"]
/ flex.sqrt(integrated["intensity.sum.variance"])
)
>= i
)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += (
"N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"
% (i, len(bright_integrated), rmsd_integrated)
)
for crystal_model in experiments_local.crystals():
if hasattr(crystal_model, "get_domain_size_ang"):
log_str += (
". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"
% (
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg(),
)
)
print(log_str)
print("")
def run():
parser = OptionParser(phil=phil_scope)
params, options = parser.parse_args(show_diff_phil=True)
assert params.input.single_img is not None
assert params.output_dir is not None
# load the image
img = dxtbx.load(params.input.single_img)
imgset = MemImageSet([img])
datablock = DataBlockFactory.from_imageset(imgset)[0]
spotfinder = SpotFinderFactory.from_parameters(params)
reflections = spotfinder(datablock)
base_name = os.path.splitext(params.input.single_img)[0]
reflections.as_pickle(
os.path.join(params.output_dir, base_name + "_strong.pickle"))
# DGW commented out as reflections.minimum_number_of_reflections no longer exists
# if len(reflections) < params.refinement.reflections.minimum_number_of_reflections:
# print "Not enough spots to index"
# return
# create the spot finder
print("Spotfinder spots found:", len(reflections))
if params.indexing.method == "fft3d":
from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer
elif params.indexing.method == "fft1d":
from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer
elif params.method == "real_space_grid_search":
from dials.algorithms.indexing.real_space_grid_search import (
indexer_real_space_grid_search as indexer, )
try:
idxr = indexer(reflections, [imgset], params=params.indexing)
except (RuntimeError, Sorry) as e:
print(str(e))
return
indexed = idxr.refined_reflections
experiments = idxr.refined_experiments
# from dxtbx.model.experiment.experiment_list import ExperimentListDumper
# dump = ExperimentListDumper(experiments)
# dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json"))
indexed.as_pickle(
os.path.join(params.output_dir, base_name + "_indexed.pickle"))
refiner = RefinerFactory.from_parameters_data_experiments(
params, indexed, experiments)
refiner.run()
refined_experiments = refiner.get_experiments()
# dump = ExperimentListDumper(refined_experiments)
# dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json"))
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
reference = indexed
reference = process_reference(reference)
profile_model = ProfileModelFactory.create(params, refined_experiments,
reference)
predicted = flex.reflection_table.from_predictions_multi(
refined_experiments,
dmin=params.prediction.dmin,
dmax=params.prediction.dmax,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
)
predicted.match_with_reference(reference)
integrator = IntegratorFactory.create(params, experiments, profile_model,
predicted)
# Integrate the reflections
integrated = integrator.integrate()
integrated.as_pickle(
os.path.join(params.output_dir, base_name + "_integrated.pickle"))
assert(matched.count(True) <= len(reference))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif matched.count(True) != len(reference):
logger.info('')
logger.info('*' * 80)
logger.info('Warning: %d reference spots were not matched to predictions' % (
len(reference) - matched.count(True)))
logger.info('*' * 80)
logger.info('')
# Compute the profile model
experiments = ProfileModelFactory.create(params, experiments, reference)
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
#print len(reflections)
stats['integrated_intensity'] = flex.sum(reflections['intensity.sum.value'])
except Exception, e:
logger.error(e)
def run(self):
""" Perform the integration. """
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from logging import info, debug
from time import time
from libtbx.utils import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry("more than 1 reflection file was given")
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry("no experiment list was specified")
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
# Configure logging
log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log)
from dials.util.version import dials_version
info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not "":
info("The following parameters have been modified:\n")
info(diff_phil)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
info("Using external mask: %s" % mask.filename)
info(" Mask has %d pixels masked" % mask.data.count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
debug("Models for experiment %d" % i)
debug("")
debug(str(exp.beam))
debug(str(exp.detector))
if exp.goniometer:
debug(str(exp.goniometer))
if exp.scan:
debug(str(exp.scan))
debug(str(exp.crystal))
info("=" * 80)
info("")
info(heading("Initialising"))
info("")
# Load the data
reference, rubbish = self.process_reference(reference)
info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(experiments, reference, params.scan_range)
# Predict the reflections
info("")
info("=" * 80)
info("")
info(heading("Predicting reflections"))
info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(reference)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reference)
if matched.count(True) == 0:
raise Sorry(
"""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
"""
)
elif len(unmatched) != 0:
info("")
info("*" * 80)
info("Warning: %d reference spots were not matched to predictions" % (len(unmatched)))
info("*" * 80)
info("")
rubbish.extend(unmatched)
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if reference is not None and params.create_profile_model:
experiments = ProfileModelFactory.create(params, experiments, reference)
else:
for expr in experiments:
expr.profile.params = params.profile
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
info("\nTotal time taken: %f" % (time() - start_time))
def run(self):
''' Perform the integration. '''
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from time import time
from libtbx.utils import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry('more than 1 reflection file was given')
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry('no experiment list was specified')
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
# Configure logging
log.config(
params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info('Using external mask: %s' % mask.filename)
logger.info(' Mask has %d pixels masked' % mask.data.count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
logger.debug("Models for experiment %d" % i)
logger.debug("")
logger.debug(str(exp.beam))
logger.debug(str(exp.detector))
if exp.goniometer:
logger.debug(str(exp.goniometer))
if exp.scan:
logger.debug(str(exp.scan))
logger.debug(str(exp.crystal))
logger.info("=" * 80)
logger.info("")
logger.info(heading("Initialising"))
logger.info("")
# Load the data
reference, rubbish = self.process_reference(reference)
logger.info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(
experiments,
reference,
params.scan_range)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(reference)
assert(len(matched) == len(predicted))
assert(matched.count(True) <= len(reference))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif len(unmatched) != 0:
logger.info('')
logger.info('*' * 80)
logger.info('Warning: %d reference spots were not matched to predictions' % (
len(unmatched)))
logger.info('*' * 80)
logger.info('')
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
from dxtbx.model.experiment.experiment_list import ExperimentList
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
tmp = src.select(src['id'] == eid)
tmp['id'] = flex.int(len(tmp), good_expt_count)
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference['id'] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info("Removing experiment %d: no reference reflections matched to predictions"%expt_id)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if (params.create_profile_model and
reference is not None and
"shoebox" in reference):
experiments = ProfileModelFactory.create(params, experiments, reference)
else:
for expr in experiments:
if expr.profile is None:
raise Sorry('No profile information in experiment list')
expr.profile.params = params.profile
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
logger.info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
logger.info("\nTotal time taken: %f" % (time() - start_time))
def work(filename, cl=None):
if cl is None:
cl = []
phil_scope = libtbx.phil.parse("""\
ice_rings {
filter = True
.type = bool
width = 0.004
.type = float(value_min=0.0)
}
index = False
.type = bool
integrate = False
.type = bool
indexing_min_spots = 10
.type = int(value_min=1)
""")
interp = phil_scope.command_line_argument_interpreter()
params, unhandled = interp.process_and_fetch(
cl, custom_processor="collect_remaining")
filter_ice = params.extract().ice_rings.filter
ice_rings_width = params.extract().ice_rings.width
index = params.extract().index
integrate = params.extract().integrate
indexing_min_spots = params.extract().indexing_min_spots
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from dxtbx.model.experiment_list import ExperimentListFactory
from dials.array_family import flex
interp = find_spots_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor="collect_remaining")
logger.info("The following spotfinding parameters have been modified:")
logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str())
params = phil_scope.extract()
# no need to write the hot mask in the server/client
params.spotfinder.write_hot_mask = False
experiments = ExperimentListFactory.from_filenames([filename])
t0 = time.time()
reflections = flex.reflection_table.from_observations(experiments, params)
t1 = time.time()
logger.info("Spotfinding took %.2f seconds" % (t1 - t0))
from dials.algorithms.spot_finding import per_image_analysis
imageset = experiments.imagesets()[0]
reflections.centroid_px_to_mm(experiments)
reflections.map_centroids_to_reciprocal_space(experiments)
stats = per_image_analysis.stats_for_reflection_table(
reflections, filter_ice=filter_ice,
ice_rings_width=ice_rings_width)._asdict()
t2 = time.time()
logger.info("Resolution analysis took %.2f seconds" % (t2 - t1))
if index and stats["n_spots_no_ice"] > indexing_min_spots:
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
from dials.algorithms.indexing import indexer
from dials.command_line.index import phil_scope as index_phil_scope
interp = index_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor="collect_remaining")
logger.info("The following indexing parameters have been modified:")
index_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
if (imageset.get_goniometer() is not None
and imageset.get_scan() is not None
and imageset.get_scan().is_still()):
imageset.set_goniometer(None)
imageset.set_scan(None)
try:
idxr = indexer.Indexer.from_parameters(reflections,
experiments,
params=params)
indexing_results = []
idxr.index()
indexed_sel = idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.indexed)
indexed_sel &= ~(idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.centroid_outlier))
for i_expt, expt in enumerate(idxr.refined_experiments):
sel = idxr.refined_reflections["id"] == i_expt
sel &= indexed_sel
indexing_results.append({
"crystal":
expt.crystal.to_dict(),
"n_indexed":
sel.count(True),
"fraction_indexed":
sel.count(True) / sel.size(),
})
stats["lattices"] = indexing_results
stats["n_indexed"] = indexed_sel.count(True)
stats["fraction_indexed"] = indexed_sel.count(True) / len(
reflections)
except Exception as e:
logger.error(e)
stats["error"] = str(e)
finally:
t3 = time.time()
logger.info("Indexing took %.2f seconds" % (t3 - t2))
if integrate and "lattices" in stats:
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.command_line.integrate import phil_scope as integrate_phil_scope
interp = integrate_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor="collect_remaining")
logger.error(
"The following integration parameters have been modified:")
integrate_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
try:
params.profile.gaussian_rs.min_spots = 0
experiments = idxr.refined_experiments
reference = idxr.refined_reflections
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
)
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reference)
if matched.count(True) == 0:
raise Sorry("""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
""")
elif matched.count(True) != len(reference):
logger.info("")
logger.info("*" * 80)
logger.info(
"Warning: %d reference spots were not matched to predictions"
% (len(reference) - matched.count(True)))
logger.info("*" * 80)
logger.info("")
# Compute the profile model
experiments = ProfileModelFactory.create(
params, experiments, reference)
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
integrator = IntegratorFactory.create(params, experiments,
predicted)
# Integrate the reflections
reflections = integrator.integrate()
# print len(reflections)
stats["integrated_intensity"] = flex.sum(
reflections["intensity.sum.value"])
except Exception as e:
logger.error(e)
stats["error"] = str(e)
finally:
t4 = time.time()
logger.info("Integration took %.2f seconds" % (t4 - t3))
return stats
def integrate(self, experiments, indexed):
# TODO: Figure out if this is necessary and/or how to do this better
indexed, _ = self.process_reference(indexed)
if self.params.integration.integration_only_overrides.trusted_range:
for detector in experiments.detectors():
for panel in detector:
panel.set_trusted_range(
self.params.integration.integration_only_overrides.trusted_range
)
# Get the integrator from the input parameters
from dials.algorithms.integration.integrator import create_integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
new_experiments = ExperimentList()
new_reflections = flex.reflection_table()
for expt_id, expt in enumerate(experiments):
if (
self.params.profile.gaussian_rs.parameters.sigma_b_cutoff is None
or expt.profile.sigma_b()
< self.params.profile.gaussian_rs.parameters.sigma_b_cutoff
):
refls = indexed.select(indexed["id"] == expt_id)
refls["id"] = flex.int(len(refls), len(new_experiments))
# refls.reset_ids()
del refls.experiment_identifiers()[expt_id]
refls.experiment_identifiers()[len(new_experiments)] = expt.identifier
new_reflections.extend(refls)
new_experiments.append(expt)
else:
# TODO: this can be done better, also
print(
"Rejected expt %d with sigma_b %f"
% (expt_id, expt.profile.sigma_b())
)
experiments = new_experiments
indexed = new_reflections
if len(experiments) == 0:
raise RuntimeError("No experiments after filtering by sigma_b")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static,
)
predicted.match_with_reference(indexed)
integrator = create_integrator(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply:
if abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction,
)
elif abs_params.algorithm == "kapton_2019":
from dials.algorithms.integration.kapton_2019_correction import (
multi_kapton_correction,
)
experiments, integrated = multi_kapton_correction(
experiments, integrated, abs_params.fuller_kapton, logger=logger
)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter,
)
sig_filter = SignificanceFilter(self.params)
filtered_refls = sig_filter(experiments, integrated)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if len(refls) > 0:
accepted_expts.append(expt)
refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
accepted_refls.extend(refls)
else:
print(
"Removed experiment %d which has no reflections left after applying significance filter",
expt_id,
)
if len(accepted_refls) == 0:
raise Sorry("No reflections left after applying significance filter")
experiments = accepted_expts
integrated = accepted_refls
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated:
del integrated["shoebox"]
# Dump experiments to disk
if self.params.output.integrated_experiments_filename:
experiments.as_json(self.params.output.integrated_experiments_filename)
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(
integrated, self.params.output.integrated_filename
)
self.write_integration_pickles(integrated, experiments)
# TODO: Figure out what this is
from dials.algorithms.indexing.stills_indexer import (
calc_2D_rmsd_and_displacements,
)
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = f"RMSD indexed (px): {rmsd_indexed:f}\n"
for i in range(6):
bright_integrated = integrated.select(
(
integrated["intensity.sum.value"]
/ flex.sqrt(integrated["intensity.sum.variance"])
)
>= i
)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += (
"N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"
% (i, len(bright_integrated), rmsd_integrated)
)
for crystal_model in experiments.crystals():
if hasattr(crystal_model, "get_domain_size_ang"):
log_str += ". Final ML model: domain size angstroms: {:f}, half mosaicity degrees: {:f}".format(
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg(),
)
print(log_str)
return integrated
def run(self, args=None):
"""Run the script."""
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.array_family import flex
from dials.util import Sorry, log
from dials.util.command_line import Command
from dials.util.options import reflections_and_experiments_from_files
log.config()
# Parse the command line
params, options = self.parser.parse_args(args, show_diff_phil=True)
reflections, experiments = reflections_and_experiments_from_files(
params.input.reflections, params.input.experiments)
if len(reflections) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reflections) != 1:
raise Sorry("exactly 1 reflection table must be specified")
if len(experiments) == 0:
raise Sorry("no experiments were specified")
if ("background.mean"
not in reflections[0]) and params.subtract_background:
raise Sorry(
"for subtract_background need background.mean in reflections")
reflections, _ = self.process_reference(reflections[0], params)
# Check pixels don't belong to neighbours
self.filter_reference_pixels(reflections, experiments)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
padding=params.prediction.padding,
)
# Match with predicted
matched, reflections, unmatched = predicted.match_with_reference(
reflections)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reflections)
if matched.count(True) == 0:
raise Sorry("""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
""")
elif len(unmatched) != 0:
logger.info("")
logger.info("*" * 80)
logger.info(
"Warning: %d reference spots were not matched to predictions",
len(unmatched),
)
logger.info("*" * 80)
logger.info("")
# Create the profile model
experiments = ProfileModelFactory.create(params, experiments,
reflections)
for model in experiments:
sigma_b = model.profile.sigma_b(deg=True)
sigma_m = model.profile.sigma_m(deg=True)
if model.profile.is_scan_varying(): # scan varying
mean_sigma_b = sum(sigma_b) / len(sigma_b)
mean_sigma_m = sum(sigma_m) / len(sigma_m)
logger.info("Sigma B: %f", mean_sigma_b)
logger.info("Sigma M: %f", mean_sigma_m)
else:
logger.info("Sigma B: %f", sigma_b)
logger.info("Sigma M: %f", sigma_m)
# Write the parameters
Command.start(f"Writing experiments to {params.output}")
experiments.as_file(params.output)
Command.end(f"Wrote experiments to {params.output}")
def integrate(self, experiments, indexed):
from time import time
st = time()
logger.info('*' * 80)
logger.info('Integrating Reflections')
logger.info('*' * 80)
indexed, _ = self.process_reference(indexed)
# Get the integrator from the input parameters
logger.info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments,
indexed)
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
logger.info("")
integrator = IntegratorFactory.create(self.params, experiments,
predicted)
# Integrate the reflections
integrated = integrator.integrate()
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import multi_kapton_correction
experiments, integrated = multi_kapton_correction(
experiments,
integrated,
abs_params.fuller_kapton,
logger=logger)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import SignificanceFilter
sig_filter = SignificanceFilter(self.params)
refls = sig_filter(experiments, integrated)
logger.info(
"Removed %d reflections out of %d when applying significance filter"
% (len(integrated) - len(refls), len(integrated)))
if len(refls) == 0:
raise Sorry(
"No reflections left after applying significance filter")
integrated = refls
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and 'shoebox' in integrated:
del integrated['shoebox']
if self.params.output.composite_output:
if self.params.output.integrated_experiments_filename or self.params.output.integrated_filename:
assert self.params.output.integrated_experiments_filename is not None and self.params.output.integrated_filename is not None
from dials.array_family import flex
n = len(self.all_integrated_experiments)
self.all_integrated_experiments.extend(experiments)
for i, experiment in enumerate(experiments):
refls = integrated.select(integrated['id'] == i)
refls['id'] = flex.int(len(refls), n)
self.all_integrated_reflections.extend(refls)
n += 1
else:
# Dump experiments to disk
if self.params.output.integrated_experiments_filename:
from dxtbx.model.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(
self.params.output.integrated_experiments_filename)
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated,
self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n" % (rmsd_indexed)
for i in xrange(6):
bright_integrated = integrated.select(
(integrated['intensity.sum.value'] /
flex.sqrt(integrated['intensity.sum.variance'])) >= i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(
bright_integrated)
else:
rmsd_integrated = 0
log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n" % (
i, len(bright_integrated), rmsd_integrated)
for crystal_model in experiments.crystals():
if hasattr(crystal_model, 'get_domain_size_ang'):
log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f" % (
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg())
logger.info(log_str)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return integrated
def work(filename, cl=None):
if cl is None:
cl = []
import libtbx.phil
phil_scope = libtbx.phil.parse('''\
ice_rings {
filter = True
.type = bool
width = 0.004
.type = float(value_min=0.0)
}
index = False
.type = bool
integrate = False
.type = bool
indexing_min_spots = 10
.type = int(value_min=1)
''')
if not os.access(filename, os.R_OK):
raise RuntimeError("Server does not have read access to file %s" %
filename)
interp = phil_scope.command_line_argument_interpreter()
params, unhandled = interp.process_and_fetch(
cl, custom_processor='collect_remaining')
filter_ice = params.extract().ice_rings.filter
ice_rings_width = params.extract().ice_rings.width
index = params.extract().index
integrate = params.extract().integrate
indexing_min_spots = params.extract().indexing_min_spots
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from dxtbx.datablock import DataBlockFactory
from dials.array_family import flex
interp = find_spots_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
logger.info('The following spotfinding parameters have been modified:')
logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str())
params = phil_scope.extract()
# no need to write the hot mask in the server/client
params.spotfinder.write_hot_mask = False
datablock = DataBlockFactory.from_filenames([filename])[0]
t0 = time.time()
reflections = flex.reflection_table.from_observations(datablock, params)
t1 = time.time()
logger.info('Spotfinding took %.2f seconds' % (t1 - t0))
from dials.algorithms.spot_finding import per_image_analysis
imageset = datablock.extract_imagesets()[0]
scan = imageset.get_scan()
if scan is not None:
i = scan.get_array_range()[0]
else:
i = 0
stats = per_image_analysis.stats_single_image(
imageset,
reflections,
i=i,
plot=False,
filter_ice=filter_ice,
ice_rings_width=ice_rings_width)
stats = stats.__dict__
t2 = time.time()
logger.info('Resolution analysis took %.2f seconds' % (t2 - t1))
if index and stats['n_spots_no_ice'] > indexing_min_spots:
import logging
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
from dials.algorithms.indexing import indexer
interp = indexer.master_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
imagesets = [imageset]
logger.info('The following indexing parameters have been modified:')
indexer.master_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
params.indexing.scan_range = []
if (imageset.get_goniometer() is not None
and imageset.get_scan() is not None
and imageset.get_scan().get_oscillation()[1] == 0):
imageset.set_goniometer(None)
imageset.set_scan(None)
try:
idxr = indexer.indexer_base.from_parameters(reflections,
imagesets,
params=params)
indexing_results = []
idxr.index()
indexed_sel = idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.indexed)
indexed_sel &= ~(idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.centroid_outlier))
for i_expt, expt in enumerate(idxr.refined_experiments):
sel = idxr.refined_reflections['id'] == i_expt
sel &= indexed_sel
indexing_results.append({
'crystal':
expt.crystal.to_dict(),
'n_indexed':
sel.count(True),
'fraction_indexed':
sel.count(True) / sel.size()
})
stats['lattices'] = indexing_results
stats['n_indexed'] = indexed_sel.count(True)
stats['fraction_indexed'] = indexed_sel.count(True) / len(
reflections)
except Exception as e:
logger.error(e)
stats['error'] = str(e)
#stats.crystal = None
#stats.n_indexed = None
#stats.fraction_indexed = None
finally:
t3 = time.time()
logger.info('Indexing took %.2f seconds' % (t3 - t2))
if integrate and 'lattices' in stats:
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.command_line.integrate import phil_scope as integrate_phil_scope
interp = integrate_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
imagesets = [imageset]
logger.error(
'The following integration parameters have been modified:')
integrate_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
try:
params.profile.gaussian_rs.min_spots = 0
experiments = idxr.refined_experiments
reference = idxr.refined_reflections
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert (len(matched) == len(predicted))
assert (matched.count(True) <= len(reference))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif matched.count(True) != len(reference):
logger.info('')
logger.info('*' * 80)
logger.info(
'Warning: %d reference spots were not matched to predictions'
% (len(reference) - matched.count(True)))
logger.info('*' * 80)
logger.info('')
# Compute the profile model
experiments = ProfileModelFactory.create(
params, experiments, reference)
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
integrator = IntegratorFactory.create(params, experiments,
predicted)
# Integrate the reflections
reflections = integrator.integrate()
#print len(reflections)
stats['integrated_intensity'] = flex.sum(
reflections['intensity.sum.value'])
except Exception as e:
logger.error(e)
stats['error'] = str(e)
finally:
t4 = time.time()
logger.info('Integration took %.2f seconds' % (t4 - t3))
return stats
def integrate(self, experiments, indexed):
if self.params.skip_hopper:
return super(Hopper_Processor,
self).integrate(experiments, indexed)
st = time.time()
logger.info("*" * 80)
logger.info("Integrating Reflections")
logger.info("*" * 80)
indexed, _ = self.process_reference(indexed)
if self.params.integration.integration_only_overrides.trusted_range:
for detector in experiments.detectors():
for panel in detector:
panel.set_trusted_range(
self.params.integration.integration_only_overrides.
trusted_range)
if self.params.dispatch.coset:
from xfel.util.sublattice_helper import integrate_coset
integrate_coset(self, experiments, indexed)
# Get the integrator from the input parameters
logger.info("Configuring integrator from input parameters")
from dials.algorithms.integration.integrator import create_integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments,
indexed)
new_experiments = ExperimentList()
new_reflections = flex.reflection_table()
for expt_id, expt in enumerate(experiments):
if (self.params.profile.gaussian_rs.parameters.sigma_b_cutoff is
None or expt.profile.sigma_b() <
self.params.profile.gaussian_rs.parameters.sigma_b_cutoff):
refls = indexed.select(indexed["id"] == expt_id)
refls["id"] = flex.int(len(refls), len(new_experiments))
# refls.reset_ids()
del refls.experiment_identifiers()[expt_id]
refls.experiment_identifiers()[len(
new_experiments)] = expt.identifier
new_reflections.extend(refls)
new_experiments.append(expt)
else:
logger.info("Rejected expt %d with sigma_b %f" %
(expt_id, expt.profile.sigma_b()))
experiments = new_experiments
indexed = new_reflections
if len(experiments) == 0:
raise RuntimeError("No experiments after filtering by sigma_b")
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
# NOTE: this is the only changed needed to dials.stills_process
# TODO: multi xtal
# TODO: add in normal dials predictions as an option
predicted, model = predictions.get_predicted_from_pandas(
self.stage1_df,
self.params.diffBragg,
self.observed,
experiments[0].identifier,
self.device_id,
spectrum_override=self.stage1_modeler.SIM.beam.spectrum)
if self.params.refine_predictions:
experiments, rnd2_refls = self.refine(experiments,
predicted,
refining_predictions=True,
best=self.stage1_df)
# TODO: match rnd2_refls with indexed.refl and re-save indexed.refl
predicted, model = predictions.get_predicted_from_pandas(
self.stage1_df,
self.params.diffBragg,
self.observed,
experiments[0].identifier,
self.device_id,
spectrum_override=self.stage1_modeler.SIM.beam.spectrum)
predicted.match_with_reference(indexed)
integrator = create_integrator(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
if self.params.partial_correct:
integrated = predictions.normalize_by_partiality(
integrated,
model,
default_F=self.params.diffBragg.predictions.default_Famplitude,
gain=self.params.diffBragg.refiner.adu_per_photon)
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply:
if abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction, )
elif abs_params.algorithm == "kapton_2019":
from dials.algorithms.integration.kapton_2019_correction import (
multi_kapton_correction, )
experiments, integrated = multi_kapton_correction(
experiments,
integrated,
abs_params.fuller_kapton,
logger=logger)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter, )
sig_filter = SignificanceFilter(self.params)
filtered_refls = sig_filter(experiments, integrated)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
logger.info(
"Removed %d reflections out of %d when applying significance filter",
len(integrated) - len(filtered_refls),
len(integrated),
)
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if len(refls) > 0:
accepted_expts.append(expt)
refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
accepted_refls.extend(refls)
else:
logger.info(
"Removed experiment %d which has no reflections left after applying significance filter",
expt_id,
)
if len(accepted_refls) == 0:
raise Sorry(
"No reflections left after applying significance filter")
experiments = accepted_expts
integrated = accepted_refls
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated:
del integrated["shoebox"]
if self.params.output.composite_output:
if (self.params.output.integrated_experiments_filename
or self.params.output.integrated_filename):
assert (self.params.output.integrated_experiments_filename
is not None
and self.params.output.integrated_filename is not None)
n = len(self.all_integrated_experiments)
self.all_integrated_experiments.extend(experiments)
for i, experiment in enumerate(experiments):
refls = integrated.select(integrated["id"] == i)
refls["id"] = flex.int(len(refls), n)
del refls.experiment_identifiers()[i]
refls.experiment_identifiers()[n] = experiment.identifier
self.all_integrated_reflections.extend(refls)
n += 1
else:
# Dump experiments to disk
if self.params.output.integrated_experiments_filename:
experiments.as_json(
self.params.output.integrated_experiments_filename)
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated,
self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import (
calc_2D_rmsd_and_displacements, )
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n" % rmsd_indexed
for i in range(6):
bright_integrated = integrated.select(
(integrated["intensity.sum.value"] /
flex.sqrt(integrated["intensity.sum.variance"])) >= i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(
bright_integrated)
else:
rmsd_integrated = 0
log_str += (
"N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"
% (i, len(bright_integrated), rmsd_integrated))
for crystal_model in experiments.crystals():
if hasattr(crystal_model, "get_domain_size_ang"):
log_str += ". Final ML model: domain size angstroms: {:f}, half mosaicity degrees: {:f}".format(
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg(),
)
logger.info(log_str)
logger.info("")
logger.info("Time Taken = %f seconds", time.time() - st)
return integrated
def integrate(self, experiments, indexed):
from time import time
st = time()
logger.info('*' * 80)
logger.info('Integrating Reflections')
logger.info('*' * 80)
indexed,_ = self.process_reference(indexed)
# Get the integrator from the input parameters
logger.info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
logger.info("")
integrator = IntegratorFactory.create(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# Select only those reflections which were integrated
if 'intensity.prf.variance' in integrated:
selection = integrated.get_flags(
integrated.flags.integrated,
all=True)
else:
selection = integrated.get_flags(
integrated.flags.integrated_sum)
integrated = integrated.select(selection)
len_all = len(integrated)
integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels))
print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all)
# verify sigmas are sensible
if 'intensity.prf.value' in integrated:
if (integrated['intensity.prf.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
if 'intensity.sum.value' in integrated:
if (integrated['intensity.sum.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
# apply detector gain to summation variances
integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain
if 'background.sum.value' in integrated:
if (integrated['background.sum.variance'] < 0).count(True) > 0:
raise Sorry("Found negative variances")
if (integrated['background.sum.variance'] == 0).count(True) > 0:
print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True))
integrated = integrated.select(integrated['background.sum.variance'] > 0)
# apply detector gain to background summation variances
integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import multi_kapton_correction
experiments, integrated = multi_kapton_correction(experiments, integrated,
abs_params.fuller_kapton, logger=logger)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import SignificanceFilter
sig_filter = SignificanceFilter(self.params)
refls = sig_filter(experiments, integrated)
logger.info("Removed %d reflections out of %d when applying significance filter"%(len(integrated)-len(refls), len(integrated)))
if len(refls) == 0:
raise Sorry("No reflections left after applying significance filter")
integrated = refls
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated, self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed)
for i in xrange(6):
bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated)
for crystal_model in experiments.crystals():
if hasattr(crystal_model, '_ML_domain_size_ang'):
log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
logger.info(log_str)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return integrated
def run():
parser = OptionParser(
phil = phil_scope)
params, options = parser.parse_args(show_diff_phil=True)
assert params.input.single_img is not None
assert params.output_dir is not None
# load the image
img = dxtbx.load(params.input.single_img)
imgset = MemImageSet([img])
datablock = DataBlockFactory.from_imageset(imgset)[0]
spotfinder = SpotFinderFactory.from_parameters(params)
reflections = spotfinder(datablock)
base_name = os.path.splitext(params.input.single_img)[0]
reflections.as_pickle(os.path.join(params.output_dir, base_name + "_strong.pickle"))
# DGW commented out as reflections.minimum_number_of_reflections no longer exists
#if len(reflections) < params.refinement.reflections.minimum_number_of_reflections:
# print "Not enough spots to index"
# return
# create the spot finder
print "Spotfinder spots found:", len(reflections)
if params.indexing.method == "fft3d":
from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer
elif params.indexing.method == "fft1d":
from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer
elif params.method == "real_space_grid_search":
from dials.algorithms.indexing.real_space_grid_search \
import indexer_real_space_grid_search as indexer
try:
idxr = indexer(reflections, [imgset], params=params.indexing)
except (RuntimeError, Sorry) as e:
print str(e)
return
indexed = idxr.refined_reflections
experiments = idxr.refined_experiments
#from dxtbx.model.experiment.experiment_list import ExperimentListDumper
#dump = ExperimentListDumper(experiments)
#dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json"))
indexed.as_pickle(os.path.join(params.output_dir, base_name + "_indexed.pickle"))
refiner = RefinerFactory.from_parameters_data_experiments(
params, indexed, experiments)
refiner.run()
refined_experiments = refiner.get_experiments()
#dump = ExperimentListDumper(refined_experiments)
#dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json"))
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
reference = indexed
reference = process_reference(reference)
profile_model = ProfileModelFactory.create(params, refined_experiments, reference)
predicted = flex.reflection_table.from_predictions_multi(
refined_experiments,
dmin=params.prediction.dmin,
dmax=params.prediction.dmax,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
predicted.match_with_reference(reference)
integrator = IntegratorFactory.create(params, experiments, profile_model, predicted)
# Integrate the reflections
integrated = integrator.integrate()
integrated.as_pickle(os.path.join(params.output_dir, base_name + "_integrated.pickle"))
def run(self, args=None):
""" Perform the integration. """
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from time import time
from dials.util import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(args=args,
show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry("more than 1 reflection file was given")
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry("no experiment list was specified")
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
if __name__ == "__main__":
# Configure logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
for abs_params in params.absorption_correction:
if abs_params.apply:
if not (params.integration.debug.output
and not params.integration.debug.separate_files):
raise Sorry(
"Shoeboxes must be saved to integration intermediates to apply an absorption correction. "
+
"Set integration.debug.output=True, integration.debug.separate_files=False and "
+
"integration.debug.delete_shoeboxes=True to temporarily store shoeboxes."
)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info("Using external mask: %s" % mask.filename)
for tile in mask.data:
logger.info(" Mask has %d pixels masked" %
tile.data().count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
logger.info("=" * 80)
logger.info("")
logger.info("Experiments")
logger.info("")
logger.info("Models for experiment %d" % i)
logger.info("")
logger.info(str(exp.beam))
logger.info(str(exp.detector))
if exp.goniometer:
logger.info(str(exp.goniometer))
if exp.scan:
logger.info(str(exp.scan))
logger.info(str(exp.crystal))
logger.info("=" * 80)
logger.info("")
logger.info(heading("Initialising"))
logger.info("")
# Load the data
reference, rubbish = self.process_reference(reference)
# Check pixels don't belong to neighbours
if reference is not None:
if exp.goniometer is not None and exp.scan is not None:
self.filter_reference_pixels(reference, experiments)
logger.info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(
experiments, reference, params.scan_range)
# Modify experiment list if exclude images is set
experiments = self.exclude_images(experiments, params.exclude_images)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
padding=params.prediction.padding,
)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reference)
if matched.count(True) == 0:
raise Sorry("""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
""")
elif len(unmatched) != 0:
logger.info("")
logger.info("*" * 80)
logger.info(
"Warning: %d reference spots were not matched to predictions"
% (len(unmatched)))
logger.info("*" * 80)
logger.info("")
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
from dxtbx.model.experiment_list import ExperimentList
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
tmp = src.select(src["id"] == eid)
tmp["id"] = flex.int(len(tmp), good_expt_count)
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference["id"] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info(
"Removing experiment %d: no reference reflections matched to predictions"
% expt_id)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if (params.create_profile_model and reference is not None
and "shoebox" in reference):
experiments = ProfileModelFactory.create(params, experiments,
reference)
else:
experiments = ProfileModelFactory.create(params, experiments)
for expr in experiments:
if expr.profile is None:
raise Sorry("No profile information in experiment list")
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
logger.info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Correct integrated intensities for absorption correction, if necessary
for abs_params in params.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction, )
experiments, reflections = multi_kapton_correction(
experiments,
reflections,
abs_params.fuller_kapton,
logger=logger)()
if params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter, )
from dxtbx.model.experiment_list import ExperimentList
sig_filter = SignificanceFilter(params)
filtered_refls = sig_filter(experiments, reflections)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
logger.info(
"Removed %d reflections out of %d when applying significance filter"
% (len(reflections) - len(filtered_refls), len(reflections)))
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if len(refls) > 0:
accepted_expts.append(expt)
refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
accepted_refls.extend(refls)
else:
logger.info(
"Removed experiment %d which has no reflections left after applying significance filter"
% expt_id)
if len(accepted_refls) == 0:
raise Sorry(
"No reflections left after applying significance filter")
experiments = accepted_expts
reflections = accepted_refls
# Delete the shoeboxes used for intermediate calculations, if requested
if params.integration.debug.delete_shoeboxes and "shoebox" in reflections:
del reflections["shoebox"]
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
logger.info("\nTotal time taken: %f" % (time() - start_time))
return experiments, reflections
def run(self):
''' Perform the integration. '''
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from time import time
from libtbx.utils import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry('more than 1 reflection file was given')
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry('no experiment list was specified')
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
# Configure logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info('Using external mask: %s' % mask.filename)
logger.info(' Mask has %d pixels masked' %
mask.data.count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
logger.debug("Models for experiment %d" % i)
logger.debug("")
logger.debug(str(exp.beam))
logger.debug(str(exp.detector))
if exp.goniometer:
logger.debug(str(exp.goniometer))
if exp.scan:
logger.debug(str(exp.scan))
logger.debug(str(exp.crystal))
logger.info("=" * 80)
logger.info("")
logger.info(heading("Initialising"))
logger.info("")
# Load the data
reference, rubbish = self.process_reference(reference)
logger.info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(
experiments, reference, params.scan_range)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert (len(matched) == len(predicted))
assert (matched.count(True) <= len(reference))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif len(unmatched) != 0:
logger.info('')
logger.info('*' * 80)
logger.info(
'Warning: %d reference spots were not matched to predictions'
% (len(unmatched)))
logger.info('*' * 80)
logger.info('')
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
from dxtbx.model.experiment.experiment_list import ExperimentList
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
tmp = src.select(src['id'] == eid)
tmp['id'] = flex.int(len(tmp), good_expt_count)
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference['id'] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info(
"Removing experiment %d: no reference reflections matched to predictions"
% expt_id)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if (params.create_profile_model and reference is not None
and "shoebox" in reference):
experiments = ProfileModelFactory.create(params, experiments,
reference)
else:
for expr in experiments:
if expr.profile is None:
raise Sorry('No profile information in experiment list')
expr.profile.params = params.profile
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
logger.info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
logger.info("\nTotal time taken: %f" % (time() - start_time))