def integrate(self):
# Process reference reflections
self.indexed, _ = self.process_reference(self.indexed)
# Get integrator from input params
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
# Compute the profile model
self.experiments = ProfileModelFactory.create(self.phil,
self.experiments,
self.indexed)
# Predict the reflections
predicted = flex.reflection_table.from_predictions_multi(
self.experiments,
dmin=self.phil.prediction.d_min,
dmax=self.phil.prediction.d_max,
margin=self.phil.prediction.margin,
force_static=self.phil.prediction.force_static)
# Match the predictions with the reference
predicted.match_with_reference(self.indexed)
# Create the integrator
integrator = IntegratorFactory.create(self.phil, self.experiments,
predicted)
# Integrate the reflections
self.integrated = integrator.integrate()
if self.integrated.has_key('intensity.prf.value'):
method = 'prf' # integration by profile fitting
elif self.integrated.has_key('intensity.sum.value'):
method = 'sum' # integration by simple summation
self.integrated = self.integrated.select(
self.integrated['intensity.' + method + '.variance'] >
0) # keep only spots with sigmas above zero
# Save the reflections if selected
if self.phil.output.integrated_filename:
self.save_reflections(self.integrated,
self.phil.output.integrated_filename)
self.write_integration_pickles()
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed)
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated)
crystal_model = self.experiments.crystals()[0]
def integrate(self):
# Process reference reflections
self.indexed,_ = self.process_reference(self.indexed)
# Get integrator from input params
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
# Compute the profile model
self.experiments = ProfileModelFactory.create(self.phil, self.experiments, self.indexed)
# Predict the reflections
predicted = flex.reflection_table.from_predictions_multi(
self.experiments,
dmin=self.phil.prediction.d_min,
dmax=self.phil.prediction.d_max,
margin=self.phil.prediction.margin,
force_static=self.phil.prediction.force_static)
# Match the predictions with the reference
predicted.match_with_reference(self.indexed)
# Create the integrator
integrator = IntegratorFactory.create(self.phil, self.experiments, predicted)
# Integrate the reflections
self.integrated = integrator.integrate()
if self.integrated.has_key('intensity.prf.value'):
method = 'prf' # integration by profile fitting
elif self.integrated.has_key('intensity.sum.value'):
method = 'sum' # integration by simple summation
self.integrated = self.integrated.select(self.integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero
# Save the reflections if selected
if self.phil.output.integrated_filename:
self.save_reflections(self.integrated, self.phil.output.integrated_filename)
self.write_integration_pickles()
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed)
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated)
crystal_model = self.experiments.crystals()[0]
def integrate(self, experiments, indexed):
from logging import info
info('*' * 80)
info('Integrating Reflections')
info('*' * 80)
from xfel.command_line.xfel_process import Script as ProcessScript
assert len(experiments) == 2
integrated = ProcessScript.integrate(self, experiments, indexed)
if 'intensity.prf.value' in integrated:
method = 'prf' # integration by profile fitting
elif 'intensity.sum.value' in integrated:
method = 'sum' # integration by simple summation
integrated = integrated.select(
integrated['intensity.' + method + '.variance'] >
0) # keep only spots with sigmas above zero
integrated = remove_invalid_reflections(integrated)
self.save_reflections(integrated,
self.params.output.integrated_filename)
def write_integration_pickles_callback(params, outfile, frame):
from cxi_xdr_xes.two_color.two_color_dump import correction_for_metal_foil_absorption, derive_scale_and_B_to_model
correction_for_metal_foil_absorption(params, frame)
if params.calc_G_and_B.do_calc:
derive_scale_and_B_to_model(params.calc_G_and_B, outfile,
frame)
self.write_integration_pickles(
integrated,
experiments,
callback=write_integration_pickles_callback)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(integrated)
crystal_model = experiments.crystals()[0]
print "Integrated. RMSD indexed,", rmsd_indexed, "RMSD integrated", rmsd_integrated, \
"Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
return integrated
def integrate(self, experiments, indexed):
from time import time
st = time()
logger.info('*' * 80)
logger.info('Integrating Reflections')
logger.info('*' * 80)
indexed,_ = self.process_reference(indexed)
# Get the integrator from the input parameters
logger.info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
logger.info("")
integrator = IntegratorFactory.create(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# Select only those reflections which were integrated
if 'intensity.prf.variance' in integrated:
selection = integrated.get_flags(
integrated.flags.integrated,
all=True)
else:
selection = integrated.get_flags(
integrated.flags.integrated_sum)
integrated = integrated.select(selection)
len_all = len(integrated)
integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels))
print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all)
# verify sigmas are sensible
if 'intensity.prf.value' in integrated:
if (integrated['intensity.prf.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
if 'intensity.sum.value' in integrated:
if (integrated['intensity.sum.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
# apply detector gain to summation variances
integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain
if 'background.sum.value' in integrated:
if (integrated['background.sum.variance'] < 0).count(True) > 0:
raise Sorry("Found negative variances")
if (integrated['background.sum.variance'] == 0).count(True) > 0:
print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True))
integrated = integrated.select(integrated['background.sum.variance'] > 0)
# apply detector gain to background summation variances
integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import multi_kapton_correction
experiments, integrated = multi_kapton_correction(experiments, integrated,
abs_params.fuller_kapton, logger=logger)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import SignificanceFilter
sig_filter = SignificanceFilter(self.params)
refls = sig_filter(experiments, integrated)
logger.info("Removed %d reflections out of %d when applying significance filter"%(len(integrated)-len(refls), len(integrated)))
if len(refls) == 0:
raise Sorry("No reflections left after applying significance filter")
integrated = refls
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated, self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed)
for i in xrange(6):
bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated)
for crystal_model in experiments.crystals():
if hasattr(crystal_model, '_ML_domain_size_ang'):
log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
logger.info(log_str)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return integrated
def integrate_coset(self, experiments, indexed):
TRANS = self.params.integration.coset.transformation
# here get a deepcopy that we are not afraid to modify:
experiments_local = copy.deepcopy(experiments)
print("*" * 80)
print("Coset Reflections for modeling or validating the background")
print("*" * 80)
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import create_integrator
# XXX Fixme later implement support for non-primitive lattices NKS
base_set = miller_set( crystal_symmetry = symmetry(
unit_cell = experiments_local[0].crystal.get_unit_cell(),
space_group = experiments_local[0].crystal.get_space_group()),
indices = indexed["miller_index"]
)
triclinic = base_set.customized_copy(
crystal_symmetry=symmetry(unit_cell = experiments_local[0].crystal.get_unit_cell(),space_group="P1"))
# ================
# Compute the profile model
# Predict the reflections
# Create the integrator
# This creates a reference to the experiment, not a copy:
experiments_local = ProfileModelFactory.create(self.params, experiments_local, indexed)
# for debug SLT[TRANS].show_summary()
for e in experiments_local:
e.crystal.set_space_group(triclinic.space_group())
Astar = e.crystal.get_A()
# debug OriAstar = crystal_orientation(Astar,True)
# debug OriAstar.show(legend="old ")
Astarprime = sqr(Astar)* ( sqr(SLT[TRANS]._reindex_N).transpose().inverse() )
e.crystal.set_A(Astarprime)
# debug OriAstarprime = crystal_orientation(Astarprime,True)
# debug OriAstarprime.show(legend="new ")
print("Predicting coset reflections")
print("")
predicted = flex.reflection_table.from_predictions_multi(
experiments_local,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static,
)
print("sublattice total predictions %d"%len(predicted))
# filter the sublattice, keep only the coset indices
miller = predicted["miller_index"]
# coset of modulus 2, wherein there is a binary choice
# see Sauter & Zwart, Acta D (2009) 65:553, Table 1; select the valid coset using eqn(5).
coset_select_algorithm_2 = flex.bool()
M_mat = SLT[TRANS].matS() # the transformation
M_p = M_mat.inverse()
for idx in miller:
H_row = row(idx)
h_orig_setting = H_row * M_p
on_coset=False
for icom in h_orig_setting.elems:
if icom.denominator() > 1: on_coset=True; break
coset_select_algorithm_2.append(on_coset)
predicted = predicted.select(coset_select_algorithm_2)
print("of which %d are in coset %d"%(len(predicted), TRANS))
print("")
integrator = create_integrator(self.params, experiments_local, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated:
del integrated["shoebox"]
if self.params.output.composite_output:
if (
self.params.output.coset_experiments_filename
or self.params.output.coset_filename
):
assert (
self.params.output.coset_experiments_filename is not None
and self.params.output.coset_filename is not None
)
n = len(self.all_coset_experiments)
self.all_coset_experiments.extend(experiments_local)
for i, experiment in enumerate(experiments_local):
refls = integrated.select(integrated["id"] == i)
refls["id"] = flex.int(len(refls), n)
del refls.experiment_identifiers()[i]
refls.experiment_identifiers()[n] = experiment.identifier
self.all_coset_reflections.extend(refls)
n += 1
else:
# Dump experiments to disk
if self.params.output.coset_experiments_filename:
experiments_local.as_json(self.params.output.coset_experiments_filename)
if self.params.output.coset_filename:
# Save the reflections
self.save_reflections(
integrated, self.params.output.coset_filename
)
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "coset RMSD indexed (px): %f\n" % (rmsd_indexed)
log_str += "integrated %d\n"%len(integrated)
for i in range(6):
bright_integrated = integrated.select(
(
integrated["intensity.sum.value"]
/ flex.sqrt(integrated["intensity.sum.variance"])
)
>= i
)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += (
"N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"
% (i, len(bright_integrated), rmsd_integrated)
)
for crystal_model in experiments_local.crystals():
if hasattr(crystal_model, "get_domain_size_ang"):
log_str += (
". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"
% (
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg(),
)
)
print(log_str)
print("")
def integrate(self, experiments, indexed):
from time import time
st = time()
logger.info('*' * 80)
logger.info('Integrating Reflections')
logger.info('*' * 80)
indexed,_ = self.process_reference(indexed)
# Get the integrator from the input parameters
logger.info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
logger.info("")
integrator = IntegratorFactory.create(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# Select only those reflections which were integrated
if 'intensity.prf.variance' in integrated:
selection = integrated.get_flags(
integrated.flags.integrated,
all=True)
else:
selection = integrated.get_flags(
integrated.flags.integrated_sum)
integrated = integrated.select(selection)
len_all = len(integrated)
integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels))
print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all)
# verify sigmas are sensible
if 'intensity.prf.value' in integrated:
if (integrated['intensity.prf.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
if 'intensity.sum.value' in integrated:
if (integrated['intensity.sum.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
# apply detector gain to summation variances
integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain
if 'background.sum.value' in integrated:
if (integrated['background.sum.variance'] < 0).count(True) > 0:
raise Sorry("Found negative variances")
if (integrated['background.sum.variance'] == 0).count(True) > 0:
print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True))
integrated = integrated.select(integrated['background.sum.variance'] > 0)
# apply detector gain to background summation variances
integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated, self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed)
for i in xrange(6):
bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated)
crystal_model = experiments.crystals()[0]
if hasattr(crystal_model, '._ML_domain_size_ang'):
log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
logger.info(log_str)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return integrated
def integrate(self, experiments, indexed):
from time import time
from logging import info
st = time()
info('*' * 80)
info('Integrating Reflections')
info('*' * 80)
indexed,_ = self.process_reference(indexed)
# Get the integrator from the input parameters
info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
info("")
info("=" * 80)
info("")
info("Predicting reflections")
info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
info("")
integrator = IntegratorFactory.create(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
if integrated.has_key('intensity.prf.value'):
method = 'prf' # integration by profile fitting
elif integrated.has_key('intensity.sum.value'):
method = 'sum' # integration by simple summation
integrated = integrated.select(integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated, self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(integrated)
crystal_model = experiments.crystals()[0]
print "Integrated. RMSD indexed,", rmsd_indexed, "RMSD integrated", rmsd_integrated, \
"Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
info('')
info('Time Taken = %f seconds' % (time() - st))
return integrated
def integrate(self, experiments, indexed):
if self.params.skip_hopper:
return super(Hopper_Processor,
self).integrate(experiments, indexed)
st = time.time()
logger.info("*" * 80)
logger.info("Integrating Reflections")
logger.info("*" * 80)
indexed, _ = self.process_reference(indexed)
if self.params.integration.integration_only_overrides.trusted_range:
for detector in experiments.detectors():
for panel in detector:
panel.set_trusted_range(
self.params.integration.integration_only_overrides.
trusted_range)
if self.params.dispatch.coset:
from xfel.util.sublattice_helper import integrate_coset
integrate_coset(self, experiments, indexed)
# Get the integrator from the input parameters
logger.info("Configuring integrator from input parameters")
from dials.algorithms.integration.integrator import create_integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments,
indexed)
new_experiments = ExperimentList()
new_reflections = flex.reflection_table()
for expt_id, expt in enumerate(experiments):
if (self.params.profile.gaussian_rs.parameters.sigma_b_cutoff is
None or expt.profile.sigma_b() <
self.params.profile.gaussian_rs.parameters.sigma_b_cutoff):
refls = indexed.select(indexed["id"] == expt_id)
refls["id"] = flex.int(len(refls), len(new_experiments))
# refls.reset_ids()
del refls.experiment_identifiers()[expt_id]
refls.experiment_identifiers()[len(
new_experiments)] = expt.identifier
new_reflections.extend(refls)
new_experiments.append(expt)
else:
logger.info("Rejected expt %d with sigma_b %f" %
(expt_id, expt.profile.sigma_b()))
experiments = new_experiments
indexed = new_reflections
if len(experiments) == 0:
raise RuntimeError("No experiments after filtering by sigma_b")
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
# NOTE: this is the only changed needed to dials.stills_process
# TODO: multi xtal
# TODO: add in normal dials predictions as an option
predicted, model = predictions.get_predicted_from_pandas(
self.stage1_df,
self.params.diffBragg,
self.observed,
experiments[0].identifier,
self.device_id,
spectrum_override=self.stage1_modeler.SIM.beam.spectrum)
if self.params.refine_predictions:
experiments, rnd2_refls = self.refine(experiments,
predicted,
refining_predictions=True,
best=self.stage1_df)
# TODO: match rnd2_refls with indexed.refl and re-save indexed.refl
predicted, model = predictions.get_predicted_from_pandas(
self.stage1_df,
self.params.diffBragg,
self.observed,
experiments[0].identifier,
self.device_id,
spectrum_override=self.stage1_modeler.SIM.beam.spectrum)
predicted.match_with_reference(indexed)
integrator = create_integrator(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
if self.params.partial_correct:
integrated = predictions.normalize_by_partiality(
integrated,
model,
default_F=self.params.diffBragg.predictions.default_Famplitude,
gain=self.params.diffBragg.refiner.adu_per_photon)
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply:
if abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction, )
elif abs_params.algorithm == "kapton_2019":
from dials.algorithms.integration.kapton_2019_correction import (
multi_kapton_correction, )
experiments, integrated = multi_kapton_correction(
experiments,
integrated,
abs_params.fuller_kapton,
logger=logger)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter, )
sig_filter = SignificanceFilter(self.params)
filtered_refls = sig_filter(experiments, integrated)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
logger.info(
"Removed %d reflections out of %d when applying significance filter",
len(integrated) - len(filtered_refls),
len(integrated),
)
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if len(refls) > 0:
accepted_expts.append(expt)
refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
accepted_refls.extend(refls)
else:
logger.info(
"Removed experiment %d which has no reflections left after applying significance filter",
expt_id,
)
if len(accepted_refls) == 0:
raise Sorry(
"No reflections left after applying significance filter")
experiments = accepted_expts
integrated = accepted_refls
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated:
del integrated["shoebox"]
if self.params.output.composite_output:
if (self.params.output.integrated_experiments_filename
or self.params.output.integrated_filename):
assert (self.params.output.integrated_experiments_filename
is not None
and self.params.output.integrated_filename is not None)
n = len(self.all_integrated_experiments)
self.all_integrated_experiments.extend(experiments)
for i, experiment in enumerate(experiments):
refls = integrated.select(integrated["id"] == i)
refls["id"] = flex.int(len(refls), n)
del refls.experiment_identifiers()[i]
refls.experiment_identifiers()[n] = experiment.identifier
self.all_integrated_reflections.extend(refls)
n += 1
else:
# Dump experiments to disk
if self.params.output.integrated_experiments_filename:
experiments.as_json(
self.params.output.integrated_experiments_filename)
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated,
self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import (
calc_2D_rmsd_and_displacements, )
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n" % rmsd_indexed
for i in range(6):
bright_integrated = integrated.select(
(integrated["intensity.sum.value"] /
flex.sqrt(integrated["intensity.sum.variance"])) >= i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(
bright_integrated)
else:
rmsd_integrated = 0
log_str += (
"N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"
% (i, len(bright_integrated), rmsd_integrated))
for crystal_model in experiments.crystals():
if hasattr(crystal_model, "get_domain_size_ang"):
log_str += ". Final ML model: domain size angstroms: {:f}, half mosaicity degrees: {:f}".format(
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg(),
)
logger.info(log_str)
logger.info("")
logger.info("Time Taken = %f seconds", time.time() - st)
return integrated
def integrate(self, experiments, indexed):
from time import time
st = time()
logger.info('*' * 80)
logger.info('Integrating Reflections')
logger.info('*' * 80)
indexed, _ = self.process_reference(indexed)
# Get the integrator from the input parameters
logger.info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments,
indexed)
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
logger.info("")
integrator = IntegratorFactory.create(self.params, experiments,
predicted)
# Integrate the reflections
integrated = integrator.integrate()
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import multi_kapton_correction
experiments, integrated = multi_kapton_correction(
experiments,
integrated,
abs_params.fuller_kapton,
logger=logger)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import SignificanceFilter
sig_filter = SignificanceFilter(self.params)
refls = sig_filter(experiments, integrated)
logger.info(
"Removed %d reflections out of %d when applying significance filter"
% (len(integrated) - len(refls), len(integrated)))
if len(refls) == 0:
raise Sorry(
"No reflections left after applying significance filter")
integrated = refls
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and 'shoebox' in integrated:
del integrated['shoebox']
if self.params.output.composite_output:
if self.params.output.integrated_experiments_filename or self.params.output.integrated_filename:
assert self.params.output.integrated_experiments_filename is not None and self.params.output.integrated_filename is not None
from dials.array_family import flex
n = len(self.all_integrated_experiments)
self.all_integrated_experiments.extend(experiments)
for i, experiment in enumerate(experiments):
refls = integrated.select(integrated['id'] == i)
refls['id'] = flex.int(len(refls), n)
self.all_integrated_reflections.extend(refls)
n += 1
else:
# Dump experiments to disk
if self.params.output.integrated_experiments_filename:
from dxtbx.model.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(
self.params.output.integrated_experiments_filename)
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated,
self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n" % (rmsd_indexed)
for i in xrange(6):
bright_integrated = integrated.select(
(integrated['intensity.sum.value'] /
flex.sqrt(integrated['intensity.sum.variance'])) >= i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(
bright_integrated)
else:
rmsd_integrated = 0
log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n" % (
i, len(bright_integrated), rmsd_integrated)
for crystal_model in experiments.crystals():
if hasattr(crystal_model, 'get_domain_size_ang'):
log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f" % (
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg())
logger.info(log_str)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return integrated
def integrate(self, experiments, indexed):
# TODO: Figure out if this is necessary and/or how to do this better
indexed, _ = self.process_reference(indexed)
if self.params.integration.integration_only_overrides.trusted_range:
for detector in experiments.detectors():
for panel in detector:
panel.set_trusted_range(
self.params.integration.integration_only_overrides.trusted_range
)
# Get the integrator from the input parameters
from dials.algorithms.integration.integrator import create_integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
new_experiments = ExperimentList()
new_reflections = flex.reflection_table()
for expt_id, expt in enumerate(experiments):
if (
self.params.profile.gaussian_rs.parameters.sigma_b_cutoff is None
or expt.profile.sigma_b()
< self.params.profile.gaussian_rs.parameters.sigma_b_cutoff
):
refls = indexed.select(indexed["id"] == expt_id)
refls["id"] = flex.int(len(refls), len(new_experiments))
# refls.reset_ids()
del refls.experiment_identifiers()[expt_id]
refls.experiment_identifiers()[len(new_experiments)] = expt.identifier
new_reflections.extend(refls)
new_experiments.append(expt)
else:
# TODO: this can be done better, also
print(
"Rejected expt %d with sigma_b %f"
% (expt_id, expt.profile.sigma_b())
)
experiments = new_experiments
indexed = new_reflections
if len(experiments) == 0:
raise RuntimeError("No experiments after filtering by sigma_b")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static,
)
predicted.match_with_reference(indexed)
integrator = create_integrator(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply:
if abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction,
)
elif abs_params.algorithm == "kapton_2019":
from dials.algorithms.integration.kapton_2019_correction import (
multi_kapton_correction,
)
experiments, integrated = multi_kapton_correction(
experiments, integrated, abs_params.fuller_kapton, logger=logger
)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter,
)
sig_filter = SignificanceFilter(self.params)
filtered_refls = sig_filter(experiments, integrated)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if len(refls) > 0:
accepted_expts.append(expt)
refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
accepted_refls.extend(refls)
else:
print(
"Removed experiment %d which has no reflections left after applying significance filter",
expt_id,
)
if len(accepted_refls) == 0:
raise Sorry("No reflections left after applying significance filter")
experiments = accepted_expts
integrated = accepted_refls
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and "shoebox" in integrated:
del integrated["shoebox"]
# Dump experiments to disk
if self.params.output.integrated_experiments_filename:
experiments.as_json(self.params.output.integrated_experiments_filename)
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(
integrated, self.params.output.integrated_filename
)
self.write_integration_pickles(integrated, experiments)
# TODO: Figure out what this is
from dials.algorithms.indexing.stills_indexer import (
calc_2D_rmsd_and_displacements,
)
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = f"RMSD indexed (px): {rmsd_indexed:f}\n"
for i in range(6):
bright_integrated = integrated.select(
(
integrated["intensity.sum.value"]
/ flex.sqrt(integrated["intensity.sum.variance"])
)
>= i
)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += (
"N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"
% (i, len(bright_integrated), rmsd_integrated)
)
for crystal_model in experiments.crystals():
if hasattr(crystal_model, "get_domain_size_ang"):
log_str += ". Final ML model: domain size angstroms: {:f}, half mosaicity degrees: {:f}".format(
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg(),
)
print(log_str)
return integrated
