def integrate(self, integrator_type):
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.algorithms.integration.integrator import phil_scope as master_phil_scope
from libtbx.phil import parse
import sys
import StringIO
rlist = self.rlist.copy()
output = StringIO.StringIO()
stdout = sys.stdout
sys.stdout = output
try:
phil_scope = parse('''
integration.background.algorithm=null
integration.intensity.algorithm=sum
integration.intensity.sum.integrator=%s
integration.block.size=0.5
integration.profile_fitting=False
''' % integrator_type)
params = master_phil_scope.fetch(source=phil_scope).extract()
integrator = IntegratorFactory.create(params, self.exlist, rlist)
result = integrator.integrate()
except Exception:
print output
raise
sys.stdout = stdout
return result
def integrate(self):
# Process reference reflections
self.indexed, _ = self.process_reference(self.indexed)
# Get integrator from input params
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
# Compute the profile model
self.experiments = ProfileModelFactory.create(self.phil,
self.experiments,
self.indexed)
# Predict the reflections
predicted = flex.reflection_table.from_predictions_multi(
self.experiments,
dmin=self.phil.prediction.d_min,
dmax=self.phil.prediction.d_max,
margin=self.phil.prediction.margin,
force_static=self.phil.prediction.force_static)
# Match the predictions with the reference
predicted.match_with_reference(self.indexed)
# Create the integrator
integrator = IntegratorFactory.create(self.phil, self.experiments,
predicted)
# Integrate the reflections
self.integrated = integrator.integrate()
if self.integrated.has_key('intensity.prf.value'):
method = 'prf' # integration by profile fitting
elif self.integrated.has_key('intensity.sum.value'):
method = 'sum' # integration by simple summation
self.integrated = self.integrated.select(
self.integrated['intensity.' + method + '.variance'] >
0) # keep only spots with sigmas above zero
# Save the reflections if selected
if self.phil.output.integrated_filename:
self.save_reflections(self.integrated,
self.phil.output.integrated_filename)
self.write_integration_pickles()
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed)
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated)
crystal_model = self.experiments.crystals()[0]
def integrate(self):
# Process reference reflections
self.indexed,_ = self.process_reference(self.indexed)
# Get integrator from input params
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
# Compute the profile model
self.experiments = ProfileModelFactory.create(self.phil, self.experiments, self.indexed)
# Predict the reflections
predicted = flex.reflection_table.from_predictions_multi(
self.experiments,
dmin=self.phil.prediction.d_min,
dmax=self.phil.prediction.d_max,
margin=self.phil.prediction.margin,
force_static=self.phil.prediction.force_static)
# Match the predictions with the reference
predicted.match_with_reference(self.indexed)
# Create the integrator
integrator = IntegratorFactory.create(self.phil, self.experiments, predicted)
# Integrate the reflections
self.integrated = integrator.integrate()
if self.integrated.has_key('intensity.prf.value'):
method = 'prf' # integration by profile fitting
elif self.integrated.has_key('intensity.sum.value'):
method = 'sum' # integration by simple summation
self.integrated = self.integrated.select(self.integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero
# Save the reflections if selected
if self.phil.output.integrated_filename:
self.save_reflections(self.integrated, self.phil.output.integrated_filename)
self.write_integration_pickles()
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(self.indexed)
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(self.integrated)
crystal_model = self.experiments.crystals()[0]
def integrate(integrator_type, rlist):
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.algorithms.integration.integrator import phil_scope as master_phil_scope
from libtbx.phil import parse
rlist = rlist.copy()
phil_scope = parse('''
integration.background.algorithm=null
integration.intensity.algorithm=sum
integration.intensity.sum.integrator=%s
integration.block.size=0.5
integration.profile_fitting=False
''' % integrator_type)
params = master_phil_scope.fetch(source=phil_scope).extract()
integrator = IntegratorFactory.create(params, exlist, rlist)
result = integrator.integrate()
return result
def integrate(self, integrator_type):
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.algorithms.integration.integrator import phil_scope as master_phil_scope
from libtbx.phil import parse
import sys
import StringIO
rlist = self.rlist.copy()
output = StringIO.StringIO()
stdout = sys.stdout
sys.stdout = output
try:
phil_scope = parse('''
integration.background.algorithm=null
integration.intensity.algorithm=sum
integration.intensity.sum.integrator=%s
integration.block.size=0.5
integration.profile_fitting=False
''' % integrator_type)
params = master_phil_scope.fetch(source=phil_scope).extract()
integrator = IntegratorFactory.create(
params,
self.exlist,
rlist)
result = integrator.integrate()
except Exception:
print output
raise
sys.stdout = stdout
return result
def work(filename, cl=None):
if cl is None:
cl = []
import libtbx.phil
phil_scope = libtbx.phil.parse('''\
ice_rings {
filter = True
.type = bool
width = 0.004
.type = float(value_min=0.0)
}
index = False
.type = bool
integrate = False
.type = bool
indexing_min_spots = 10
.type = int(value_min=1)
''')
if not os.access(filename, os.R_OK):
raise RuntimeError("Server does not have read access to file %s" %
filename)
interp = phil_scope.command_line_argument_interpreter()
params, unhandled = interp.process_and_fetch(
cl, custom_processor='collect_remaining')
filter_ice = params.extract().ice_rings.filter
ice_rings_width = params.extract().ice_rings.width
index = params.extract().index
integrate = params.extract().integrate
indexing_min_spots = params.extract().indexing_min_spots
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from dxtbx.datablock import DataBlockFactory
from dials.array_family import flex
interp = find_spots_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
logger.info('The following spotfinding parameters have been modified:')
logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str())
params = phil_scope.extract()
# no need to write the hot mask in the server/client
params.spotfinder.write_hot_mask = False
datablock = DataBlockFactory.from_filenames([filename])[0]
t0 = time.time()
reflections = flex.reflection_table.from_observations(datablock, params)
t1 = time.time()
logger.info('Spotfinding took %.2f seconds' % (t1 - t0))
from dials.algorithms.spot_finding import per_image_analysis
imageset = datablock.extract_imagesets()[0]
scan = imageset.get_scan()
if scan is not None:
i = scan.get_array_range()[0]
else:
i = 0
stats = per_image_analysis.stats_single_image(
imageset,
reflections,
i=i,
plot=False,
filter_ice=filter_ice,
ice_rings_width=ice_rings_width)
stats = stats.__dict__
t2 = time.time()
logger.info('Resolution analysis took %.2f seconds' % (t2 - t1))
if index and stats['n_spots_no_ice'] > indexing_min_spots:
import logging
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
from dials.algorithms.indexing import indexer
interp = indexer.master_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
imagesets = [imageset]
logger.info('The following indexing parameters have been modified:')
indexer.master_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
params.indexing.scan_range = []
if (imageset.get_goniometer() is not None
and imageset.get_scan() is not None
and imageset.get_scan().get_oscillation()[1] == 0):
imageset.set_goniometer(None)
imageset.set_scan(None)
try:
idxr = indexer.indexer_base.from_parameters(reflections,
imagesets,
params=params)
indexing_results = []
idxr.index()
indexed_sel = idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.indexed)
indexed_sel &= ~(idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.centroid_outlier))
for i_expt, expt in enumerate(idxr.refined_experiments):
sel = idxr.refined_reflections['id'] == i_expt
sel &= indexed_sel
indexing_results.append({
'crystal':
expt.crystal.to_dict(),
'n_indexed':
sel.count(True),
'fraction_indexed':
sel.count(True) / sel.size()
})
stats['lattices'] = indexing_results
stats['n_indexed'] = indexed_sel.count(True)
stats['fraction_indexed'] = indexed_sel.count(True) / len(
reflections)
except Exception as e:
logger.error(e)
stats['error'] = str(e)
#stats.crystal = None
#stats.n_indexed = None
#stats.fraction_indexed = None
finally:
t3 = time.time()
logger.info('Indexing took %.2f seconds' % (t3 - t2))
if integrate and 'lattices' in stats:
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.command_line.integrate import phil_scope as integrate_phil_scope
interp = integrate_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor='collect_remaining')
imagesets = [imageset]
logger.error(
'The following integration parameters have been modified:')
integrate_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
try:
params.profile.gaussian_rs.min_spots = 0
experiments = idxr.refined_experiments
reference = idxr.refined_reflections
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert (len(matched) == len(predicted))
assert (matched.count(True) <= len(reference))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif matched.count(True) != len(reference):
logger.info('')
logger.info('*' * 80)
logger.info(
'Warning: %d reference spots were not matched to predictions'
% (len(reference) - matched.count(True)))
logger.info('*' * 80)
logger.info('')
# Compute the profile model
experiments = ProfileModelFactory.create(
params, experiments, reference)
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
integrator = IntegratorFactory.create(params, experiments,
predicted)
# Integrate the reflections
reflections = integrator.integrate()
#print len(reflections)
stats['integrated_intensity'] = flex.sum(
reflections['intensity.sum.value'])
except Exception as e:
logger.error(e)
stats['error'] = str(e)
finally:
t4 = time.time()
logger.info('Integration took %.2f seconds' % (t4 - t3))
return stats
def work(filename, cl=None):
if cl is None:
cl = []
phil_scope = libtbx.phil.parse("""\
ice_rings {
filter = True
.type = bool
width = 0.004
.type = float(value_min=0.0)
}
index = False
.type = bool
integrate = False
.type = bool
indexing_min_spots = 10
.type = int(value_min=1)
""")
interp = phil_scope.command_line_argument_interpreter()
params, unhandled = interp.process_and_fetch(
cl, custom_processor="collect_remaining")
filter_ice = params.extract().ice_rings.filter
ice_rings_width = params.extract().ice_rings.width
index = params.extract().index
integrate = params.extract().integrate
indexing_min_spots = params.extract().indexing_min_spots
from dials.command_line.find_spots import phil_scope as find_spots_phil_scope
from dxtbx.model.experiment_list import ExperimentListFactory
from dials.array_family import flex
interp = find_spots_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor="collect_remaining")
logger.info("The following spotfinding parameters have been modified:")
logger.info(find_spots_phil_scope.fetch_diff(source=phil_scope).as_str())
params = phil_scope.extract()
# no need to write the hot mask in the server/client
params.spotfinder.write_hot_mask = False
experiments = ExperimentListFactory.from_filenames([filename])
t0 = time.time()
reflections = flex.reflection_table.from_observations(experiments, params)
t1 = time.time()
logger.info("Spotfinding took %.2f seconds" % (t1 - t0))
from dials.algorithms.spot_finding import per_image_analysis
imageset = experiments.imagesets()[0]
reflections.centroid_px_to_mm(experiments)
reflections.map_centroids_to_reciprocal_space(experiments)
stats = per_image_analysis.stats_for_reflection_table(
reflections, filter_ice=filter_ice,
ice_rings_width=ice_rings_width)._asdict()
t2 = time.time()
logger.info("Resolution analysis took %.2f seconds" % (t2 - t1))
if index and stats["n_spots_no_ice"] > indexing_min_spots:
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
from dials.algorithms.indexing import indexer
from dials.command_line.index import phil_scope as index_phil_scope
interp = index_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor="collect_remaining")
logger.info("The following indexing parameters have been modified:")
index_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
if (imageset.get_goniometer() is not None
and imageset.get_scan() is not None
and imageset.get_scan().is_still()):
imageset.set_goniometer(None)
imageset.set_scan(None)
try:
idxr = indexer.Indexer.from_parameters(reflections,
experiments,
params=params)
indexing_results = []
idxr.index()
indexed_sel = idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.indexed)
indexed_sel &= ~(idxr.refined_reflections.get_flags(
idxr.refined_reflections.flags.centroid_outlier))
for i_expt, expt in enumerate(idxr.refined_experiments):
sel = idxr.refined_reflections["id"] == i_expt
sel &= indexed_sel
indexing_results.append({
"crystal":
expt.crystal.to_dict(),
"n_indexed":
sel.count(True),
"fraction_indexed":
sel.count(True) / sel.size(),
})
stats["lattices"] = indexing_results
stats["n_indexed"] = indexed_sel.count(True)
stats["fraction_indexed"] = indexed_sel.count(True) / len(
reflections)
except Exception as e:
logger.error(e)
stats["error"] = str(e)
finally:
t3 = time.time()
logger.info("Indexing took %.2f seconds" % (t3 - t2))
if integrate and "lattices" in stats:
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.command_line.integrate import phil_scope as integrate_phil_scope
interp = integrate_phil_scope.command_line_argument_interpreter()
phil_scope, unhandled = interp.process_and_fetch(
unhandled, custom_processor="collect_remaining")
logger.error(
"The following integration parameters have been modified:")
integrate_phil_scope.fetch_diff(source=phil_scope).show()
params = phil_scope.extract()
try:
params.profile.gaussian_rs.min_spots = 0
experiments = idxr.refined_experiments
reference = idxr.refined_reflections
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
)
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reference)
if matched.count(True) == 0:
raise Sorry("""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
""")
elif matched.count(True) != len(reference):
logger.info("")
logger.info("*" * 80)
logger.info(
"Warning: %d reference spots were not matched to predictions"
% (len(reference) - matched.count(True)))
logger.info("*" * 80)
logger.info("")
# Compute the profile model
experiments = ProfileModelFactory.create(
params, experiments, reference)
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
integrator = IntegratorFactory.create(params, experiments,
predicted)
# Integrate the reflections
reflections = integrator.integrate()
# print len(reflections)
stats["integrated_intensity"] = flex.sum(
reflections["intensity.sum.value"])
except Exception as e:
logger.error(e)
stats["error"] = str(e)
finally:
t4 = time.time()
logger.info("Integration took %.2f seconds" % (t4 - t3))
return stats
def integrate(self, experiments, indexed):
from time import time
st = time()
logger.info('*' * 80)
logger.info('Integrating Reflections')
logger.info('*' * 80)
indexed,_ = self.process_reference(indexed)
# Get the integrator from the input parameters
logger.info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
logger.info("")
integrator = IntegratorFactory.create(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# Select only those reflections which were integrated
if 'intensity.prf.variance' in integrated:
selection = integrated.get_flags(
integrated.flags.integrated,
all=True)
else:
selection = integrated.get_flags(
integrated.flags.integrated_sum)
integrated = integrated.select(selection)
len_all = len(integrated)
integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels))
print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all)
# verify sigmas are sensible
if 'intensity.prf.value' in integrated:
if (integrated['intensity.prf.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
if 'intensity.sum.value' in integrated:
if (integrated['intensity.sum.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
# apply detector gain to summation variances
integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain
if 'background.sum.value' in integrated:
if (integrated['background.sum.variance'] < 0).count(True) > 0:
raise Sorry("Found negative variances")
if (integrated['background.sum.variance'] == 0).count(True) > 0:
print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True))
integrated = integrated.select(integrated['background.sum.variance'] > 0)
# apply detector gain to background summation variances
integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated, self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed)
for i in xrange(6):
bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated)
crystal_model = experiments.crystals()[0]
if hasattr(crystal_model, '._ML_domain_size_ang'):
log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
logger.info(log_str)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return integrated
def integrate(self, experiments, indexed):
from time import time
from logging import info
st = time()
info('*' * 80)
info('Integrating Reflections')
info('*' * 80)
indexed,_ = self.process_reference(indexed)
# Get the integrator from the input parameters
info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
info("")
info("=" * 80)
info("")
info("Predicting reflections")
info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
info("")
integrator = IntegratorFactory.create(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
if integrated.has_key('intensity.prf.value'):
method = 'prf' # integration by profile fitting
elif integrated.has_key('intensity.sum.value'):
method = 'sum' # integration by simple summation
integrated = integrated.select(integrated['intensity.' + method + '.variance'] > 0) # keep only spots with sigmas above zero
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated, self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(integrated)
crystal_model = experiments.crystals()[0]
print "Integrated. RMSD indexed,", rmsd_indexed, "RMSD integrated", rmsd_integrated, \
"Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
info('')
info('Time Taken = %f seconds' % (time() - st))
return integrated
def run():
parser = OptionParser(
phil = phil_scope)
params, options = parser.parse_args(show_diff_phil=True)
assert params.input.single_img is not None
assert params.output_dir is not None
# load the image
img = dxtbx.load(params.input.single_img)
imgset = MemImageSet([img])
datablock = DataBlockFactory.from_imageset(imgset)[0]
spotfinder = SpotFinderFactory.from_parameters(params)
reflections = spotfinder(datablock)
base_name = os.path.splitext(params.input.single_img)[0]
reflections.as_pickle(os.path.join(params.output_dir, base_name + "_strong.pickle"))
# DGW commented out as reflections.minimum_number_of_reflections no longer exists
#if len(reflections) < params.refinement.reflections.minimum_number_of_reflections:
# print "Not enough spots to index"
# return
# create the spot finder
print "Spotfinder spots found:", len(reflections)
if params.indexing.method == "fft3d":
from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer
elif params.indexing.method == "fft1d":
from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer
elif params.method == "real_space_grid_search":
from dials.algorithms.indexing.real_space_grid_search \
import indexer_real_space_grid_search as indexer
try:
idxr = indexer(reflections, [imgset], params=params.indexing)
except (RuntimeError, Sorry) as e:
print str(e)
return
indexed = idxr.refined_reflections
experiments = idxr.refined_experiments
#from dxtbx.model.experiment.experiment_list import ExperimentListDumper
#dump = ExperimentListDumper(experiments)
#dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json"))
indexed.as_pickle(os.path.join(params.output_dir, base_name + "_indexed.pickle"))
refiner = RefinerFactory.from_parameters_data_experiments(
params, indexed, experiments)
refiner.run()
refined_experiments = refiner.get_experiments()
#dump = ExperimentListDumper(refined_experiments)
#dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json"))
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
reference = indexed
reference = process_reference(reference)
profile_model = ProfileModelFactory.create(params, refined_experiments, reference)
predicted = flex.reflection_table.from_predictions_multi(
refined_experiments,
dmin=params.prediction.dmin,
dmax=params.prediction.dmax,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
predicted.match_with_reference(reference)
integrator = IntegratorFactory.create(params, experiments, profile_model, predicted)
# Integrate the reflections
integrated = integrator.integrate()
integrated.as_pickle(os.path.join(params.output_dir, base_name + "_integrated.pickle"))
print "Predicting Reflections"
rlist = flex.reflection_table.from_predictions(experiments[0])
rlist['id'] = flex.int(len(rlist), 0)
rlist.compute_bbox(experiments, profile_model)
rlist.compute_zeta_multi(experiments)
rlist.compute_d(experiments)
print ""
print "Creating params"
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.algorithms.integration.integrator import phil_scope
from libtbx import phil
user_phil = phil.parse('''
integration {
mp.max_procs = %d
block.size=5
filter.ice_rings.filter=False
intensity.algorithm=sum3d
}
''' % nproc)
working_phil = phil_scope.fetch(source=user_phil)
params = working_phil.extract()
print "Integrating"
integrator = IntegratorFactory.create(params, experiments, profile_model, rlist)
result = integrator.integrate()
result.as_pickle("temp.pickle")
rlist = flex.reflection_table.from_predictions(experiments[0])
rlist["id"] = flex.int(len(rlist), 0)
rlist.compute_bbox(experiments, profile_model)
rlist.compute_zeta_multi(experiments)
rlist.compute_d(experiments)
print("")
print("Creating params")
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.algorithms.integration.integrator import phil_scope
from libtbx import phil
user_phil = phil.parse("""
integration {
mp.max_procs = %d
block.size=5
filter.ice_rings.filter=False
intensity.algorithm=sum3d
}
""" % nproc)
working_phil = phil_scope.fetch(source=user_phil)
params = working_phil.extract()
print("Integrating")
integrator = IntegratorFactory.create(params, experiments, profile_model,
rlist)
result = integrator.integrate()
result.as_pickle("temp.pickle")
elif matched.count(True) != len(reference):
logger.info('')
logger.info('*' * 80)
logger.info('Warning: %d reference spots were not matched to predictions' % (
len(reference) - matched.count(True)))
logger.info('*' * 80)
logger.info('')
# Compute the profile model
experiments = ProfileModelFactory.create(params, experiments, reference)
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
#print len(reflections)
stats['integrated_intensity'] = flex.sum(reflections['intensity.sum.value'])
except Exception, e:
logger.error(e)
stats['error'] = str(e)
finally:
t4 = time.time()
logger.info('Integration took %.2f seconds' %(t4-t3))
return stats
def run():
parser = OptionParser(phil=phil_scope)
params, options = parser.parse_args(show_diff_phil=True)
assert params.input.single_img is not None
assert params.output_dir is not None
# load the image
img = dxtbx.load(params.input.single_img)
imgset = MemImageSet([img])
datablock = DataBlockFactory.from_imageset(imgset)[0]
spotfinder = SpotFinderFactory.from_parameters(params)
reflections = spotfinder(datablock)
base_name = os.path.splitext(params.input.single_img)[0]
reflections.as_pickle(
os.path.join(params.output_dir, base_name + "_strong.pickle"))
# DGW commented out as reflections.minimum_number_of_reflections no longer exists
# if len(reflections) < params.refinement.reflections.minimum_number_of_reflections:
# print "Not enough spots to index"
# return
# create the spot finder
print("Spotfinder spots found:", len(reflections))
if params.indexing.method == "fft3d":
from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer
elif params.indexing.method == "fft1d":
from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer
elif params.method == "real_space_grid_search":
from dials.algorithms.indexing.real_space_grid_search import (
indexer_real_space_grid_search as indexer, )
try:
idxr = indexer(reflections, [imgset], params=params.indexing)
except (RuntimeError, Sorry) as e:
print(str(e))
return
indexed = idxr.refined_reflections
experiments = idxr.refined_experiments
# from dxtbx.model.experiment.experiment_list import ExperimentListDumper
# dump = ExperimentListDumper(experiments)
# dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json"))
indexed.as_pickle(
os.path.join(params.output_dir, base_name + "_indexed.pickle"))
refiner = RefinerFactory.from_parameters_data_experiments(
params, indexed, experiments)
refiner.run()
refined_experiments = refiner.get_experiments()
# dump = ExperimentListDumper(refined_experiments)
# dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json"))
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
reference = indexed
reference = process_reference(reference)
profile_model = ProfileModelFactory.create(params, refined_experiments,
reference)
predicted = flex.reflection_table.from_predictions_multi(
refined_experiments,
dmin=params.prediction.dmin,
dmax=params.prediction.dmax,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
)
predicted.match_with_reference(reference)
integrator = IntegratorFactory.create(params, experiments, profile_model,
predicted)
# Integrate the reflections
integrated = integrator.integrate()
integrated.as_pickle(
os.path.join(params.output_dir, base_name + "_integrated.pickle"))
elif matched.count(True) != len(reference):
logger.info('')
logger.info('*' * 80)
logger.info('Warning: %d reference spots were not matched to predictions' % (
len(reference) - matched.count(True)))
logger.info('*' * 80)
logger.info('')
# Compute the profile model
experiments = ProfileModelFactory.create(params, experiments, reference)
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
#print len(reflections)
stats['integrated_intensity'] = flex.sum(reflections['intensity.sum.value'])
except Exception, e:
logger.error(e)
stats['error'] = str(e)
finally:
t4 = time.time()
logger.info('Integration took %.2f seconds' %(t4-t3))
return stats
def run(self):
""" Perform the integration. """
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from logging import info, debug
from time import time
from libtbx.utils import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry("more than 1 reflection file was given")
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry("no experiment list was specified")
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
# Configure logging
log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log)
from dials.util.version import dials_version
info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not "":
info("The following parameters have been modified:\n")
info(diff_phil)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
info("Using external mask: %s" % mask.filename)
info(" Mask has %d pixels masked" % mask.data.count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
debug("Models for experiment %d" % i)
debug("")
debug(str(exp.beam))
debug(str(exp.detector))
if exp.goniometer:
debug(str(exp.goniometer))
if exp.scan:
debug(str(exp.scan))
debug(str(exp.crystal))
info("=" * 80)
info("")
info(heading("Initialising"))
info("")
# Load the data
reference, rubbish = self.process_reference(reference)
info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(experiments, reference, params.scan_range)
# Predict the reflections
info("")
info("=" * 80)
info("")
info(heading("Predicting reflections"))
info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(reference)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reference)
if matched.count(True) == 0:
raise Sorry(
"""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
"""
)
elif len(unmatched) != 0:
info("")
info("*" * 80)
info("Warning: %d reference spots were not matched to predictions" % (len(unmatched)))
info("*" * 80)
info("")
rubbish.extend(unmatched)
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if reference is not None and params.create_profile_model:
experiments = ProfileModelFactory.create(params, experiments, reference)
else:
for expr in experiments:
expr.profile.params = params.profile
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
info("\nTotal time taken: %f" % (time() - start_time))
def run(self):
''' Perform the integration. '''
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from time import time
from libtbx.utils import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry('more than 1 reflection file was given')
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry('no experiment list was specified')
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
# Configure logging
log.config(
params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info('Using external mask: %s' % mask.filename)
logger.info(' Mask has %d pixels masked' % mask.data.count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
logger.debug("Models for experiment %d" % i)
logger.debug("")
logger.debug(str(exp.beam))
logger.debug(str(exp.detector))
if exp.goniometer:
logger.debug(str(exp.goniometer))
if exp.scan:
logger.debug(str(exp.scan))
logger.debug(str(exp.crystal))
logger.info("=" * 80)
logger.info("")
logger.info(heading("Initialising"))
logger.info("")
# Load the data
reference, rubbish = self.process_reference(reference)
logger.info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(
experiments,
reference,
params.scan_range)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(reference)
assert(len(matched) == len(predicted))
assert(matched.count(True) <= len(reference))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif len(unmatched) != 0:
logger.info('')
logger.info('*' * 80)
logger.info('Warning: %d reference spots were not matched to predictions' % (
len(unmatched)))
logger.info('*' * 80)
logger.info('')
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
from dxtbx.model.experiment.experiment_list import ExperimentList
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
tmp = src.select(src['id'] == eid)
tmp['id'] = flex.int(len(tmp), good_expt_count)
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference['id'] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info("Removing experiment %d: no reference reflections matched to predictions"%expt_id)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if (params.create_profile_model and
reference is not None and
"shoebox" in reference):
experiments = ProfileModelFactory.create(params, experiments, reference)
else:
for expr in experiments:
if expr.profile is None:
raise Sorry('No profile information in experiment list')
expr.profile.params = params.profile
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
logger.info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
logger.info("\nTotal time taken: %f" % (time() - start_time))
def integrate(self, experiments, indexed):
from time import time
st = time()
logger.info('*' * 80)
logger.info('Integrating Reflections')
logger.info('*' * 80)
indexed,_ = self.process_reference(indexed)
# Get the integrator from the input parameters
logger.info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments, indexed)
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
logger.info("")
integrator = IntegratorFactory.create(self.params, experiments, predicted)
# Integrate the reflections
integrated = integrator.integrate()
# Select only those reflections which were integrated
if 'intensity.prf.variance' in integrated:
selection = integrated.get_flags(
integrated.flags.integrated,
all=True)
else:
selection = integrated.get_flags(
integrated.flags.integrated_sum)
integrated = integrated.select(selection)
len_all = len(integrated)
integrated = integrated.select(~integrated.get_flags(integrated.flags.foreground_includes_bad_pixels))
print "Filtering %d reflections with at least one bad foreground pixel out of %d"%(len_all-len(integrated), len_all)
# verify sigmas are sensible
if 'intensity.prf.value' in integrated:
if (integrated['intensity.prf.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
if 'intensity.sum.value' in integrated:
if (integrated['intensity.sum.variance'] <= 0).count(True) > 0:
raise Sorry("Found negative variances")
# apply detector gain to summation variances
integrated['intensity.sum.variance'] *= self.params.integration.summation.detector_gain
if 'background.sum.value' in integrated:
if (integrated['background.sum.variance'] < 0).count(True) > 0:
raise Sorry("Found negative variances")
if (integrated['background.sum.variance'] == 0).count(True) > 0:
print "Filtering %d reflections with zero background variance" % ((integrated['background.sum.variance'] == 0).count(True))
integrated = integrated.select(integrated['background.sum.variance'] > 0)
# apply detector gain to background summation variances
integrated['background.sum.variance'] *= self.params.integration.summation.detector_gain
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import multi_kapton_correction
experiments, integrated = multi_kapton_correction(experiments, integrated,
abs_params.fuller_kapton, logger=logger)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import SignificanceFilter
sig_filter = SignificanceFilter(self.params)
refls = sig_filter(experiments, integrated)
logger.info("Removed %d reflections out of %d when applying significance filter"%(len(integrated)-len(refls), len(integrated)))
if len(refls) == 0:
raise Sorry("No reflections left after applying significance filter")
integrated = refls
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated, self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n"%(rmsd_indexed)
for i in xrange(6):
bright_integrated = integrated.select((integrated['intensity.sum.value']/flex.sqrt(integrated['intensity.sum.variance']))>=i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(bright_integrated)
else:
rmsd_integrated = 0
log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n"%(i, len(bright_integrated), rmsd_integrated)
for crystal_model in experiments.crystals():
if hasattr(crystal_model, '_ML_domain_size_ang'):
log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f"%(crystal_model._ML_domain_size_ang, crystal_model._ML_half_mosaicity_deg)
logger.info(log_str)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return integrated
def integrate(self, experiments, indexed):
from time import time
st = time()
logger.info('*' * 80)
logger.info('Integrating Reflections')
logger.info('*' * 80)
indexed, _ = self.process_reference(indexed)
# Get the integrator from the input parameters
logger.info('Configuring integrator from input parameters')
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Compute the profile model
# Predict the reflections
# Match the predictions with the reference
# Create the integrator
experiments = ProfileModelFactory.create(self.params, experiments,
indexed)
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info("Predicting reflections")
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=self.params.prediction.d_min,
dmax=self.params.prediction.d_max,
margin=self.params.prediction.margin,
force_static=self.params.prediction.force_static)
predicted.match_with_reference(indexed)
logger.info("")
integrator = IntegratorFactory.create(self.params, experiments,
predicted)
# Integrate the reflections
integrated = integrator.integrate()
# correct integrated intensities for absorption correction, if necessary
for abs_params in self.params.integration.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import multi_kapton_correction
experiments, integrated = multi_kapton_correction(
experiments,
integrated,
abs_params.fuller_kapton,
logger=logger)()
if self.params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import SignificanceFilter
sig_filter = SignificanceFilter(self.params)
refls = sig_filter(experiments, integrated)
logger.info(
"Removed %d reflections out of %d when applying significance filter"
% (len(integrated) - len(refls), len(integrated)))
if len(refls) == 0:
raise Sorry(
"No reflections left after applying significance filter")
integrated = refls
# Delete the shoeboxes used for intermediate calculations, if requested
if self.params.integration.debug.delete_shoeboxes and 'shoebox' in integrated:
del integrated['shoebox']
if self.params.output.composite_output:
if self.params.output.integrated_experiments_filename or self.params.output.integrated_filename:
assert self.params.output.integrated_experiments_filename is not None and self.params.output.integrated_filename is not None
from dials.array_family import flex
n = len(self.all_integrated_experiments)
self.all_integrated_experiments.extend(experiments)
for i, experiment in enumerate(experiments):
refls = integrated.select(integrated['id'] == i)
refls['id'] = flex.int(len(refls), n)
self.all_integrated_reflections.extend(refls)
n += 1
else:
# Dump experiments to disk
if self.params.output.integrated_experiments_filename:
from dxtbx.model.experiment_list import ExperimentListDumper
dump = ExperimentListDumper(experiments)
dump.as_json(
self.params.output.integrated_experiments_filename)
if self.params.output.integrated_filename:
# Save the reflections
self.save_reflections(integrated,
self.params.output.integrated_filename)
self.write_integration_pickles(integrated, experiments)
from dials.algorithms.indexing.stills_indexer import calc_2D_rmsd_and_displacements
rmsd_indexed, _ = calc_2D_rmsd_and_displacements(indexed)
log_str = "RMSD indexed (px): %f\n" % (rmsd_indexed)
for i in xrange(6):
bright_integrated = integrated.select(
(integrated['intensity.sum.value'] /
flex.sqrt(integrated['intensity.sum.variance'])) >= i)
if len(bright_integrated) > 0:
rmsd_integrated, _ = calc_2D_rmsd_and_displacements(
bright_integrated)
else:
rmsd_integrated = 0
log_str += "N reflections integrated at I/sigI >= %d: % 4d, RMSD (px): %f\n" % (
i, len(bright_integrated), rmsd_integrated)
for crystal_model in experiments.crystals():
if hasattr(crystal_model, 'get_domain_size_ang'):
log_str += ". Final ML model: domain size angstroms: %f, half mosaicity degrees: %f" % (
crystal_model.get_domain_size_ang(),
crystal_model.get_half_mosaicity_deg())
logger.info(log_str)
logger.info('')
logger.info('Time Taken = %f seconds' % (time() - st))
return integrated
def run(self, args=None):
""" Perform the integration. """
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from time import time
from dials.util import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(args=args,
show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry("more than 1 reflection file was given")
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry("no experiment list was specified")
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
if __name__ == "__main__":
# Configure logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil != "":
logger.info("The following parameters have been modified:\n")
logger.info(diff_phil)
for abs_params in params.absorption_correction:
if abs_params.apply:
if not (params.integration.debug.output
and not params.integration.debug.separate_files):
raise Sorry(
"Shoeboxes must be saved to integration intermediates to apply an absorption correction. "
+
"Set integration.debug.output=True, integration.debug.separate_files=False and "
+
"integration.debug.delete_shoeboxes=True to temporarily store shoeboxes."
)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info("Using external mask: %s" % mask.filename)
for tile in mask.data:
logger.info(" Mask has %d pixels masked" %
tile.data().count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
logger.info("=" * 80)
logger.info("")
logger.info("Experiments")
logger.info("")
logger.info("Models for experiment %d" % i)
logger.info("")
logger.info(str(exp.beam))
logger.info(str(exp.detector))
if exp.goniometer:
logger.info(str(exp.goniometer))
if exp.scan:
logger.info(str(exp.scan))
logger.info(str(exp.crystal))
logger.info("=" * 80)
logger.info("")
logger.info(heading("Initialising"))
logger.info("")
# Load the data
reference, rubbish = self.process_reference(reference)
# Check pixels don't belong to neighbours
if reference is not None:
if exp.goniometer is not None and exp.scan is not None:
self.filter_reference_pixels(reference, experiments)
logger.info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(
experiments, reference, params.scan_range)
# Modify experiment list if exclude images is set
experiments = self.exclude_images(experiments, params.exclude_images)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static,
padding=params.prediction.padding,
)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert len(matched) == len(predicted)
assert matched.count(True) <= len(reference)
if matched.count(True) == 0:
raise Sorry("""
Invalid input for reference reflections.
Zero reference spots were matched to predictions
""")
elif len(unmatched) != 0:
logger.info("")
logger.info("*" * 80)
logger.info(
"Warning: %d reference spots were not matched to predictions"
% (len(unmatched)))
logger.info("*" * 80)
logger.info("")
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
from dxtbx.model.experiment_list import ExperimentList
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
tmp = src.select(src["id"] == eid)
tmp["id"] = flex.int(len(tmp), good_expt_count)
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference["id"] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info(
"Removing experiment %d: no reference reflections matched to predictions"
% expt_id)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if (params.create_profile_model and reference is not None
and "shoebox" in reference):
experiments = ProfileModelFactory.create(params, experiments,
reference)
else:
experiments = ProfileModelFactory.create(params, experiments)
for expr in experiments:
if expr.profile is None:
raise Sorry("No profile information in experiment list")
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
logger.info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Correct integrated intensities for absorption correction, if necessary
for abs_params in params.absorption_correction:
if abs_params.apply and abs_params.algorithm == "fuller_kapton":
from dials.algorithms.integration.kapton_correction import (
multi_kapton_correction, )
experiments, reflections = multi_kapton_correction(
experiments,
reflections,
abs_params.fuller_kapton,
logger=logger)()
if params.significance_filter.enable:
from dials.algorithms.integration.stills_significance_filter import (
SignificanceFilter, )
from dxtbx.model.experiment_list import ExperimentList
sig_filter = SignificanceFilter(params)
filtered_refls = sig_filter(experiments, reflections)
accepted_expts = ExperimentList()
accepted_refls = flex.reflection_table()
logger.info(
"Removed %d reflections out of %d when applying significance filter"
% (len(reflections) - len(filtered_refls), len(reflections)))
for expt_id, expt in enumerate(experiments):
refls = filtered_refls.select(filtered_refls["id"] == expt_id)
if len(refls) > 0:
accepted_expts.append(expt)
refls["id"] = flex.int(len(refls), len(accepted_expts) - 1)
accepted_refls.extend(refls)
else:
logger.info(
"Removed experiment %d which has no reflections left after applying significance filter"
% expt_id)
if len(accepted_refls) == 0:
raise Sorry(
"No reflections left after applying significance filter")
experiments = accepted_expts
reflections = accepted_refls
# Delete the shoeboxes used for intermediate calculations, if requested
if params.integration.debug.delete_shoeboxes and "shoebox" in reflections:
del reflections["shoebox"]
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
logger.info("\nTotal time taken: %f" % (time() - start_time))
return experiments, reflections
def run(self):
''' Perform the integration. '''
from dials.util.command_line import heading
from dials.util.options import flatten_reflections, flatten_experiments
from dials.util import log
from time import time
from libtbx.utils import Sorry
# Check the number of arguments is correct
start_time = time()
# Parse the command line
params, options = self.parser.parse_args(show_diff_phil=False)
reference = flatten_reflections(params.input.reflections)
experiments = flatten_experiments(params.input.experiments)
if len(reference) == 0 and len(experiments) == 0:
self.parser.print_help()
return
if len(reference) == 0:
reference = None
elif len(reference) != 1:
raise Sorry('more than 1 reflection file was given')
else:
reference = reference[0]
if len(experiments) == 0:
raise Sorry('no experiment list was specified')
# Save phil parameters
if params.output.phil is not None:
with open(params.output.phil, "w") as outfile:
outfile.write(self.parser.diff_phil.as_str())
# Configure logging
log.config(params.verbosity,
info=params.output.log,
debug=params.output.debug_log)
from dials.util.version import dials_version
logger.info(dials_version())
# Log the diff phil
diff_phil = self.parser.diff_phil.as_str()
if diff_phil is not '':
logger.info('The following parameters have been modified:\n')
logger.info(diff_phil)
# Print if we're using a mask
for i, exp in enumerate(experiments):
mask = exp.imageset.external_lookup.mask
if mask.filename is not None:
if mask.data:
logger.info('Using external mask: %s' % mask.filename)
logger.info(' Mask has %d pixels masked' %
mask.data.count(False))
# Print the experimental models
for i, exp in enumerate(experiments):
logger.debug("Models for experiment %d" % i)
logger.debug("")
logger.debug(str(exp.beam))
logger.debug(str(exp.detector))
if exp.goniometer:
logger.debug(str(exp.goniometer))
if exp.scan:
logger.debug(str(exp.scan))
logger.debug(str(exp.crystal))
logger.info("=" * 80)
logger.info("")
logger.info(heading("Initialising"))
logger.info("")
# Load the data
reference, rubbish = self.process_reference(reference)
logger.info("")
# Initialise the integrator
from dials.algorithms.profile_model.factory import ProfileModelFactory
from dials.algorithms.integration.integrator import IntegratorFactory
from dials.array_family import flex
# Modify experiment list if scan range is set.
experiments, reference = self.split_for_scan_range(
experiments, reference, params.scan_range)
# Predict the reflections
logger.info("")
logger.info("=" * 80)
logger.info("")
logger.info(heading("Predicting reflections"))
logger.info("")
predicted = flex.reflection_table.from_predictions_multi(
experiments,
dmin=params.prediction.d_min,
dmax=params.prediction.d_max,
margin=params.prediction.margin,
force_static=params.prediction.force_static)
# Match reference with predicted
if reference:
matched, reference, unmatched = predicted.match_with_reference(
reference)
assert (len(matched) == len(predicted))
assert (matched.count(True) <= len(reference))
if matched.count(True) == 0:
raise Sorry('''
Invalid input for reference reflections.
Zero reference spots were matched to predictions
''')
elif len(unmatched) != 0:
logger.info('')
logger.info('*' * 80)
logger.info(
'Warning: %d reference spots were not matched to predictions'
% (len(unmatched)))
logger.info('*' * 80)
logger.info('')
rubbish.extend(unmatched)
if len(experiments) > 1:
# filter out any experiments without matched reference reflections
# f_: filtered
from dxtbx.model.experiment.experiment_list import ExperimentList
f_reference = flex.reflection_table()
f_predicted = flex.reflection_table()
f_rubbish = flex.reflection_table()
f_experiments = ExperimentList()
good_expt_count = 0
def refl_extend(src, dest, eid):
tmp = src.select(src['id'] == eid)
tmp['id'] = flex.int(len(tmp), good_expt_count)
dest.extend(tmp)
for expt_id, experiment in enumerate(experiments):
if len(reference.select(reference['id'] == expt_id)) != 0:
refl_extend(reference, f_reference, expt_id)
refl_extend(predicted, f_predicted, expt_id)
refl_extend(rubbish, f_rubbish, expt_id)
f_experiments.append(experiment)
good_expt_count += 1
else:
logger.info(
"Removing experiment %d: no reference reflections matched to predictions"
% expt_id)
reference = f_reference
predicted = f_predicted
experiments = f_experiments
rubbish = f_rubbish
# Select a random sample of the predicted reflections
if not params.sampling.integrate_all_reflections:
predicted = self.sample_predictions(experiments, predicted, params)
# Compute the profile model
if (params.create_profile_model and reference is not None
and "shoebox" in reference):
experiments = ProfileModelFactory.create(params, experiments,
reference)
else:
for expr in experiments:
if expr.profile is None:
raise Sorry('No profile information in experiment list')
expr.profile.params = params.profile
del reference
# Compute the bounding box
predicted.compute_bbox(experiments)
# Create the integrator
logger.info("")
integrator = IntegratorFactory.create(params, experiments, predicted)
# Integrate the reflections
reflections = integrator.integrate()
# Append rubbish data onto the end
if rubbish is not None and params.output.include_bad_reference:
mask = flex.bool(len(rubbish), True)
rubbish.unset_flags(mask, rubbish.flags.integrated_sum)
rubbish.unset_flags(mask, rubbish.flags.integrated_prf)
rubbish.set_flags(mask, rubbish.flags.bad_reference)
reflections.extend(rubbish)
# Save the reflections
self.save_reflections(reflections, params.output.reflections)
self.save_experiments(experiments, params.output.experiments)
# Write a report if requested
if params.output.report is not None:
integrator.report().as_file(params.output.report)
# Print the total time taken
logger.info("\nTotal time taken: %f" % (time() - start_time))
