class ImageIntegrator(object):
'''
A class that does integration directly on the image skipping the shoebox
creation step.
'''
def __init__(self, experiments, reflections, params):
'''
Initialize the integrator
:param experiments: The experiment list
:param reflections: The reflections to process
:param params: The parameters to use
'''
# Check all reflections have same imageset and get it
imageset = experiments[0].imageset
for expr in experiments:
assert expr.imageset == imageset, "All experiments must share and imageset"
# Save some stuff
self.experiments = experiments
self.reflections = reflections
self.params = params
self.profile_model_report = None
self.integration_report = None
def integrate(self):
'''
Integrate the data
'''
from dials.algorithms.integration.report import IntegrationReport
from dials.util.command_line import heading
# Init the report
self.profile_model_report = None
self.integration_report = None
# Heading
logger.info("=" * 80)
logger.info("")
logger.info(heading("Processing reflections"))
logger.info("")
# Create summary format
fmt = (' Processing the following experiments:\n'
'\n'
' Experiments: %d\n'
' Beams: %d\n'
' Detectors: %d\n'
' Goniometers: %d\n'
' Scans: %d\n'
' Crystals: %d\n'
' Imagesets: %d\n')
# Print the summary
logger.info(
fmt %
(len(self.experiments), len(
self.experiments.beams()), len(self.experiments.detectors()),
len(self.experiments.goniometers()), len(
self.experiments.scans()), len(self.experiments.crystals()),
len(self.experiments.imagesets())))
# Print a heading
logger.info("=" * 80)
logger.info("")
logger.info(heading("Integrating reflections"))
logger.info("")
# Initialise the processing
initialize = InitializerRot(self.experiments, self.params)
initialize(self.reflections)
# Construvt the image integrator processor
processor = ProcessorImage(self.experiments, self.reflections,
self.params)
processor.executor = ImageIntegratorExecutor()
# Do the processing
self.reflections, time_info = processor.process()
# Finalise the processing
finalize = FinalizerRot(self.experiments, self.params)
finalize(self.reflections)
# Create the integration report
self.integration_report = IntegrationReport(self.experiments,
self.reflections)
logger.info("")
logger.info(self.integration_report.as_str(prefix=' '))
# Print the time info
logger.info(str(time_info))
logger.info("")
# Return the reflections
return self.reflections
class Integrator(object):
'''
The integrator class
'''
def __init__(self, experiments, reflections, params):
'''
Initialize the integrator
:param experiments: The experiment list
:param reflections: The reflections to process
:param params: The parameters to use
'''
# Save some stuff
self.experiments = experiments
self.reflections = reflections
self.params = params
self.profile_model_report = None
self.integration_report = None
def integrate(self):
'''
Integrate the data
'''
from dials.algorithms.integration.report import IntegrationReport
from dials.algorithms.integration.report import ProfileModelReport
from dials.algorithms.integration.report import ProfileValidationReport
from dials.util.command_line import heading
from dials.util import pprint
from random import shuffle, seed
from math import floor, ceil
from dials.array_family import flex
from dials.algorithms.profile_model.modeller import MultiExpProfileModeller
from dials.algorithms.integration.validation import ValidatedMultiExpProfileModeller
# Ensure we get the same random sample each time
seed(0)
# Init the report
self.profile_model_report = None
self.integration_report = None
# Heading
logger.info("=" * 80)
logger.info("")
logger.info(heading("Processing reflections"))
logger.info("")
# Create summary format
fmt = (' Processing the following experiments:\n'
'\n'
' Experiments: %d\n'
' Beams: %d\n'
' Detectors: %d\n'
' Goniometers: %d\n'
' Scans: %d\n'
' Crystals: %d\n'
' Imagesets: %d\n')
# Print the summary
logger.info(
fmt %
(len(self.experiments), len(
self.experiments.beams()), len(self.experiments.detectors()),
len(self.experiments.goniometers()), len(
self.experiments.scans()), len(self.experiments.crystals()),
len(self.experiments.imagesets())))
# Initialize the reflections
initialize = self.InitializerClass(self.experiments, self.params)
initialize(self.reflections)
# Check if we want to do some profile fitting
fitting_class = [e.profile.fitting_class() for e in self.experiments]
fitting_avail = all(c is not None for c in fitting_class)
if self.params.profile.fitting and fitting_avail:
profile_fitting = True
profile_fitter = None
else:
profile_fitting = False
profile_fitter = None
# Do profile modelling
if profile_fitting:
logger.info("=" * 80)
logger.info("")
logger.info(heading("Modelling reflection profiles"))
logger.info("")
# Get the selection
selection = self.reflections.get_flags(
self.reflections.flags.reference_spot)
# Get the reference spots
reference = self.reflections.select(selection)
# Check if we need to skip
if len(reference) == 0:
logger.info(
"** Skipping profile modelling - no reference profiles given **"
)
else:
# Try to set up the validation
if self.params.profile.validation.number_of_partitions > 1:
n = len(reference)
k_max = int(
floor(
n /
self.params.profile.validation.min_partition_size))
if k_max < self.params.profile.validation.number_of_partitions:
num_folds = k_max
else:
num_folds = self.params.profile.validation.number_of_partitions
if num_folds > 1:
indices = (list(range(num_folds)) *
int(ceil(n / num_folds)))[0:n]
shuffle(indices)
reference['profile.index'] = flex.size_t(indices)
if num_folds < 1:
num_folds = 1
else:
num_folds = 1
# Create the profile fitter
profile_fitter = ValidatedMultiExpProfileModeller()
for i in range(num_folds):
profile_fitter_single = MultiExpProfileModeller(
) #(num_folds)
for expr in self.experiments:
profile_fitter_single.add(
expr.profile.fitting_class()(expr))
profile_fitter.add(profile_fitter_single)
# Create the data processor
executor = ProfileModellerExecutor(self.experiments,
profile_fitter)
processor = ProcessorBuilder(self.ProcessorClass,
self.experiments, reference,
self.params.modelling).build()
processor.executor = executor
# Process the reference profiles
reference, profile_fitter_list, time_info = processor.process()
# Set the reference spots info
#self.reflections.set_selected(selection, reference)
# Finalize the profile models for validation
assert len(profile_fitter_list) > 0, "No profile fitters"
profile_fitter = None
for index, pf in profile_fitter_list.iteritems():
if pf is None:
continue
if profile_fitter is None:
profile_fitter = pf
else:
profile_fitter.accumulate(pf)
profile_fitter.finalize()
# Get the finalized modeller
finalized_profile_fitter = profile_fitter.finalized_model()
# Print profiles
if self.params.debug_reference_output:
reference_debug = []
for i in range(len(finalized_profile_fitter)):
m = finalized_profile_fitter[i]
p = []
for j in range(len(m)):
try:
p.append(m.data(j))
except Exception:
p.append(None)
reference_debug.append(p)
with open("reference_profiles.pickle", "wb") as outfile:
import cPickle as pickle
pickle.dump(reference_debug, outfile)
for i in range(len(finalized_profile_fitter)):
m = finalized_profile_fitter[i]
logger.debug("")
logger.debug("Profiles for experiment %d" % i)
for j in range(len(m)):
logger.debug("Profile %d" % j)
try:
logger.debug(pprint.profile3d(m.data(j)))
except Exception:
logger.debug("** NO PROFILE **")
# Print the modeller report
self.profile_model_report = ProfileModelReport(
self.experiments, finalized_profile_fitter, reference)
logger.info("")
logger.info(self.profile_model_report.as_str(prefix=' '))
# Print the time info
logger.info("")
logger.info(str(time_info))
logger.info("")
# If we have more than 1 fold then do the validation
if num_folds > 1:
# Create the data processor
executor = ProfileValidatorExecutor(
self.experiments, profile_fitter)
processor = ProcessorBuilder(
self.ProcessorClass, self.experiments, reference,
self.params.modelling).build()
processor.executor = executor
# Process the reference profiles
reference, validation, time_info = processor.process()
# Print the modeller report
self.profile_validation_report = ProfileValidationReport(
self.experiments, profile_fitter, reference, num_folds)
logger.info("")
logger.info(
self.profile_validation_report.as_str(prefix=' '))
# Print the time info
logger.info("")
logger.info(str(time_info))
logger.info("")
# Set to the finalized fitter
profile_fitter = finalized_profile_fitter
logger.info("=" * 80)
logger.info("")
logger.info(heading("Integrating reflections"))
logger.info("")
# Create the data processor
executor = IntegratorExecutor(self.experiments, profile_fitter)
processor = ProcessorBuilder(self.ProcessorClass, self.experiments,
self.reflections,
self.params.integration).build()
processor.executor = executor
# Process the reflections
self.reflections, _, time_info = processor.process()
# Finalize the reflections
finalize = self.FinalizerClass(self.experiments, self.params)
finalize(self.reflections)
# Create the integration report
self.integration_report = IntegrationReport(self.experiments,
self.reflections)
logger.info("")
logger.info(self.integration_report.as_str(prefix=' '))
# Print the time info
logger.info(str(time_info))
logger.info("")
# Return the reflections
return self.reflections
def report(self):
'''
Return the report of the processing
'''
from dials.util.report import Report
result = Report()
if self.profile_model_report is not None:
result.combine(self.profile_model_report)
result.combine(self.integration_report)
return result
def summary(self, block_size, block_size_units):
''' Print a summary of the integration stuff. '''
from libtbx.table_utils import format as table
from dials.util.command_line import heading
# Compute the task table
if self._experiments.all_stills():
rows = [["#", "Group", "Frame From", "Frame To"]]
for i in range(len(self)):
job = self._manager.job(i)
group = job.index()
f0, f1 = job.frames()
rows.append([str(i), str(group), str(f0), str(f1)])
elif self._experiments.all_sweeps():
rows = [[
"#", "Group", "Frame From", "Frame To", "Angle From",
"Angle To"
]]
for i in range(len(self)):
job = self._manager.job(i)
group = job.index()
expr = job.expr()
f0, f1 = job.frames()
scan = self._experiments[expr[0]].scan
p0 = scan.get_angle_from_array_index(f0)
p1 = scan.get_angle_from_array_index(f1)
rows.append(
[str(i),
str(group),
str(f0),
str(f1),
str(p0),
str(p1)])
else:
raise RuntimeError('Experiments must be all sweeps or all stills')
task_table = table(rows, has_header=True, justify="right", prefix=" ")
class Integrator(object):
'''
The integrator class
'''
def __init__(self,
experiments,
reflections,
params):
'''
Initialize the integrator
:param experiments: The experiment list
:param reflections: The reflections to process
:param params: The parameters to use
'''
# Save some stuff
self.experiments = experiments
self.reflections = reflections
self.params = params
self.profile_model_report = None
self.integration_report = None
def integrate(self):
'''
Integrate the data
'''
from dials.algorithms.integration.report import IntegrationReport
from dials.algorithms.integration.report import ProfileModelReport
from dials.algorithms.integration.report import ProfileValidationReport
from dials.util.command_line import heading
from logging import info, debug
from dials.util import pprint
from random import shuffle, seed
from math import floor, ceil
from dials.array_family import flex
from dials.algorithms.profile_model.modeller import MultiExpProfileModeller
from dials.algorithms.integration.validation import ValidatedMultiExpProfileModeller
# Ensure we get the same random sample each time
seed(0)
# Init the report
self.profile_model_report = None
self.integration_report = None
# Heading
info("=" * 80)
info("")
info(heading("Processing reflections"))
info("")
# Create summary format
fmt = (
' Processing the following experiments:\n'
'\n'
' Experiments: %d\n'
' Beams: %d\n'
' Detectors: %d\n'
' Goniometers: %d\n'
' Scans: %d\n'
' Crystals: %d\n'
' Imagesets: %d\n'
)
# Print the summary
info(fmt % (
len(self.experiments),
len(self.experiments.beams()),
len(self.experiments.detectors()),
len(self.experiments.goniometers()),
len(self.experiments.scans()),
len(self.experiments.crystals()),
len(self.experiments.imagesets())))
# Initialize the reflections
initialize = self.InitializerClass(
self.experiments,
self.params)
initialize(self.reflections)
# Check if we want to do some profile fitting
fitting_class = [e.profile.fitting_class() for e in self.experiments]
fitting_avail = all([c is not None for c in fitting_class])
if self.params.profile.fitting and fitting_avail:
profile_fitting = True
profile_fitter = None
else:
profile_fitting = False
profile_fitter = None
# Do profile modelling
if profile_fitting:
info("=" * 80)
info("")
info(heading("Modelling reflection profiles"))
info("")
# Get the selection
selection = self.reflections.get_flags(
self.reflections.flags.reference_spot)
# Get the reference spots
reference = self.reflections.select(selection)
# Check if we need to skip
if len(reference) == 0:
info("** Skipping profile modelling - no reference profiles given **")
else:
# Try to set up the validation
if self.params.profile.validation.number_of_partitions > 1:
n = len(reference)
k_max = int(floor(n / self.params.profile.validation.min_partition_size))
if k_max < self.params.profile.validation.number_of_partitions:
num_folds = k_max
else:
num_folds = self.params.profile.validation.number_of_partitions
if num_folds > 1:
indices = (list(range(num_folds)) * int(ceil(n/num_folds)))[0:n]
shuffle(indices)
reference['profile.index'] = flex.size_t(indices)
if num_folds < 1:
num_folds = 1
else:
num_folds = 1
# Create the profile fitter
profile_fitter = ValidatedMultiExpProfileModeller()
for i in range(num_folds):
profile_fitter_single = MultiExpProfileModeller()#(num_folds)
for expr in self.experiments:
profile_fitter_single.add(expr.profile.fitting_class()(expr))
profile_fitter.add(profile_fitter_single)
# Create the data processor
executor = ProfileModellerExecutor(
self.experiments,
profile_fitter)
processor = ProcessorBuilder(
self.ProcessorClass,
self.experiments,
reference,
self.params.modelling).build()
processor.executor = executor
# Process the reference profiles
reference, profile_fitter_list, time_info = processor.process()
# Set the reference spots info
#self.reflections.set_selected(selection, reference)
# Finalize the profile models for validation
assert len(profile_fitter_list) > 0, "No profile fitters"
profile_fitter = None
for index, pf in profile_fitter_list.iteritems():
if pf is None:
continue
if profile_fitter is None:
profile_fitter = pf
else:
profile_fitter.accumulate(pf)
profile_fitter.finalize()
# Get the finalized modeller
finalized_profile_fitter = profile_fitter.finalized_model()
# Print profiles
if self.params.debug_reference_output:
reference_debug = []
for i in range(len(finalized_profile_fitter)):
m = finalized_profile_fitter[i]
p = []
for j in range(len(m)):
try:
p.append(m.data(j))
except Exception:
p.append(None)
reference_debug.append(p)
with open("reference_profiles.pickle", "wb") as outfile:
import cPickle as pickle
pickle.dump(reference_debug, outfile)
for i in range(len(finalized_profile_fitter)):
m = finalized_profile_fitter[i]
debug("")
debug("Profiles for experiment %d" % i)
for j in range(len(m)):
debug("Profile %d" % j)
try:
debug(pprint.profile3d(m.data(j)))
except Exception:
debug("** NO PROFILE **")
# Print the modeller report
self.profile_model_report = ProfileModelReport(
self.experiments,
finalized_profile_fitter,
reference)
info("")
info(self.profile_model_report.as_str(prefix=' '))
# Print the time info
info("")
info(str(time_info))
info("")
# If we have more than 1 fold then do the validation
if num_folds > 1:
# Create the data processor
executor = ProfileValidatorExecutor(
self.experiments,
profile_fitter)
processor = ProcessorBuilder(
self.ProcessorClass,
self.experiments,
reference,
self.params.modelling).build()
processor.executor = executor
# Process the reference profiles
reference, validation, time_info = processor.process()
# Print the modeller report
self.profile_validation_report = ProfileValidationReport(
self.experiments,
profile_fitter,
reference,
num_folds)
info("")
info(self.profile_validation_report.as_str(prefix=' '))
# Print the time info
info("")
info(str(time_info))
info("")
# Set to the finalized fitter
profile_fitter = finalized_profile_fitter
info("=" * 80)
info("")
info(heading("Integrating reflections"))
info("")
# Create the data processor
executor = IntegratorExecutor(
self.experiments,
profile_fitter)
processor = ProcessorBuilder(
self.ProcessorClass,
self.experiments,
self.reflections,
self.params.integration).build()
processor.executor = executor
# Process the reflections
self.reflections, _, time_info = processor.process()
# Finalize the reflections
finalize = self.FinalizerClass(
self.experiments,
self.params)
finalize(self.reflections)
# Create the integration report
self.integration_report = IntegrationReport(
self.experiments,
self.reflections)
info("")
info(self.integration_report.as_str(prefix=' '))
# Print the time info
info(str(time_info))
info("")
# Return the reflections
return self.reflections
def report(self):
'''
Return the report of the processing
'''
from dials.util.report import Report
result = Report()
if self.profile_model_report is not None:
result.combine(self.profile_model_report)
result.combine(self.integration_report)
return result
def summary(self, block_size, block_size_units):
''' Print a summary of the integration stuff. '''
from libtbx.table_utils import format as table
from dials.util.command_line import heading
from logging import info
# Compute the task table
if self._experiments.all_stills():
rows = [["#", "Group", "Frame From", "Frame To"]]
for i in range(len(self)):
job = self._manager.job(i)
group = job.index()
f0, f1 = job.frames()
rows.append([str(i), str(group), str(f0), str(f1)])
elif self._experiments.all_sweeps():
rows = [["#", "Group", "Frame From", "Frame To", "Angle From", "Angle To"]]
for i in range(len(self)):
job = self._manager.job(i)
group = job.index()
expr = job.expr()
f0, f1 = job.frames()
scan = self._experiments[expr[0]].scan
p0 = scan.get_angle_from_array_index(f0)
p1 = scan.get_angle_from_array_index(f1)
rows.append([str(i), str(group), str(f0), str(f1), str(p0), str(p1)])
else:
raise RuntimeError('Experiments must be all sweeps or all stills')
task_table = table(rows, has_header=True, justify="right", prefix=" ")
class ImageIntegrator(object):
'''
A class that does integration directly on the image skipping the shoebox
creation step.
'''
def __init__(self,
experiments,
reflections,
params):
'''
Initialize the integrator
:param experiments: The experiment list
:param reflections: The reflections to process
:param params: The parameters to use
'''
# Check all reflections have same imageset and get it
imageset = experiments[0].imageset
for expr in experiments:
assert expr.imageset == imageset, "All experiments must share and imageset"
# Save some stuff
self.experiments = experiments
self.reflections = reflections
self.params = params
self.profile_model_report = None
self.integration_report = None
def integrate(self):
'''
Integrate the data
'''
from dials.algorithms.integration.report import IntegrationReport
from dials.util.command_line import heading
# Init the report
self.profile_model_report = None
self.integration_report = None
# Heading
logger.info("=" * 80)
logger.info("")
logger.info(heading("Processing reflections"))
logger.info("")
# Create summary format
fmt = (
' Processing the following experiments:\n'
'\n'
' Experiments: %d\n'
' Beams: %d\n'
' Detectors: %d\n'
' Goniometers: %d\n'
' Scans: %d\n'
' Crystals: %d\n'
' Imagesets: %d\n'
)
# Print the summary
logger.info(fmt % (
len(self.experiments),
len(self.experiments.beams()),
len(self.experiments.detectors()),
len(self.experiments.goniometers()),
len(self.experiments.scans()),
len(self.experiments.crystals()),
len(self.experiments.imagesets())))
# Print a heading
logger.info("=" * 80)
logger.info("")
logger.info(heading("Integrating reflections"))
logger.info("")
# Initialise the processing
initialize = InitializerRot(
self.experiments,
self.params)
initialize(self.reflections)
# Construvt the image integrator processor
processor = ProcessorImage(
self.experiments,
self.reflections,
self.params)
processor.executor = ImageIntegratorExecutor()
# Do the processing
self.reflections, time_info = processor.process()
# Finalise the processing
finalize = FinalizerRot(
self.experiments,
self.params)
finalize(self.reflections)
# Create the integration report
self.integration_report = IntegrationReport(
self.experiments,
self.reflections)
logger.info("")
logger.info(self.integration_report.as_str(prefix=' '))
# Print the time info
logger.info(str(time_info))
logger.info("")
# Return the reflections
return self.reflections