def run(self):
from dials.algorithms.profile_model.modeller import ProfileModellerIface
from dials.algorithms.profile_model.modeller import MultiExpProfileModeller
from dials.array_family import flex
class Modeller(ProfileModellerIface):
def __init__(self, index, expected):
self.index = index
self.accumulated = False
self.finalized = False
self.expected = expected
super(Modeller, self).__init__()
def model(self, reflections):
assert reflections["id"].all_eq(self.index)
assert len(reflections) == self.expected
def accumulate(self, other):
self.accumulated = True
assert self.index == other.index
def finalize(self):
assert self.accumulated
self.finalized = True
# The expected number of reflections
expected = [100, 200, 300, 400, 500]
# Create some reflections
reflections = flex.reflection_table()
reflections["id"] = flex.int()
for idx in range(len(expected)):
for n in range(expected[idx]):
reflections.append({"id": idx})
# Create two modellers
modeller1 = MultiExpProfileModeller()
modeller2 = MultiExpProfileModeller()
for idx in range(len(expected)):
modeller1.add(Modeller(idx, expected[idx]))
modeller2.add(Modeller(idx, expected[idx]))
# Model the reflections
modeller1.model(reflections)
modeller2.model(reflections)
# Accumulate
modeller1.accumulate(modeller2)
# Finalize
modeller1.finalize()
# Check finalized
assert modeller1.finalized
# Test passed
print "OK"
def integrate(self):
'''
Integrate the data
'''
from dials.algorithms.integration.report import IntegrationReport
from dials.algorithms.integration.report import ProfileModelReport
from dials.algorithms.integration.report import ProfileValidationReport
from dials.util.command_line import heading
from logging import info, debug
from dials.util import pprint
from random import shuffle, seed
from math import floor, ceil
from dials.array_family import flex
from dials.algorithms.profile_model.modeller import MultiExpProfileModeller
from dials.algorithms.integration.validation import ValidatedMultiExpProfileModeller
# Ensure we get the same random sample each time
seed(0)
# Init the report
self.profile_model_report = None
self.integration_report = None
# Heading
info("=" * 80)
info("")
info(heading("Processing reflections"))
info("")
# Create summary format
fmt = (
' Processing the following experiments:\n'
'\n'
' Experiments: %d\n'
' Beams: %d\n'
' Detectors: %d\n'
' Goniometers: %d\n'
' Scans: %d\n'
' Crystals: %d\n'
' Imagesets: %d\n'
)
# Print the summary
info(fmt % (
len(self.experiments),
len(self.experiments.beams()),
len(self.experiments.detectors()),
len(self.experiments.goniometers()),
len(self.experiments.scans()),
len(self.experiments.crystals()),
len(self.experiments.imagesets())))
# Initialize the reflections
initialize = self.InitializerClass(
self.experiments,
self.params)
initialize(self.reflections)
# Check if we want to do some profile fitting
fitting_class = [e.profile.fitting_class() for e in self.experiments]
fitting_avail = all([c is not None for c in fitting_class])
if self.params.profile.fitting and fitting_avail:
profile_fitting = True
profile_fitter = None
else:
profile_fitting = False
profile_fitter = None
# Do profile modelling
if profile_fitting:
info("=" * 80)
info("")
info(heading("Modelling reflection profiles"))
info("")
# Get the selection
selection = self.reflections.get_flags(
self.reflections.flags.reference_spot)
# Get the reference spots
reference = self.reflections.select(selection)
# Check if we need to skip
if len(reference) == 0:
info("** Skipping profile modelling - no reference profiles given **")
else:
# Try to set up the validation
if self.params.profile.validation.number_of_partitions > 1:
n = len(reference)
k_max = int(floor(n / self.params.profile.validation.min_partition_size))
if k_max < self.params.profile.validation.number_of_partitions:
num_folds = k_max
else:
num_folds = self.params.profile.validation.number_of_partitions
if num_folds > 1:
indices = (list(range(num_folds)) * int(ceil(n/num_folds)))[0:n]
shuffle(indices)
reference['profile.index'] = flex.size_t(indices)
if num_folds < 1:
num_folds = 1
else:
num_folds = 1
# Create the profile fitter
profile_fitter = ValidatedMultiExpProfileModeller()
for i in range(num_folds):
profile_fitter_single = MultiExpProfileModeller()#(num_folds)
for expr in self.experiments:
profile_fitter_single.add(expr.profile.fitting_class()(expr))
profile_fitter.add(profile_fitter_single)
# Create the data processor
executor = ProfileModellerExecutor(
self.experiments,
profile_fitter)
processor = ProcessorBuilder(
self.ProcessorClass,
self.experiments,
reference,
self.params.modelling).build()
processor.executor = executor
# Process the reference profiles
reference, profile_fitter_list, time_info = processor.process()
# Set the reference spots info
#self.reflections.set_selected(selection, reference)
# Finalize the profile models for validation
assert len(profile_fitter_list) > 0, "No profile fitters"
profile_fitter = None
for index, pf in profile_fitter_list.iteritems():
if pf is None:
continue
if profile_fitter is None:
profile_fitter = pf
else:
profile_fitter.accumulate(pf)
profile_fitter.finalize()
# Get the finalized modeller
finalized_profile_fitter = profile_fitter.finalized_model()
# Print profiles
if self.params.debug_reference_output:
reference_debug = []
for i in range(len(finalized_profile_fitter)):
m = finalized_profile_fitter[i]
p = []
for j in range(len(m)):
try:
p.append(m.data(j))
except Exception:
p.append(None)
reference_debug.append(p)
with open("reference_profiles.pickle", "wb") as outfile:
import cPickle as pickle
pickle.dump(reference_debug, outfile)
for i in range(len(finalized_profile_fitter)):
m = finalized_profile_fitter[i]
debug("")
debug("Profiles for experiment %d" % i)
for j in range(len(m)):
debug("Profile %d" % j)
try:
debug(pprint.profile3d(m.data(j)))
except Exception:
debug("** NO PROFILE **")
# Print the modeller report
self.profile_model_report = ProfileModelReport(
self.experiments,
finalized_profile_fitter,
reference)
info("")
info(self.profile_model_report.as_str(prefix=' '))
# Print the time info
info("")
info(str(time_info))
info("")
# If we have more than 1 fold then do the validation
if num_folds > 1:
# Create the data processor
executor = ProfileValidatorExecutor(
self.experiments,
profile_fitter)
processor = ProcessorBuilder(
self.ProcessorClass,
self.experiments,
reference,
self.params.modelling).build()
processor.executor = executor
# Process the reference profiles
reference, validation, time_info = processor.process()
# Print the modeller report
self.profile_validation_report = ProfileValidationReport(
self.experiments,
profile_fitter,
reference,
num_folds)
info("")
info(self.profile_validation_report.as_str(prefix=' '))
# Print the time info
info("")
info(str(time_info))
info("")
# Set to the finalized fitter
profile_fitter = finalized_profile_fitter
info("=" * 80)
info("")
info(heading("Integrating reflections"))
info("")
# Create the data processor
executor = IntegratorExecutor(
self.experiments,
profile_fitter)
processor = ProcessorBuilder(
self.ProcessorClass,
self.experiments,
self.reflections,
self.params.integration).build()
processor.executor = executor
# Process the reflections
self.reflections, _, time_info = processor.process()
# Finalize the reflections
finalize = self.FinalizerClass(
self.experiments,
self.params)
finalize(self.reflections)
# Create the integration report
self.integration_report = IntegrationReport(
self.experiments,
self.reflections)
info("")
info(self.integration_report.as_str(prefix=' '))
# Print the time info
info(str(time_info))
info("")
# Return the reflections
return self.reflections
def test():
from dials.algorithms.profile_model.modeller import (
MultiExpProfileModeller,
ProfileModellerIface,
)
from dials.array_family import flex
class Modeller(ProfileModellerIface):
def __init__(self, index, expected):
self.index = index
self.accumulated = False
self.finalized = False
self.expected = expected
super().__init__()
def model(self, reflections):
assert reflections["id"].all_eq(self.index)
assert len(reflections) == self.expected
def accumulate(self, other):
self.accumulated = True
assert self.index == other.index
def finalize(self):
assert self.accumulated
self.finalized = True
# The expected number of reflections
expected = [100, 200, 300, 400, 500]
# Create some reflections
reflections = flex.reflection_table()
reflections["id"] = flex.int()
for idx in range(len(expected)):
for n in range(expected[idx]):
reflections.append({"id": idx})
# Create two modellers
modeller1 = MultiExpProfileModeller()
modeller2 = MultiExpProfileModeller()
for idx in range(len(expected)):
modeller1.add(Modeller(idx, expected[idx]))
modeller2.add(Modeller(idx, expected[idx]))
# Model the reflections
modeller1.model(reflections)
modeller2.model(reflections)
# Accumulate
modeller1.accumulate(modeller2)
# Finalize
modeller1.finalize()
# Check finalized
assert modeller1.finalized
def integrate(self):
'''
Integrate the data
'''
from dials.algorithms.integration.report import IntegrationReport
from dials.algorithms.integration.report import ProfileModelReport
from dials.algorithms.integration.report import ProfileValidationReport
from dials.util.command_line import heading
from dials.util import pprint
from random import shuffle, seed
from math import floor, ceil
from dials.array_family import flex
from dials.algorithms.profile_model.modeller import MultiExpProfileModeller
from dials.algorithms.integration.validation import ValidatedMultiExpProfileModeller
# Ensure we get the same random sample each time
seed(0)
# Init the report
self.profile_model_report = None
self.integration_report = None
# Heading
logger.info("=" * 80)
logger.info("")
logger.info(heading("Processing reflections"))
logger.info("")
# Create summary format
fmt = (' Processing the following experiments:\n'
'\n'
' Experiments: %d\n'
' Beams: %d\n'
' Detectors: %d\n'
' Goniometers: %d\n'
' Scans: %d\n'
' Crystals: %d\n'
' Imagesets: %d\n')
# Print the summary
logger.info(
fmt %
(len(self.experiments), len(
self.experiments.beams()), len(self.experiments.detectors()),
len(self.experiments.goniometers()), len(
self.experiments.scans()), len(self.experiments.crystals()),
len(self.experiments.imagesets())))
# Initialize the reflections
initialize = self.InitializerClass(self.experiments, self.params)
initialize(self.reflections)
# Check if we want to do some profile fitting
fitting_class = [e.profile.fitting_class() for e in self.experiments]
fitting_avail = all(c is not None for c in fitting_class)
if self.params.profile.fitting and fitting_avail:
profile_fitting = True
profile_fitter = None
else:
profile_fitting = False
profile_fitter = None
# Do profile modelling
if profile_fitting:
logger.info("=" * 80)
logger.info("")
logger.info(heading("Modelling reflection profiles"))
logger.info("")
# Get the selection
selection = self.reflections.get_flags(
self.reflections.flags.reference_spot)
# Get the reference spots
reference = self.reflections.select(selection)
# Check if we need to skip
if len(reference) == 0:
logger.info(
"** Skipping profile modelling - no reference profiles given **"
)
else:
# Try to set up the validation
if self.params.profile.validation.number_of_partitions > 1:
n = len(reference)
k_max = int(
floor(
n /
self.params.profile.validation.min_partition_size))
if k_max < self.params.profile.validation.number_of_partitions:
num_folds = k_max
else:
num_folds = self.params.profile.validation.number_of_partitions
if num_folds > 1:
indices = (list(range(num_folds)) *
int(ceil(n / num_folds)))[0:n]
shuffle(indices)
reference['profile.index'] = flex.size_t(indices)
if num_folds < 1:
num_folds = 1
else:
num_folds = 1
# Create the profile fitter
profile_fitter = ValidatedMultiExpProfileModeller()
for i in range(num_folds):
profile_fitter_single = MultiExpProfileModeller(
) #(num_folds)
for expr in self.experiments:
profile_fitter_single.add(
expr.profile.fitting_class()(expr))
profile_fitter.add(profile_fitter_single)
# Create the data processor
executor = ProfileModellerExecutor(self.experiments,
profile_fitter)
processor = ProcessorBuilder(self.ProcessorClass,
self.experiments, reference,
self.params.modelling).build()
processor.executor = executor
# Process the reference profiles
reference, profile_fitter_list, time_info = processor.process()
# Set the reference spots info
#self.reflections.set_selected(selection, reference)
# Finalize the profile models for validation
assert len(profile_fitter_list) > 0, "No profile fitters"
profile_fitter = None
for index, pf in profile_fitter_list.iteritems():
if pf is None:
continue
if profile_fitter is None:
profile_fitter = pf
else:
profile_fitter.accumulate(pf)
profile_fitter.finalize()
# Get the finalized modeller
finalized_profile_fitter = profile_fitter.finalized_model()
# Print profiles
if self.params.debug_reference_output:
reference_debug = []
for i in range(len(finalized_profile_fitter)):
m = finalized_profile_fitter[i]
p = []
for j in range(len(m)):
try:
p.append(m.data(j))
except Exception:
p.append(None)
reference_debug.append(p)
with open("reference_profiles.pickle", "wb") as outfile:
import cPickle as pickle
pickle.dump(reference_debug, outfile)
for i in range(len(finalized_profile_fitter)):
m = finalized_profile_fitter[i]
logger.debug("")
logger.debug("Profiles for experiment %d" % i)
for j in range(len(m)):
logger.debug("Profile %d" % j)
try:
logger.debug(pprint.profile3d(m.data(j)))
except Exception:
logger.debug("** NO PROFILE **")
# Print the modeller report
self.profile_model_report = ProfileModelReport(
self.experiments, finalized_profile_fitter, reference)
logger.info("")
logger.info(self.profile_model_report.as_str(prefix=' '))
# Print the time info
logger.info("")
logger.info(str(time_info))
logger.info("")
# If we have more than 1 fold then do the validation
if num_folds > 1:
# Create the data processor
executor = ProfileValidatorExecutor(
self.experiments, profile_fitter)
processor = ProcessorBuilder(
self.ProcessorClass, self.experiments, reference,
self.params.modelling).build()
processor.executor = executor
# Process the reference profiles
reference, validation, time_info = processor.process()
# Print the modeller report
self.profile_validation_report = ProfileValidationReport(
self.experiments, profile_fitter, reference, num_folds)
logger.info("")
logger.info(
self.profile_validation_report.as_str(prefix=' '))
# Print the time info
logger.info("")
logger.info(str(time_info))
logger.info("")
# Set to the finalized fitter
profile_fitter = finalized_profile_fitter
logger.info("=" * 80)
logger.info("")
logger.info(heading("Integrating reflections"))
logger.info("")
# Create the data processor
executor = IntegratorExecutor(self.experiments, profile_fitter)
processor = ProcessorBuilder(self.ProcessorClass, self.experiments,
self.reflections,
self.params.integration).build()
processor.executor = executor
# Process the reflections
self.reflections, _, time_info = processor.process()
# Finalize the reflections
finalize = self.FinalizerClass(self.experiments, self.params)
finalize(self.reflections)
# Create the integration report
self.integration_report = IntegrationReport(self.experiments,
self.reflections)
logger.info("")
logger.info(self.integration_report.as_str(prefix=' '))
# Print the time info
logger.info(str(time_info))
logger.info("")
# Return the reflections
return self.reflections
