def run(self): from dials.algorithms.profile_model.modeller import ProfileModellerIface from dials.algorithms.profile_model.modeller import MultiExpProfileModeller from dials.array_family import flex class Modeller(ProfileModellerIface): def __init__(self, index, expected): self.index = index self.accumulated = False self.finalized = False self.expected = expected super(Modeller, self).__init__() def model(self, reflections): assert(reflections['id'].all_eq(self.index)) assert(len(reflections) == self.expected) def accumulate(self, other): self.accumulated = True assert(self.index == other.index) def finalize(self): assert(self.accumulated == True) self.finalized = True # The expected number of reflections expected = [100, 200, 300, 400, 500] # Create some reflections reflections = flex.reflection_table() reflections["id"] = flex.int() for idx in range(len(expected)): for n in range(expected[idx]): reflections.append({ "id" : idx }) # Create two modellers modeller1 = MultiExpProfileModeller() modeller2 = MultiExpProfileModeller() for idx in range(len(expected)): modeller1.add(Modeller(idx, expected[idx])) modeller2.add(Modeller(idx, expected[idx])) # Model the reflections modeller1.model(reflections) modeller2.model(reflections) # Accumulate modeller1.accumulate(modeller2) # Finalize modeller1.finalize() # Check finalized assert(modeller1.finalized) # Test passed print 'OK'
def run(self): from dials.algorithms.profile_model.modeller import ProfileModellerIface from dials.algorithms.profile_model.modeller import MultiExpProfileModeller from dials.array_family import flex class Modeller(ProfileModellerIface): def __init__(self, index, expected): self.index = index self.accumulated = False self.finalized = False self.expected = expected super(Modeller, self).__init__() def model(self, reflections): assert (reflections['id'].all_eq(self.index)) assert (len(reflections) == self.expected) def accumulate(self, other): self.accumulated = True assert (self.index == other.index) def finalize(self): assert (self.accumulated) self.finalized = True # The expected number of reflections expected = [100, 200, 300, 400, 500] # Create some reflections reflections = flex.reflection_table() reflections["id"] = flex.int() for idx in range(len(expected)): for n in range(expected[idx]): reflections.append({"id": idx}) # Create two modellers modeller1 = MultiExpProfileModeller() modeller2 = MultiExpProfileModeller() for idx in range(len(expected)): modeller1.add(Modeller(idx, expected[idx])) modeller2.add(Modeller(idx, expected[idx])) # Model the reflections modeller1.model(reflections) modeller2.model(reflections) # Accumulate modeller1.accumulate(modeller2) # Finalize modeller1.finalize() # Check finalized assert (modeller1.finalized) # Test passed print 'OK'
def integrate(self): ''' Integrate the data ''' from dials.algorithms.integration.report import IntegrationReport from dials.algorithms.integration.report import ProfileModelReport from dials.algorithms.integration.report import ProfileValidationReport from dials.util.command_line import heading from logging import info, debug from dials.util import pprint from random import shuffle, seed from math import floor, ceil from dials.array_family import flex from dials.algorithms.profile_model.modeller import MultiExpProfileModeller from dials.algorithms.integration.validation import ValidatedMultiExpProfileModeller # Ensure we get the same random sample each time seed(0) # Init the report self.profile_model_report = None self.integration_report = None # Heading info("=" * 80) info("") info(heading("Processing reflections")) info("") # Create summary format fmt = ( ' Processing the following experiments:\n' '\n' ' Experiments: %d\n' ' Beams: %d\n' ' Detectors: %d\n' ' Goniometers: %d\n' ' Scans: %d\n' ' Crystals: %d\n' ' Imagesets: %d\n' ) # Print the summary info(fmt % ( len(self.experiments), len(self.experiments.beams()), len(self.experiments.detectors()), len(self.experiments.goniometers()), len(self.experiments.scans()), len(self.experiments.crystals()), len(self.experiments.imagesets()))) # Initialize the reflections initialize = self.InitializerClass( self.experiments, self.params) initialize(self.reflections) # Check if we want to do some profile fitting fitting_class = [e.profile.fitting_class() for e in self.experiments] fitting_avail = all([c is not None for c in fitting_class]) if self.params.profile.fitting and fitting_avail: profile_fitting = True profile_fitter = None else: profile_fitting = False profile_fitter = None # Do profile modelling if profile_fitting: info("=" * 80) info("") info(heading("Modelling reflection profiles")) info("") # Get the selection selection = self.reflections.get_flags( self.reflections.flags.reference_spot) # Get the reference spots reference = self.reflections.select(selection) # Check if we need to skip if len(reference) == 0: info("** Skipping profile modelling - no reference profiles given **") else: # Try to set up the validation if self.params.profile.validation.number_of_partitions > 1: n = len(reference) k_max = int(floor(n / self.params.profile.validation.min_partition_size)) if k_max < self.params.profile.validation.number_of_partitions: num_folds = k_max else: num_folds = self.params.profile.validation.number_of_partitions if num_folds > 1: indices = (list(range(num_folds)) * int(ceil(n/num_folds)))[0:n] shuffle(indices) reference['profile.index'] = flex.size_t(indices) if num_folds < 1: num_folds = 1 else: num_folds = 1 # Create the profile fitter profile_fitter = ValidatedMultiExpProfileModeller() for i in range(num_folds): profile_fitter_single = MultiExpProfileModeller()#(num_folds) for expr in self.experiments: profile_fitter_single.add(expr.profile.fitting_class()(expr)) profile_fitter.add(profile_fitter_single) # Create the data processor executor = ProfileModellerExecutor( self.experiments, profile_fitter) processor = ProcessorBuilder( self.ProcessorClass, self.experiments, reference, self.params.modelling).build() processor.executor = executor # Process the reference profiles reference, profile_fitter_list, time_info = processor.process() # Set the reference spots info #self.reflections.set_selected(selection, reference) # Finalize the profile models for validation assert len(profile_fitter_list) > 0, "No profile fitters" profile_fitter = None for index, pf in profile_fitter_list.iteritems(): if pf is None: continue if profile_fitter is None: profile_fitter = pf else: profile_fitter.accumulate(pf) profile_fitter.finalize() # Get the finalized modeller finalized_profile_fitter = profile_fitter.finalized_model() # Print profiles if self.params.debug_reference_output: reference_debug = [] for i in range(len(finalized_profile_fitter)): m = finalized_profile_fitter[i] p = [] for j in range(len(m)): try: p.append(m.data(j)) except Exception: p.append(None) reference_debug.append(p) with open("reference_profiles.pickle", "wb") as outfile: import cPickle as pickle pickle.dump(reference_debug, outfile) for i in range(len(finalized_profile_fitter)): m = finalized_profile_fitter[i] debug("") debug("Profiles for experiment %d" % i) for j in range(len(m)): debug("Profile %d" % j) try: debug(pprint.profile3d(m.data(j))) except Exception: debug("** NO PROFILE **") # Print the modeller report self.profile_model_report = ProfileModelReport( self.experiments, finalized_profile_fitter, reference) info("") info(self.profile_model_report.as_str(prefix=' ')) # Print the time info info("") info(str(time_info)) info("") # If we have more than 1 fold then do the validation if num_folds > 1: # Create the data processor executor = ProfileValidatorExecutor( self.experiments, profile_fitter) processor = ProcessorBuilder( self.ProcessorClass, self.experiments, reference, self.params.modelling).build() processor.executor = executor # Process the reference profiles reference, validation, time_info = processor.process() # Print the modeller report self.profile_validation_report = ProfileValidationReport( self.experiments, profile_fitter, reference, num_folds) info("") info(self.profile_validation_report.as_str(prefix=' ')) # Print the time info info("") info(str(time_info)) info("") # Set to the finalized fitter profile_fitter = finalized_profile_fitter info("=" * 80) info("") info(heading("Integrating reflections")) info("") # Create the data processor executor = IntegratorExecutor( self.experiments, profile_fitter) processor = ProcessorBuilder( self.ProcessorClass, self.experiments, self.reflections, self.params.integration).build() processor.executor = executor # Process the reflections self.reflections, _, time_info = processor.process() # Finalize the reflections finalize = self.FinalizerClass( self.experiments, self.params) finalize(self.reflections) # Create the integration report self.integration_report = IntegrationReport( self.experiments, self.reflections) info("") info(self.integration_report.as_str(prefix=' ')) # Print the time info info(str(time_info)) info("") # Return the reflections return self.reflections
def integrate(self): ''' Integrate the data ''' from dials.algorithms.integration.report import IntegrationReport from dials.algorithms.integration.report import ProfileModelReport from dials.algorithms.integration.report import ProfileValidationReport from dials.util.command_line import heading from dials.util import pprint from random import shuffle, seed from math import floor, ceil from dials.array_family import flex from dials.algorithms.profile_model.modeller import MultiExpProfileModeller from dials.algorithms.integration.validation import ValidatedMultiExpProfileModeller # Ensure we get the same random sample each time seed(0) # Init the report self.profile_model_report = None self.integration_report = None # Heading logger.info("=" * 80) logger.info("") logger.info(heading("Processing reflections")) logger.info("") # Create summary format fmt = (' Processing the following experiments:\n' '\n' ' Experiments: %d\n' ' Beams: %d\n' ' Detectors: %d\n' ' Goniometers: %d\n' ' Scans: %d\n' ' Crystals: %d\n' ' Imagesets: %d\n') # Print the summary logger.info( fmt % (len(self.experiments), len( self.experiments.beams()), len(self.experiments.detectors()), len(self.experiments.goniometers()), len( self.experiments.scans()), len(self.experiments.crystals()), len(self.experiments.imagesets()))) # Initialize the reflections initialize = self.InitializerClass(self.experiments, self.params) initialize(self.reflections) # Check if we want to do some profile fitting fitting_class = [e.profile.fitting_class() for e in self.experiments] fitting_avail = all(c is not None for c in fitting_class) if self.params.profile.fitting and fitting_avail: profile_fitting = True profile_fitter = None else: profile_fitting = False profile_fitter = None # Do profile modelling if profile_fitting: logger.info("=" * 80) logger.info("") logger.info(heading("Modelling reflection profiles")) logger.info("") # Get the selection selection = self.reflections.get_flags( self.reflections.flags.reference_spot) # Get the reference spots reference = self.reflections.select(selection) # Check if we need to skip if len(reference) == 0: logger.info( "** Skipping profile modelling - no reference profiles given **" ) else: # Try to set up the validation if self.params.profile.validation.number_of_partitions > 1: n = len(reference) k_max = int( floor( n / self.params.profile.validation.min_partition_size)) if k_max < self.params.profile.validation.number_of_partitions: num_folds = k_max else: num_folds = self.params.profile.validation.number_of_partitions if num_folds > 1: indices = (list(range(num_folds)) * int(ceil(n / num_folds)))[0:n] shuffle(indices) reference['profile.index'] = flex.size_t(indices) if num_folds < 1: num_folds = 1 else: num_folds = 1 # Create the profile fitter profile_fitter = ValidatedMultiExpProfileModeller() for i in range(num_folds): profile_fitter_single = MultiExpProfileModeller( ) #(num_folds) for expr in self.experiments: profile_fitter_single.add( expr.profile.fitting_class()(expr)) profile_fitter.add(profile_fitter_single) # Create the data processor executor = ProfileModellerExecutor(self.experiments, profile_fitter) processor = ProcessorBuilder(self.ProcessorClass, self.experiments, reference, self.params.modelling).build() processor.executor = executor # Process the reference profiles reference, profile_fitter_list, time_info = processor.process() # Set the reference spots info #self.reflections.set_selected(selection, reference) # Finalize the profile models for validation assert len(profile_fitter_list) > 0, "No profile fitters" profile_fitter = None for index, pf in profile_fitter_list.iteritems(): if pf is None: continue if profile_fitter is None: profile_fitter = pf else: profile_fitter.accumulate(pf) profile_fitter.finalize() # Get the finalized modeller finalized_profile_fitter = profile_fitter.finalized_model() # Print profiles if self.params.debug_reference_output: reference_debug = [] for i in range(len(finalized_profile_fitter)): m = finalized_profile_fitter[i] p = [] for j in range(len(m)): try: p.append(m.data(j)) except Exception: p.append(None) reference_debug.append(p) with open("reference_profiles.pickle", "wb") as outfile: import cPickle as pickle pickle.dump(reference_debug, outfile) for i in range(len(finalized_profile_fitter)): m = finalized_profile_fitter[i] logger.debug("") logger.debug("Profiles for experiment %d" % i) for j in range(len(m)): logger.debug("Profile %d" % j) try: logger.debug(pprint.profile3d(m.data(j))) except Exception: logger.debug("** NO PROFILE **") # Print the modeller report self.profile_model_report = ProfileModelReport( self.experiments, finalized_profile_fitter, reference) logger.info("") logger.info(self.profile_model_report.as_str(prefix=' ')) # Print the time info logger.info("") logger.info(str(time_info)) logger.info("") # If we have more than 1 fold then do the validation if num_folds > 1: # Create the data processor executor = ProfileValidatorExecutor( self.experiments, profile_fitter) processor = ProcessorBuilder( self.ProcessorClass, self.experiments, reference, self.params.modelling).build() processor.executor = executor # Process the reference profiles reference, validation, time_info = processor.process() # Print the modeller report self.profile_validation_report = ProfileValidationReport( self.experiments, profile_fitter, reference, num_folds) logger.info("") logger.info( self.profile_validation_report.as_str(prefix=' ')) # Print the time info logger.info("") logger.info(str(time_info)) logger.info("") # Set to the finalized fitter profile_fitter = finalized_profile_fitter logger.info("=" * 80) logger.info("") logger.info(heading("Integrating reflections")) logger.info("") # Create the data processor executor = IntegratorExecutor(self.experiments, profile_fitter) processor = ProcessorBuilder(self.ProcessorClass, self.experiments, self.reflections, self.params.integration).build() processor.executor = executor # Process the reflections self.reflections, _, time_info = processor.process() # Finalize the reflections finalize = self.FinalizerClass(self.experiments, self.params) finalize(self.reflections) # Create the integration report self.integration_report = IntegrationReport(self.experiments, self.reflections) logger.info("") logger.info(self.integration_report.as_str(prefix=' ')) # Print the time info logger.info(str(time_info)) logger.info("") # Return the reflections return self.reflections