def run(self): '''Execute the script.''' from dials.array_family import flex from dials.util.options import flatten_datablocks from dials.util.options import flatten_reflections from time import time from dials.util import log from logging import info, debug from libtbx.utils import Sorry start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure the logging log.config( params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) # Ensure we have a data block datablocks = flatten_datablocks(params.input.datablock) reflections = flatten_reflections(params.input.reflections) if len(datablocks) == 0 and len(reflections) == 0: self.parser.print_help() return elif len(datablocks) != len(reflections): raise Sorry("Must have same number of datablocks and reflection tables") # Combine the datablocks and reflections datablock, reflections = combine( datablocks, reflections, params) # Save the reflections to file info('\n' + '-' * 80) reflections.as_pickle(params.output.reflections) info('Saved {0} reflections to {1}'.format( len(reflections), params.output.reflections)) # Save the datablock from dxtbx.datablock import DataBlockDumper info('Saving datablocks to {0}'.format( params.output.datablock)) dump = DataBlockDumper(datablocks) dump.as_file(params.output.datablock) # Print the time info("Time Taken: %f" % (time() - start_time))
def run(args): import libtbx.load_env from libtbx.utils import Sorry from dials.util import log usage = "%s [options] datablock.json strong.pickle" %libtbx.env.dispatcher_name parser = OptionParser( usage=usage, phil=phil_scope, read_datablocks=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=False) datablocks = flatten_datablocks(params.input.datablock) reflections = flatten_reflections(params.input.reflections) if len(datablocks) == 0 or len(reflections) == 0: parser.print_help() exit(0) # Configure the logging log.config( info=params.output.log, debug=params.output.debug_log) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) imagesets = [] for datablock in datablocks: imagesets.extend(datablock.extract_imagesets()) assert len(imagesets) > 0 assert len(reflections) == len(imagesets) if params.scan_range is not None and len(params.scan_range) > 0: reflections = [ filter_reflections_by_scan_range(refl, params.scan_range) for refl in reflections] dps_params = dps_phil_scope.extract() # for development, we want an exhaustive plot of beam probability map: dps_params.indexing.plot_search_scope = params.plot_search_scope dps_params.indexing.mm_search_scope = params.mm_search_scope new_detector, new_beam = discover_better_experimental_model( imagesets, reflections, params, dps_params, nproc=params.nproc, wide_search_binning=params.wide_search_binning) for imageset in imagesets: imageset.set_detector(new_detector) imageset.set_beam(new_beam) from dxtbx.serialize import dump dump.datablock(datablock, params.output.datablock)
def run(args): import libtbx.load_env from libtbx.utils import Sorry from dials.util import log from logging import info import cPickle as pickle usage = "%s [options] datablock.json strong.pickle" % \ libtbx.env.dispatcher_name # Create the option parser parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_datablocks=True, check_format=False, epilog=help_message) # Get the parameters params, options = parser.parse_args(show_diff_phil=False) # Configure the log log.config( params.verbosity, info='dials.find_hot_pixels.log', debug='dials.find_hot_pixels.debug.log') # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) datablocks = flatten_datablocks(params.input.datablock) reflections = flatten_reflections(params.input.reflections) if len(datablocks) == 0 and len(reflections) == 0: parser.print_help() exit(0) if len(datablocks) > 1: raise Sorry("Only one DataBlock can be processed at a time") else: imagesets = datablocks[0].extract_imagesets() if len(reflections) == 0: raise Sorry("No reflection lists found in input") if len(reflections) > 1: raise Sorry("Multiple reflections lists provided in input") assert(len(reflections) == 1) reflections = reflections[0] mask = hot_pixel_mask(imagesets[0], reflections) pickle.dump(mask, open(params.output.mask, 'w'), pickle.HIGHEST_PROTOCOL) print 'Wrote hot pixel mask to %s' % params.output.mask return
def run(self): ''' Perform the integration. ''' from dials.util.command_line import heading from dials.util.options import flatten_experiments from dials.util import log from time import time from libtbx.utils import Sorry from dials.array_family import flex # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) experiments = flatten_experiments(params.input.experiments) if len(experiments) == 0: self.parser.print_help() return assert len(experiments) == 1 imageset = experiments[0].imageset beam = experiments[0].beam detector = experiments[0].detector goniometer = experiments[0].goniometer assert len(detector) == 1 # Configure logging log.config() from dials.algorithms.background.gmodel import PolarTransform import cPickle as pickle model = pickle.load(open(params.model)) image = model.data(0) mask = flex.bool(image.accessor(), True) # Do the transformation transform = PolarTransform(beam, detector[0], goniometer) result = transform.to_polar(image, mask) data = result.data() mask = result.mask() pickle.dump((data, mask), open(params.output.data, "w")) from matplotlib import pylab vmax = sorted(list(data))[int(0.99 * len(data))] figure = pylab.figure(figsize=(6,4)) pylab.imshow( data.as_numpy_array(), interpolation = 'none', vmin = 0, vmax = vmax) ax1 = pylab.gca() ax1.get_xaxis().set_visible(False) ax1.get_yaxis().set_visible(False) cb = pylab.colorbar() cb.ax.tick_params(labelsize=8) logger.info("Saving polar model %s" % (params.output.image)) pylab.savefig("%s" % (params.output.image), dpi=600, bbox_inches='tight')
def run(args=None, phil=phil_scope): # type: (List[str], phil.scope) -> None """Run the scaling from the command-line.""" usage = """Usage: dials.scale integrated.refl integrated.expt [integrated.refl(2) integrated.expt(2) ....] [options]""" parser = OptionParser( usage=usage, read_experiments=True, read_reflections=True, phil=phil, check_format=False, epilog=__doc__, ) params, options = parser.parse_args(args=args, show_diff_phil=False) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) log.config(verbosity=options.verbose, logfile=params.output.log) logger.info(dials_version()) diff_phil = parser.diff_phil.as_str() if diff_phil: logger.info("The following parameters have been modified:\n%s", diff_phil) try: scaled_experiments, joint_table = run_scaling(params, experiments, reflections) except ValueError as e: raise Sorry(e) else: # Note, cross validation mode does not produce scaled datafiles if scaled_experiments and joint_table: logger.info("Saving the experiments to %s", params.output.experiments) scaled_experiments.as_file(params.output.experiments) logger.info("Saving the scaled reflections to %s", params.output.reflections) joint_table.as_file(params.output.reflections) if params.output.unmerged_mtz: _export_unmerged_mtz(params, scaled_experiments, joint_table) if params.output.merged_mtz: _export_merged_mtz(params, scaled_experiments, joint_table) logger.info( "See dials.github.io/dials_scale_user_guide.html for more info on scaling options" )
def run(args=None): usage = "dials.estimate_resolution [options] (scaled.expt scaled.refl | scaled_unmerged.mtz)" parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=__doc__, ) params, options, unhandled = parser.parse_args(args=args, return_unhandled=True, show_diff_phil=True) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) if (not reflections or not experiments) and not unhandled: parser.print_help() return if reflections and experiments and unhandled: sys.exit( "Must provide either scaled unmerged mtz OR dials-format scaled reflections and experiments files" ) # Configure the logging log.config(logfile=params.output.log, verbosity=options.verbose) logger.info(dials_version()) if len(unhandled) == 1: scaled_unmerged = unhandled[0] m = resolution_analysis.Resolutionizer.from_unmerged_mtz( scaled_unmerged, params.resolution) else: reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): sys.exit( f"Mismatched number of experiments and reflection tables found: {len(experiments)} & {len(reflections)}." ) m = resolution_analysis.Resolutionizer.from_reflections_and_experiments( reflections, experiments, params.resolution) plots = m.resolution_auto() if params.output.html: output_html_report(plots, params.output.html) if params.output.json: with open(params.output.json, "w") as fh: json.dump(plots, fh) return plots
def run(self): """ Perform the integration. """ from time import time # Check the number of arguments is correct start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(reflections) == 0 or len(experiments) == 0: self.parser.print_help() return elif len(reflections) != 1: raise Sorry("more than 1 reflection file was given") elif len(experiments) == 0: raise Sorry("no experiment list was specified") reflections = reflections[0] # Configure logging if __name__ == "__main__": # Configure the logging log.config(verbosity=options.verbose, logfile="dials.potato.log") # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) # Contruct the integrator integrator = Integrator(experiments, reflections, params) # Do cycles of indexing and refinement for i in range(params.refinement.n_macro_cycles): integrator.reindex_strong_spots() integrator.integrate_strong_spots() integrator.refine() # Do the integration integrator.predict() integrator.integrate() # Get the reflections reflections = integrator.reflections experiments = integrator.experiments # Save the reflections reflections.as_pickle(params.output.reflections) experiments.as_file(params.output.experiments)
def run(args=None, phil=phil_scope): """Run the command-line script.""" usage = "dials.compute_delta_cchalf [options] scaled.expt scaled.refl" parser = ArgumentParser( usage=usage, phil=phil, epilog=help_message, read_experiments=True, read_reflections=True, check_format=False, ) params, _ = parser.parse_args(args=args, show_diff_phil=False) log.config(logfile=params.output.log) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) if not experiments and not reflections: if not params.input.mtzfile: parser.print_help() return else: try: script = CCHalfFromMTZ(params, params.input.mtzfile) except ValueError as e: sys.exit(f"Error: {e}") else: if not experiments or not reflections: parser.print_help() return else: if not len(reflections) == 1: exit("Only one reflection table can be provided") n_datasets = len(set(reflections[0]["id"]).difference({-1})) if n_datasets != len(experiments): exit( """ The number of experiments (%s) does not match the number of datasets in the reflection table (%s) """, len(experiments), n_datasets, ) try: script = CCHalfFromDials(params, experiments, reflections[0]) except ValueError as e: sys.exit(f"Error: {e}") script.run() script.output()
def run(self): '''Execute the script.''' from dials.array_family import flex from dials.util.options import flatten_datablocks from dials.util.options import flatten_reflections from time import time from dials.util import log from libtbx.utils import Sorry start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure the logging log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Ensure we have a data block datablocks = flatten_datablocks(params.input.datablock) reflections = flatten_reflections(params.input.reflections) if len(datablocks) == 0 and len(reflections) == 0: self.parser.print_help() return elif len(datablocks) != len(reflections): raise Sorry( "Must have same number of datablocks and reflection tables") # Combine the datablocks and reflections datablock, reflections = combine(datablocks, reflections, params) # Save the reflections to file logger.info('\n' + '-' * 80) reflections.as_pickle(params.output.reflections) logger.info('Saved {0} reflections to {1}'.format( len(reflections), params.output.reflections)) # Save the datablock from dxtbx.datablock import DataBlockDumper logger.info('Saving datablocks to {0}'.format(params.output.datablock)) dump = DataBlockDumper(datablocks) dump.as_file(params.output.datablock) # Print the time logger.info("Time Taken: %f" % (time() - start_time))
def run(args=None, phil=phil_scope): # type: (List[str], phil.scope) -> None """Run the command-line script.""" usage = "dials.damage_analysis [options] scaled.expt scaled.refl | scaled.mtz" parser = OptionParser( usage=usage, phil=phil, epilog=__doc__, read_experiments=True, read_reflections=True, check_format=False, ) params, _, unhandled = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) log.config(logfile=params.output.log) logger.info(dials_version()) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) try: if experiments and reflections: if len(reflections) != 1: raise ValueError( "A single input reflections datafile is required") if "inverse_scale_factor" not in reflections[0]: raise KeyError("Input data must be scaled.") script = PychefRunner.from_dials_data_files( params, experiments, reflections[0], ) elif unhandled and os.path.isfile(unhandled[0]): try: mtz_object = mtz.object(file_name=unhandled[0]) except RuntimeError: # If an error is encountered trying to read the file as an mtzfile raise ValueError( "Input file cannot be read as a valid experiment/reflection file or MTZ file" ) else: script = PychefRunner.from_mtz(params, mtz_object) else: parser.print_help() raise ValueError("Suitable input datafiles not provided") except (ValueError, KeyError) as e: sys.exit("Error: %s" % str(e)) else: script.run() script.make_html_report(params.output.html, params.output.json)
def run(args): usage = "dials.symmetry [options] models.expt observations.refl" parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, _, args = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) # Configure the logging log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.seed is not None: flex.set_random_seed(params.seed) random.seed(params.seed) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): raise Sorry( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) try: experiments, reflections = assign_unique_identifiers( experiments, reflections) symmetry(experiments, reflections, params=params) except ValueError as e: raise Sorry(e)
def run(args=None): """Run the script from the command-line.""" usage = """Usage: dials.systematic_absences scaled.refl scaled.expt [options]""" parser = OptionParser( usage=usage, read_experiments=True, read_reflections=True, phil=phil_scope, check_format=False, epilog=help_message, ) params, _ = parser.parse_args(args=args, show_diff_phil=False) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) log.config(verbosity=1, info=params.output.log) logger.info(dials_version()) diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) ### Assert that all data have been scaled with dials - should only be # able to input one reflection table and experimentlist that are # matching and scaled together. if not len(reflections) == 1: raise Sorry("Only one reflection table can be given as input.") if (not "intensity.scale.value" in reflections[0]) and ( not "intensity.prf.value" in reflections[0]): raise Sorry( "Unable to find integrated or scaled reflections in the reflection table." ) try: run_sys_abs_checks(experiments, reflections, params.d_min) except ValueError as e: raise Sorry(e) if params.output.html: ScrewAxisObserver().generate_html_report(params.output.html) if params.output.experiments: dump = ExperimentListDumper(experiments) with open(params.output.experiments, "w") as outfile: outfile.write(dump.as_json(split=True))
def run(self): ''' Perform the integration. ''' from dials.util.options import flatten_experiments from dials.util import log from dials.array_family import flex # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) experiments = flatten_experiments(params.input.experiments) if len(experiments) == 0: self.parser.print_help() return assert len(experiments) == 1 imageset = experiments[0].imageset beam = experiments[0].beam detector = experiments[0].detector goniometer = experiments[0].goniometer assert len(detector) == 1 # Configure logging log.config() from dials.algorithms.background.gmodel import PolarTransform import cPickle as pickle model = pickle.load(open(params.model)) image = model.data(0) mask = flex.bool(image.accessor(), True) # Do the transformation transform = PolarTransform(beam, detector[0], goniometer) result = transform.to_polar(image, mask) data = result.data() mask = result.mask() pickle.dump((data, mask), open(params.output.data, "w")) from matplotlib import pylab vmax = sorted(list(data))[int(0.99 * len(data))] figure = pylab.figure(figsize=(6, 4)) pylab.imshow(data.as_numpy_array(), interpolation='none', vmin=0, vmax=vmax) ax1 = pylab.gca() ax1.get_xaxis().set_visible(False) ax1.get_yaxis().set_visible(False) cb = pylab.colorbar() cb.ax.tick_params(labelsize=8) logger.info("Saving polar model %s" % (params.output.image)) pylab.savefig("%s" % (params.output.image), dpi=600, bbox_inches='tight')
def run(args=None): import six.moves.cPickle as pickle from dials.util import Sorry, log usage = "dials.find_hot_pixels [options] models.expt strong.refl" # Create the option parser parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) # Get the parameters params, options = parser.parse_args(args, show_diff_phil=False) # Configure the log log.config(verbosity=options.verbose, logfile="dials.find_hot_pixels.log") # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) if len(experiments) == 0 and len(reflections) == 0: parser.print_help() exit(0) if len(experiments) > 1: raise Sorry("Only one experiment can be processed at a time") else: imagesets = experiments.imagesets() if len(reflections) == 0: raise Sorry("No reflection lists found in input") if len(reflections) > 1: raise Sorry("Multiple reflections lists provided in input") assert len(reflections) == 1 reflections = reflections[0] mask = hot_pixel_mask(imagesets[0], reflections) with open(params.output.mask, "wb") as fh: pickle.dump(mask, fh, pickle.HIGHEST_PROTOCOL) print("Wrote hot pixel mask to %s" % params.output.mask)
def run(args=None): """Run symmetry analysis from the command-line.""" usage = "dials.symmetry [options] models.expt observations.refl" parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options, args = parser.parse_args( args=args, show_diff_phil=False, return_unhandled=True ) # Configure the logging log.config(verbosity=options.verbose, logfile=params.output.log) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.seed is not None: flex.set_random_seed(params.seed) random.seed(params.seed) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments ) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): sys.exit( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections)) ) try: experiments, reflections = assign_unique_identifiers(experiments, reflections) symmetry(experiments, reflections, params=params) except ValueError as e: sys.exit(e)
def run(args): import random interp = phil_scope.command_line_argument_interpreter() params, unhandled = interp.process_and_fetch( args, custom_processor='collect_remaining') params = params.extract() if params.seed is not None: flex.set_random_seed(params.seed) random.seed(params.seed) from dials.util import log # Configure the logging log.config() datasets, expected_reindexing_ops = generate_test_data( space_group=params.space_group.group(), lattice_group=params.lattice_group, unit_cell=params.unit_cell, unit_cell_volume=params.unit_cell_volume, seed=params.seed, d_min=params.d_min, sigma=params.sigma, sample_size=params.sample_size, map_to_p1=params.map_to_p1, twin_fractions=params.twin_fractions) result = analyse_datasets(datasets, params) space_groups = {} reindexing_ops = {} for dataset_id in result.reindexing_ops.iterkeys(): if 0 in result.reindexing_ops[dataset_id]: cb_op = result.reindexing_ops[dataset_id][0] reindexing_ops.setdefault(cb_op, []) reindexing_ops[cb_op].append(dataset_id) if dataset_id in result.space_groups: space_groups.setdefault(result.space_groups[dataset_id], []) space_groups[result.space_groups[dataset_id]].append(dataset_id) logger.info('Space groups:') for sg, datasets in space_groups.iteritems(): logger.info(str(sg.info().reference_setting())) logger.info(datasets) logger.info('Reindexing operators:') for cb_op, datasets in reindexing_ops.iteritems(): logger.info(cb_op) logger.info(datasets)
def run(args): import libtbx.load_env from dials.array_family import flex from dials.util import log from dials.util.version import dials_version usage = "%s [options] experiment.json indexed.pickle" % \ libtbx.env.dispatcher_name parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=True) # Configure the logging log.config(info=params.output.log, debug=params.output.debug_log) logger.info(dials_version()) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(reflections) == 0 or len(experiments) == 0: parser.print_help() return assert(len(reflections) == 1) assert(len(experiments) == 1) experiment = experiments[0] reflections = reflections[0] # remove reflections with 0, 0, 0 index zero = (reflections['miller_index'] == (0, 0, 0)) logger.info('Removing %d unindexed reflections' % zero.count(True)) reflections = reflections.select(~zero) h, k, l = reflections['miller_index'].as_vec3_double().parts() h = h.iround() k = k.iround() l = l.iround() logger.info('Range on h: %d to %d' % (flex.min(h), flex.max(h))) logger.info('Range on k: %d to %d' % (flex.min(k), flex.max(k))) logger.info('Range on l: %d to %d' % (flex.min(l), flex.max(l))) test_P1_crystal_indexing(reflections, experiment, params) test_crystal_pointgroup_symmetry(reflections, experiment, params)
def run(self, experiments, reflections): from dials.util import log self.logger.log_step_time("INTEGRATE") logfile = os.path.splitext( self.logger.rank_log_file_path)[0] + "_integrate.log" log.config(logfile=logfile) processor = integrate_only_processor(self.params) # Re-generate the image sets using their format classes so we can read the raw data # Integrate the experiments one at a time to not use up memory all_integrated_expts = ExperimentList() all_integrated_refls = flex.reflection_table() current_imageset = None current_imageset_path = None for expt_id, expt in enumerate(experiments): assert len(expt.imageset.paths()) == 1 and len(expt.imageset) == 1 self.logger.log("Starting integration experiment %d" % expt_id) refls = reflections.select( reflections['exp_id'] == expt.identifier) if expt.imageset.paths()[0] != current_imageset_path: current_imageset_path = expt.imageset.paths()[0] current_imageset = ImageSetFactory.make_imageset( expt.imageset.paths()) idx = expt.imageset.indices()[0] expt.imageset = current_imageset[idx:idx + 1] idents = refls.experiment_identifiers() del idents[expt_id] idents[0] = expt.identifier refls['id'] = flex.int(len(refls), 0) try: integrated = processor.integrate( experiments[expt_id:expt_id + 1], refls) except RuntimeError: self.logger.log("Error integrating expt %d" % expt_id) continue all_integrated_expts.append(expt) idents = integrated.experiment_identifiers() del idents[0] idents[expt_id] = expt.identifier integrated['id'] = flex.int(len(integrated), len(all_integrated_expts) - 1) all_integrated_refls.extend(integrated) self.logger.log("Integration done, %d experiments, %d reflections" % (len(all_integrated_expts), len(all_integrated_refls))) return all_integrated_expts, all_integrated_refls
def run(phil=working_phil, args=None): usage = "dials.index [options] models.expt strong.refl" parser = OptionParser( usage=usage, phil=phil, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options = parser.parse_args(args=args, show_diff_phil=False) # Configure the logging log.config(verbosity=options.verbose, logfile=params.output.log) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if len(experiments) == 0: parser.print_help() return try: indexed_experiments, indexed_reflections = index( experiments, reflections, params) except (DialsIndexError, ValueError) as e: sys.exit(str(e)) # Save experiments if params.output.split_experiments: logger.info("Splitting experiments before output") indexed_experiments = ExperimentList( [copy.deepcopy(re) for re in indexed_experiments]) logger.info("Saving refined experiments to %s" % params.output.experiments) assert indexed_experiments.is_consistent() indexed_experiments.as_file(params.output.experiments) # Save reflections logger.info("Saving refined reflections to %s" % params.output.reflections) indexed_reflections.as_msgpack_file(filename=params.output.reflections)
def run(self): ''' Parse the options. ''' from dxtbx.datablock import DataBlockFactory from dxtbx.datablock import DataBlockTemplateImporter from dials.util.options import flatten_datablocks from dials.util import log from logging import info, debug import cPickle as pickle from libtbx.utils import Sorry # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=False) datablocks = flatten_datablocks(params.input.datablock) # Configure logging log.config( params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) # Load reference geometry reference_detector = None reference_beam = None if params.input.reference_geometry is not None: from dxtbx.serialize import load try: experiments = load.experiment_list( params.input.reference_geometry, check_format=False) assert len(experiments.detectors()) == 1 assert len(experiments.beams()) == 1 reference_detector = experiments.detectors()[0] reference_beam = experiments.beams()[0] except Exception, e: datablock = load.datablock(params.input.reference_geometry) assert len(datablock) == 1 imageset = datablock[0].extract_imagesets()[0] reference_detector = imageset.get_detector() reference_beam = imageset.get_beam()
def run(self): ''' Run the script. ''' from dials.util.masking import MaskGenerator from dials.util.options import flatten_datablocks from libtbx.utils import Sorry import six.moves.cPickle as pickle from dials.util import log from dxtbx.format.image import ImageBool # Parse the command line arguments params, options = self.parser.parse_args(show_diff_phil=True) datablocks = flatten_datablocks(params.input.datablock) # Configure logging log.config() # Check number of args if len(datablocks) == 0: self.parser.print_help() return if len(datablocks) != 1: raise Sorry('exactly 1 datablock must be specified') datablock = datablocks[0] imagesets = datablock.extract_imagesets() if len(imagesets) != 1: raise Sorry('datablock must contain exactly 1 imageset') imageset = imagesets[0] # Generate the mask generator = MaskGenerator(params) mask = generator.generate(imageset) # Save the mask to file print("Writing mask to %s" % params.output.mask) with open(params.output.mask, "wb") as fh: pickle.dump(mask, fh) # Save the datablock if params.output.datablock is not None: imageset.external_lookup.mask.data = ImageBool(mask) imageset.external_lookup.mask.filename = params.output.mask from dxtbx.datablock import DataBlockDumper print('Saving datablocks to {0}'.format(params.output.datablock)) dump = DataBlockDumper(datablocks) dump.as_file(params.output.datablock)
def run(args): usage = "dials.check_indexing_symmetry [options] indexed.expt indexed.refl" parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options = parser.parse_args(show_diff_phil=True) # Configure the logging log.config(logfile=params.output.log) logger.info(dials_version()) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) if len(reflections) == 0 or len(experiments) == 0: parser.print_help() return assert len(reflections) == 1 assert len(experiments) == 1 experiment = experiments[0] reflections = reflections[0] # remove reflections with 0, 0, 0 index zero = reflections["miller_index"] == (0, 0, 0) logger.info("Removing %d unindexed reflections" % zero.count(True)) reflections = reflections.select(~zero) h, k, l = reflections["miller_index"].as_vec3_double().parts() h = h.iround() k = k.iround() l = l.iround() logger.info("Range on h: %d to %d" % (flex.min(h), flex.max(h))) logger.info("Range on k: %d to %d" % (flex.min(k), flex.max(k))) logger.info("Range on l: %d to %d" % (flex.min(l), flex.max(l))) test_P1_crystal_indexing(reflections, experiment, params) test_crystal_pointgroup_symmetry(reflections, experiment, params)
def run(): """Run the command line filtering script.""" flags = list(flex.reflection_table.flags.names.items()) flags.sort(key=itemgetter(0)) phil_scope = parse(phil_str, process_includes=True) # The script usage usage = "usage: dials.filter_reflections [options] experiment.expt" # Create the parser parser = OptionParser( usage=usage, phil=phil_scope, epilog=help_message, read_reflections=True, read_experiments=True, check_format=False, ) params, options = parser.parse_args(show_diff_phil=True) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) log.config(verbosity=options.verbose) if not reflections: parser.print_help() raise Sorry("No valid reflection file given") if len(reflections) != 1: parser.print_help() raise Sorry("Exactly 1 reflection file must be specified") reflections = reflections[0] # Check if any filter has been set using diff_phil filter_def = [ o for o in parser.diff_phil.objects if o.name not in ["input", "output"] ] if not filter_def: print("No filter specified. Performing analysis instead.") run_analysis(flags, reflections) else: run_filtering(params, experiments, reflections)
def run(args=None, phil=working_phil): usage = "dials.index [options] models.expt strong.refl" parser = ArgumentParser( usage=usage, phil=phil, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options = parser.parse_args(args=args, show_diff_phil=False) # Configure the logging log.config(verbosity=options.verbose, logfile=params.output.log) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) if len(experiments) == 0: parser.print_help() return try: indexed_experiments, indexed_reflections = index( experiments, reflections, params) except (DialsIndexError, ValueError) as e: sys.exit(str(e)) # Save experiments logger.info("Saving refined experiments to %s", params.output.experiments) assert indexed_experiments.is_consistent() indexed_experiments.as_file(params.output.experiments) # Save reflections logger.info("Saving refined reflections to %s", params.output.reflections) indexed_reflections.as_file(filename=params.output.reflections)
def run(args): from dials.util.options import OptionParser from dials.util.options import flatten_experiments import libtbx.load_env usage = "%s [options] models.expt" % (libtbx.env.dispatcher_name) parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, check_format=False, epilog=help_message, ) from dials.util.version import dials_version logger.info(dials_version()) params, options = parser.parse_args(show_diff_phil=False) experiments = flatten_experiments(params.input.experiments) if len(experiments) == 0: parser.print_help() exit(0) log.config() # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) uc_params = uc_params_from_experiments(experiments) panel_distances = panel_distances_from_experiments(experiments) outliers = outlier_selection(uc_params, iqr_ratio=params.iqr_ratio) plot_uc_histograms(uc_params, outliers, params.steps_per_angstrom) plot_uc_vs_detector_distance( uc_params, panel_distances, outliers, params.steps_per_angstrom ) plot_number_of_crystals(experiments)
def run(self): """ Perform the integration. """ from time import time # Check the number of arguments is correct start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(reflections) == 0 or len(experiments) == 0: self.parser.print_help() return elif len(reflections) != 1: raise Sorry("more than 1 reflection file was given") elif len(experiments) == 0: raise Sorry("no experiment list was specified") reflections = reflections[0] # Configure logging log.config(info="dials.merge_stills.log", debug="dials.merge_stills.debug.log") # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) # Read the reference if params.input.reference: reference = self._read_reference(params.input.reference) else: reference = None # Do the merging reflections = scale_and_merge(experiments, reflections, params, reference)
def run(args): usage = ( "dials.resolutionizer [options] (scaled.expt scaled.refl | scaled_unmerged.mtz)" ) parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options, unhandled = parser.parse_args(return_unhandled=True, show_diff_phil=True) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) if (not reflections or not experiments) and not unhandled: parser.print_help() return if reflections and experiments and unhandled: sys.exit( "Must provide either scaled unmerged mtz OR dials-format scaled reflections and experiments files" ) # Configure the logging log.config(logfile=params.output.log, verbosity=options.verbose) logger.info(dials_version()) if len(unhandled) == 1: scaled_unmerged = unhandled[0] m = resolutionizer.Resolutionizer.from_unmerged_mtz( scaled_unmerged, params.resolutionizer) else: reflections = parse_multiple_datasets(reflections) m = resolutionizer.Resolutionizer.from_reflections_and_experiments( reflections, experiments, params.resolutionizer) m.resolution_auto()
def run(args=None): from dials.util.options import ( ArgumentParser, reflections_and_experiments_from_files, ) from dials.util.version import dials_version usage = "dials.export models.expt reflections.refl [options]" parser = ArgumentParser( usage=usage, read_experiments=True, read_reflections=True, phil=phil_scope, epilog=help_message, ) # Get the parameters params, options = parser.parse_args(args, show_diff_phil=False) # Configure the logging log.config(logfile=params.output.log) # Print the version number logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if not params.input.experiments and not params.input.reflections: parser.print_help() sys.exit() # Get the experiments and reflections reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) exporter = BestExporter(params, experiments, reflections) exporter.export()
def __init__(self, params_filename, output_tag, logfile=None): """ @param params_filename cctbx.xfel/DIALS parameter file for processing @output_tag String that will prefix output files @logfile File name for logging """ self.parsed_params = parse(file_name=params_filename) dials_params = phil_scope.fetch(self.parsed_params).extract() super(CctbxPsanaEventProcessor, self).__init__(dials_params, output_tag) self.update_geometry = ManualGeometryUpdater(dials_params) simple_script = SimpleScript(dials_params) simple_script.load_reference_geometry() self.reference_detector = getattr(simple_script, 'reference_detector', None) self.output_tag = output_tag self.detector_params = None if logfile is not None: log.config(logfile=logfile)
def run(phil=working_phil, args=None): usage = "dials.index [options] models.expt strong.refl" parser = OptionParser( usage=usage, phil=phil, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options = parser.parse_args(args=args, show_diff_phil=False) from dials.util import log # Configure the logging log.config(verbosity=options.verbose, logfile=params.output.log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if len(experiments) == 0: parser.print_help() return indexed = Index(experiments, reflections, params) indexed.export_experiments(params.output.experiments) indexed.export_reflections(params.output.reflections)
def run(args=None): usage = "dials.unit_cell_histogram [options] models.expt" parser = ArgumentParser( usage=usage, phil=phil_scope, read_experiments=True, check_format=False, epilog=help_message, ) logger.info(dials_version()) params, options = parser.parse_args(args, show_diff_phil=False) experiments = flatten_experiments(params.input.experiments) if len(experiments) == 0: parser.print_help() exit(0) log.config() # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) uc_params = uc_params_from_experiments(experiments) panel_distances = panel_distances_from_experiments(experiments) outliers = outlier_selection(uc_params, iqr_ratio=params.iqr_ratio) plot_uc_histograms(uc_params, outliers, params.steps_per_angstrom) plot_uc_vs_detector_distance( uc_params, panel_distances, outliers, params.steps_per_angstrom ) plot_number_of_crystals(experiments)
def run(self): '''Execute the script.''' from dials.util import log from time import time from libtbx import easy_mp import copy # Parse the command line params, options, all_paths = self.parser.parse_args(show_diff_phil=False, return_unhandled=True) # Check we have some filenames if not all_paths: self.parser.print_help() return # Save the options self.options = options self.params = params st = time() # Configure logging log.config( params.verbosity, info='dials.process.log', debug='dials.process.debug.log') # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) self.load_reference_geometry() from dials.command_line.dials_import import ManualGeometryUpdater update_geometry = ManualGeometryUpdater(params) # Import stuff logger.info("Loading files...") pre_import = params.dispatch.pre_import or len(all_paths) == 1 if pre_import: # Handle still imagesets by breaking them apart into multiple datablocks # Further handle single file still imagesets (like HDF5) by tagging each # frame using its index datablocks = [do_import(path) for path in all_paths] if self.reference_detector is not None: from dxtbx.model import Detector for datablock in datablocks: for imageset in datablock.extract_imagesets(): for i in range(len(imageset)): imageset.set_detector( Detector.from_dict(self.reference_detector.to_dict()), index=i) for datablock in datablocks: for imageset in datablock.extract_imagesets(): update_geometry(imageset) indices = [] basenames = [] split_datablocks = [] for datablock in datablocks: for imageset in datablock.extract_imagesets(): paths = imageset.paths() for i in xrange(len(imageset)): subset = imageset[i:i+1] split_datablocks.append(DataBlockFactory.from_imageset(subset)[0]) indices.append(i) basenames.append(os.path.splitext(os.path.basename(paths[i]))[0]) tags = [] for i, basename in zip(indices, basenames): if basenames.count(basename) > 1: tags.append("%s_%05d"%(basename, i)) else: tags.append(basename) # Wrapper function def do_work(item): Processor(copy.deepcopy(params)).process_datablock(item[0], item[1]) iterable = zip(tags, split_datablocks) else: basenames = [os.path.splitext(os.path.basename(filename))[0] for filename in all_paths] tags = [] for i, basename in enumerate(basenames): if basenames.count(basename) > 1: tags.append("%s_%05d"%(basename, i)) else: tags.append(basename) # Wrapper function def do_work(item): tag, filename = item datablock = do_import(filename) imagesets = datablock.extract_imagesets() if len(imagesets) == 0 or len(imagesets[0]) == 0: logger.info("Zero length imageset in file: %s"%filename) return if len(imagesets) > 1: raise Abort("Found more than one imageset in file: %s"%filename) if len(imagesets[0]) > 1: raise Abort("Found a multi-image file. Run again with pre_import=True") if self.reference_detector is not None: from dxtbx.model import Detector imagesets[0].set_detector(Detector.from_dict(self.reference_detector.to_dict())) update_geometry(imagesets[0]) Processor(copy.deepcopy(params)).process_datablock(tag, datablock) iterable = zip(tags, all_paths) # Process the data if params.mp.method == 'mpi': from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed size = comm.Get_size() # size: number of processes running in this job for i, item in enumerate(iterable): if (i+rank)%size == 0: do_work(item) else: easy_mp.parallel_map( func=do_work, iterable=iterable, processes=params.mp.nproc, method=params.mp.method, preserve_order=True, preserve_exception_message=True) # Total Time logger.info("") logger.info("Total Time Taken = %f seconds" % (time() - st))
def run(args): from dials.util import log from logging import info import libtbx.load_env usage = "%s experiments.json indexed.pickle [options]" %libtbx.env.dispatcher_name parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=False) # Configure the logging log.config(info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) assert(len(reflections) == 1) reflections = reflections[0] if len(experiments) == 0: parser.print_help() return elif len(experiments.crystals()) > 1: if params.crystal_id is not None: assert params.crystal_id < len(experiments.crystals()) experiment_ids = experiments.where(crystal=experiments.crystals()[params.crystal_id]) from dxtbx.model.experiment.experiment_list import ExperimentList experiments = ExperimentList([experiments[i] for i in experiment_ids]) refl_selections = [reflections['id'] == i for i in experiment_ids] reflections['id'] = flex.int(len(reflections), -1) for i, sel in enumerate(refl_selections): reflections['id'].set_selected(sel, i) reflections = reflections.select(reflections['id'] > -1) else: raise Sorry("Only one crystal can be processed at a time: set crystal_id to choose experiment.") if params.refinement.reflections.outlier.algorithm in ('auto', libtbx.Auto): if experiments[0].goniometer is None: params.refinement.reflections.outlier.algorithm = 'sauter_poon' else: # different default to dials.refine # tukey is faster and more appropriate at the indexing step params.refinement.reflections.outlier.algorithm = 'tukey' from dials.algorithms.indexing.symmetry \ import refined_settings_factory_from_refined_triclinic cb_op_to_primitive = experiments[0].crystal.get_space_group().info()\ .change_of_basis_op_to_primitive_setting() if experiments[0].crystal.get_space_group().n_ltr() > 1: effective_group = experiments[0].crystal.get_space_group()\ .build_derived_reflection_intensity_group(anomalous_flag=True) sys_absent_flags = effective_group.is_sys_absent( reflections['miller_index']) reflections = reflections.select(~sys_absent_flags) experiments[0].crystal.update(experiments[0].crystal.change_basis(cb_op_to_primitive)) miller_indices = reflections['miller_index'] miller_indices = cb_op_to_primitive.apply(miller_indices) reflections['miller_index'] = miller_indices Lfat = refined_settings_factory_from_refined_triclinic( params, experiments, reflections, lepage_max_delta=params.lepage_max_delta, nproc=params.nproc, refiner_verbosity=params.verbosity) s = StringIO() Lfat.labelit_printout(out=s) info(s.getvalue()) from json import dumps from os.path import join open(join(params.output.directory, 'bravais_summary.json'), 'wb').write(dumps(Lfat.as_dict())) from dxtbx.serialize import dump import copy for subgroup in Lfat: expts = copy.deepcopy(experiments) for expt in expts: expt.crystal.update(subgroup.refined_crystal) expt.detector = subgroup.detector expt.beam = subgroup.beam dump.experiment_list( expts, join(params.output.directory, 'bravais_setting_%i.json' % (int(subgroup.setting_number)))) return
def run(self): """ Process all images assigned to this thread """ params, options = self.parser.parse_args( show_diff_phil=True) # Check inputs if params.input.experiment is None or \ params.input.run_num is None or \ params.input.address is None: raise Usage(self.usage) if params.format.file_format == "cbf": if params.format.cbf.detz_offset is None: raise Usage(self.usage) elif params.format.file_format == "pickle": if params.input.cfg is None: raise Usage(self.usage) else: raise Usage(self.usage) if not os.path.exists(params.output.output_dir): raise Sorry("Output path not found:" + params.output.output_dir) self.params = params self.load_reference_geometry() # The convention is to put %s in the phil parameter to add a time stamp to # each output datafile. Save the initial templates here. self.strong_filename_template = params.output.strong_filename self.indexed_filename_template = params.output.indexed_filename self.refined_experiments_filename_template = params.output.refined_experiments_filename self.integrated_filename_template = params.output.integrated_filename self.reindexedstrong_filename_template = params.output.reindexedstrong_filename # Don't allow the strong reflections to be written unless there are enough to # process params.output.strong_filename = None # Save the paramters self.params_cache = copy.deepcopy(params) self.options = options if params.mp.method == "mpi": from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed size = comm.Get_size() # size: number of processes running in this job elif params.mp.method == "sge" and \ 'SGE_TASK_ID' in os.environ and \ 'SGE_TASK_FIRST' in os.environ and \ 'SGE_TASK_LAST' in os.environ: if 'SGE_STEP_SIZE' in os.environ: assert int(os.environ['SGE_STEP_SIZE']) == 1 if os.environ['SGE_TASK_ID'] == 'undefined' or os.environ['SGE_TASK_ID'] == 'undefined' or os.environ['SGE_TASK_ID'] == 'undefined': rank = 0 size = 1 else: rank = int(os.environ['SGE_TASK_ID']) - int(os.environ['SGE_TASK_FIRST']) size = int(os.environ['SGE_TASK_LAST']) - int(os.environ['SGE_TASK_FIRST']) + 1 else: rank = 0 size = 1 # Configure the logging if params.output.logging_dir is None: info_path = '' debug_path = '' else: log_path = os.path.join(params.output.logging_dir, "log_rank%04d.out"%rank) error_path = os.path.join(params.output.logging_dir, "error_rank%04d.out"%rank) print "Redirecting stdout to %s"%log_path print "Redirecting stderr to %s"%error_path sys.stdout = open(log_path,'a', buffering=0) sys.stderr = open(error_path,'a',buffering=0) print "Should be redirected now" info_path = os.path.join(params.output.logging_dir, "info_rank%04d.out"%rank) debug_path = os.path.join(params.output.logging_dir, "debug_rank%04d.out"%rank) from dials.util import log log.config(params.verbosity, info=info_path, debug=debug_path) debug_dir = os.path.join(params.output.output_dir, "debug") if not os.path.exists(debug_dir): try: os.makedirs(debug_dir) except OSError, e: pass # due to multiprocessing, makedirs can sometimes fail
def run(self, args=None): params, options = self.parser.parse_args(args, show_diff_phil=True) log.config(logfile="dials.complete_full_sphere.log") model_shadow = params.shadow experiments = flatten_experiments(params.input.experiments) if len(experiments) != 1: self.parser.print_help() return expt = experiments[0] axes = expt.goniometer.get_axes() if len(axes) != 3: sys.exit("This will only work with 3-axis goniometers") if not expt.imageset.reader().get_format(): sys.exit("This will only work with images available") if not expt.imageset.reader().get_format().get_goniometer_shadow_masker(): model_shadow = False beam = expt.beam det = expt.detector if params.resolution: resolution = params.resolution else: resolution = det.get_max_inscribed_resolution(expt.beam.get_s0()) # at this point, predict all of the reflections in the scan possible (i.e. # extend scan to 360 degrees) - this points back to expt self.make_scan_360(expt.scan) # now get a full set of all unique miller indices all_indices = miller.build_set( crystal_symmetry=crystal.symmetry( space_group=expt.crystal.get_space_group(), unit_cell=expt.crystal.get_unit_cell(), ), anomalous_flag=True, d_min=resolution, ) if model_shadow: obs, shadow = self.predict_to_miller_set_with_shadow(expt, resolution) else: obs = self.predict_to_miller_set(expt, resolution) logger.info( "Fraction of unique observations at datum: %.1f%%", 100.0 * len(obs.indices()) / len(all_indices.indices()), ) missing = all_indices.lone_set(other=obs) logger.info("%d unique reflections in blind region", len(missing.indices())) e1 = matrix.col(axes[0]) e2 = matrix.col(axes[1]) e3 = matrix.col(axes[2]) s0n = matrix.col(beam.get_s0()).normalize() # rotate blind region about beam by +/- two theta two_theta = 2.0 * math.asin(0.5 * beam.get_wavelength() / resolution) R_ptt = s0n.axis_and_angle_as_r3_rotation_matrix(two_theta) R_ntt = s0n.axis_and_angle_as_r3_rotation_matrix(-two_theta) # now decompose to e3, e2, e1 sol_plus = rotation_decomposition.solve_r3_rotation_for_angles_given_axes( R_ptt, e3, e2, e1, return_both_solutions=True, deg=True ) sol_minus = rotation_decomposition.solve_r3_rotation_for_angles_given_axes( R_ntt, e3, e2, e1, return_both_solutions=True, deg=True ) solutions = [] if sol_plus: solutions.extend(sol_plus) if sol_minus: solutions.extend(sol_minus) if not solutions: sys.exit(f"Impossible two theta: {two_theta * 180.0 / math.pi:.3f},") logger.info("Maximum two theta: %.3f,", two_theta * 180.0 / math.pi) logger.info("%d solutions found", len(solutions)) names = tuple( [n.replace("GON_", "").lower() for n in expt.goniometer.get_names()] ) logger.info(" %8s %8s %8s coverage expt.expt" % names) self.write_expt(experiments, "solution_0.expt") for j, s in enumerate(solutions): expt.goniometer.set_angles(s) if model_shadow: obs, shadow = self.predict_to_miller_set_with_shadow(expt, resolution) else: obs = self.predict_to_miller_set(expt, resolution) new = missing.common_set(obs) fout = "solution_%d.expt" % (j + 1) f = len(new.indices()) / len(missing.indices()) logger.info("%8.3f %8.3f %8.3f %4.2f %s", s[0], s[1], s[2], f, fout) self.write_expt(experiments, fout)
def run(self): """ Process all images assigned to this thread """ params, options = self.parser.parse_args( show_diff_phil=True) # Configure the logging from dials.util import log log.config(params.verbosity) # Check inputs if params.input.experiment is None or \ params.input.run_num is None or \ params.input.address is None: raise Usage(self.usage) if params.format.file_format == "cbf": if params.format.cbf.detz_offset is None: raise Usage(self.usage) elif params.format.file_format == "pickle": if params.format.pickle.cfg is None: raise Usage(self.usage) else: raise Usage(self.usage) if not os.path.exists(params.output.output_dir): raise Sorry("Output path not found:" + params.output.output_dir) # The convention is to put %s in the phil parameter to add a time stamp to # each output datafile. Save the initial templates here. self.strong_filename_template = params.output.strong_filename self.indexed_filename_template = params.output.indexed_filename self.refined_experiments_filename_template = params.output.refined_experiments_filename self.integrated_filename_template = params.output.integrated_filename # Don't allow the strong reflections to be written unless there are enough to # process params.output.strong_filename = None # Save the paramters self.params_cache = copy.deepcopy(params) self.options = options if params.mp.method == "mpi": from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed size = comm.Get_size() # size: number of processes running in this job elif params.mp.method == "sge" and \ 'SGE_TASK_ID' in os.environ and \ 'SGE_TASK_FIRST' in os.environ and \ 'SGE_TASK_LAST' in os.environ: if 'SGE_STEP_SIZE' in os.environ: assert int(os.environ['SGE_STEP_SIZE']) == 1 if os.environ['SGE_TASK_ID'] == 'undefined' or os.environ['SGE_TASK_ID'] == 'undefined' or os.environ['SGE_TASK_ID'] == 'undefined': rank = 0 size = 1 else: rank = int(os.environ['SGE_TASK_ID']) - int(os.environ['SGE_TASK_FIRST']) size = int(os.environ['SGE_TASK_LAST']) - int(os.environ['SGE_TASK_FIRST']) + 1 else: rank = 0 size = 1 if params.output.logging_dir is not None: log_path = os.path.join(params.output.logging_dir, "log_rank%04d.out"%rank) error_path = os.path.join(params.output.logging_dir, "error_rank%04d.out"%rank) print "Redirecting stdout to %s"%log_path print "Redirecting stderr to %s"%error_path assert os.path.exists(log_path) sys.stdout = open(log_path,'a', buffering=0) sys.stderr = open(error_path,'a',buffering=0) print "Should be redirected now" debug_dir = os.path.join(params.output.output_dir, "debug") if not os.path.exists(debug_dir): os.makedirs(debug_dir) if params.debug.skip_processed_events or params.debug.skip_processed_events or params.debug.skip_bad_events: print "Reading debug files..." self.known_events = {} for filename in os.listdir(debug_dir): # format: hostname,timestamp,status for line in open(os.path.join(debug_dir, filename)): vals = line.strip().split(',') if len(vals) == 2: self.known_events[vals[1]] = "unknown" elif len(vals) == 3: self.known_events[vals[1]] = vals[2] debug_file_path = os.path.join(debug_dir, "debug_%d.txt"%rank) write_newline = os.path.exists(debug_file_path) self.debug_file_handle = open(debug_file_path, 'a', 0) # 0 for unbuffered if write_newline: # needed if the there was a crash self.debug_file_handle.write("\n") # set up psana if params.format.file_format=="pickle": psana.setConfigFile(params.format.pickle.cfg) dataset_name = "exp=%s:run=%s:idx"%(params.input.experiment,params.input.run_num) if params.input.xtc_dir is not None: if params.input.use_ffb: raise Sorry("Cannot specify the xtc_dir and use SLAC's ffb system") dataset_name += ":dir=%s"%params.input.xtc_dir elif params.input.use_ffb: # as ffb is only at SLAC, ok to hardcode /reg/d here dataset_name += ":dir=/reg/d/ffb/%s/%s/xtc"%(params.input.experiment[0:3],params.input.experiment) ds = psana.DataSource(dataset_name) if params.format.file_format == "cbf": self.psana_det = psana.Detector(params.input.address, ds.env()) # set this to sys.maxint to analyze all events if params.dispatch.max_events is None: max_events = sys.maxint else: max_events = params.dispatch.max_events for run in ds.runs(): if params.format.file_format == "cbf": # load a header only cspad cbf from the slac metrology self.base_dxtbx = cspad_cbf_tbx.env_dxtbx_from_slac_metrology(run, params.input.address) if self.base_dxtbx is None: raise Sorry("Couldn't load calibration file for run %d"%run.run()) if params.format.cbf.gain_mask_value is not None: self.gain_mask = self.psana_det.gain_mask(gain=params.format.cbf.gain_mask_value) # list of all events times = run.times() nevents = min(len(times),max_events) if params.mp.method == "mpi" and size > 2: # use a client/server approach to be sure every process is busy as much as possible # only do this if there are more than 2 processes, as one process will be a server if rank == 0: # server process for t in times[:nevents]: # a client process will indicate it's ready by sending its rank rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send(t,dest=rankreq) # send a stop command to each process for rankreq in range(size-1): rankreq = comm.recv(source=MPI.ANY_SOURCE) comm.send('endrun',dest=rankreq) else: # client process while True: # inform the server this process is ready for an event comm.send(rank,dest=0) evttime = comm.recv(source=0) if evttime == 'endrun': break self.process_event(run, evttime) else: # chop the list into pieces, depending on rank. This assigns each process # events such that the get every Nth event where N is the number of processes mytimes = [times[i] for i in xrange(nevents) if (i+rank)%size == 0] for i in xrange(len(mytimes)): self.process_event(run, mytimes[i]) run.end() ds.end()
def run(self): ''' Parse the options. ''' from dials.util import log import libtbx from uuid import uuid4 from dials.util.stream import ZMQStream, Decoder from os.path import join, exists import os import json from dxtbx.datablock import DataBlock # Parse the command line arguments in two passes to set up logging early params, options = self.parser.parse_args(show_diff_phil=False, quick_parse=True) # Configure logging log.config( params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Parse the command line arguments completely params, options = self.parser.parse_args(show_diff_phil=False) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Check a stream is given if params.input.host is None: raise Sorry("An input host needs to be given") # Check the directory if params.output.directory is None: raise Sorry("An output directory needs to be given") elif params.output.directory is libtbx.Auto: params.output.directory = "/dev/shm/dials-%s" % uuid4() # Make the output directory if exists(params.output.directory): raise Sorry('Directory "%s" already exists' % (params.output.directory)) # Make the directory os.mkdir(params.output.directory) # Initialise the stream stream = ZMQStream(params.input.host, params.input.port) decoder = Decoder( params.output.directory, params.output.image_template) imageset = None while True: # Get the frames from zmq frames = stream.receive() # Decode the frames obj = decoder.decode(frames) # Process the object if obj.is_header(): filename = join(params.output.directory, "metadata.json") with open(filename, "w") as outfile: json.dump(obj.header, outfile) imageset = obj.as_imageset(filename) datablocks = [DataBlock([imageset])] self.write_datablocks(datablocks, params) elif obj.is_image(): assert imageset is not None filename = join( params.output.directory, params.output.image_template % obj.count) with open(filename, "wb") as outfile: outfile.write(obj.data) filename = join( params.output.directory, "%s.info" % (params.output.image_template % obj.count)) with open(filename, "w") as outfile: json.dump(obj.info, outfile) elif obj.is_endofseries(): assert imageset is not None break else: raise RuntimeError("Unknown object") # Close the stream stream.close()
def run(self): print("Parsing input") params, options = self.parser.parse_args(show_diff_phil=True) #Configure the logging log.config(params.detector_phase.refinement.verbosity, info='dials.refine.log', debug='dials.refine.debug.log') # Try to obtain the models and data if not params.input.experiments: raise Sorry("No Experiments found in the input") if not params.input.reflections: raise Sorry("No reflection data found in the input") try: assert len(params.input.reflections) == len( params.input.experiments) except AssertionError: raise Sorry( "The number of input reflections files does not match the " "number of input experiments") # set up global experiments and reflections lists from dials.array_family import flex reflections = flex.reflection_table() global_id = 0 from dxtbx.model.experiment_list import ExperimentList experiments = ExperimentList() if params.reference_detector == "first": # Use the first experiment of the first experiment list as the reference detector ref_exp = params.input.experiments[0].data[0] else: # Average all the detectors to generate a reference detector assert params.detector_phase.refinement.parameterisation.detector.hierarchy_level == 0 from scitbx.matrix import col panel_fasts = [] panel_slows = [] panel_oris = [] for exp_wrapper in params.input.experiments: exp = exp_wrapper.data[0] if panel_oris: for i, panel in enumerate(exp.detector): panel_fasts[i] += col(panel.get_fast_axis()) panel_slows[i] += col(panel.get_slow_axis()) panel_oris[i] += col(panel.get_origin()) else: for i, panel in enumerate(exp.detector): panel_fasts.append(col(panel.get_fast_axis())) panel_slows.append(col(panel.get_slow_axis())) panel_oris.append(col(panel.get_origin())) ref_exp = copy.deepcopy(params.input.experiments[0].data[0]) for i, panel in enumerate(ref_exp.detector): # Averaging the fast and slow axes can make them be non-orthagonal. Fix by finding # the vector that goes exactly between them and rotate # around their cross product 45 degrees from that vector in either direction vf = panel_fasts[i] / len(params.input.experiments) vs = panel_slows[i] / len(params.input.experiments) c = vf.cross(vs) angle = vf.angle(vs, deg=True) v45 = vf.rotate(c, angle / 2, deg=True) vf = v45.rotate(c, -45, deg=True) vs = v45.rotate(c, 45, deg=True) panel.set_frame(vf, vs, panel_oris[i] / len(params.input.experiments)) print("Reference detector (averaged):", str(ref_exp.detector)) # set the experiment factory that combines a crystal with the reference beam # and the reference detector experiment_from_crystal = ExperimentFromCrystal( ref_exp.beam, ref_exp.detector) # keep track of the number of refl per accepted experiment for a table nrefs_per_exp = [] # loop through the input, building up the global lists for ref_wrapper, exp_wrapper in zip(params.input.reflections, params.input.experiments): refs = ref_wrapper.data exps = exp_wrapper.data # there might be multiple experiments already here. Loop through them for i, exp in enumerate(exps): # select the relevant reflections sel = refs['id'] == i sub_ref = refs.select(sel) ## DGW commented out as reflections.minimum_number_of_reflections no longer exists #if len(sub_ref) < params.crystals_phase.refinement.reflections.minimum_number_of_reflections: # print "skipping experiment", i, "in", exp_wrapper.filename, "due to insufficient strong reflections in", ref_wrapper.filename # continue # build an experiment with this crystal plus the reference models combined_exp = experiment_from_crystal(exp.crystal) # next experiment ID in series exp_id = len(experiments) # check this experiment if not check_experiment(combined_exp, sub_ref): print("skipping experiment", i, "in", exp_wrapper.filename, "due to poor RMSDs") continue # set reflections ID sub_ref['id'] = flex.int(len(sub_ref), exp_id) # keep number of reflections for the table nrefs_per_exp.append(len(sub_ref)) # obtain mm positions on the reference detector sub_ref = indexer_base.map_spots_pixel_to_mm_rad( sub_ref, combined_exp.detector, combined_exp.scan) # extend refl and experiments lists reflections.extend(sub_ref) experiments.append(combined_exp) # print number of reflections per accepted experiment from libtbx.table_utils import simple_table header = ["Experiment", "Nref"] rows = [(str(i), str(n)) for (i, n) in enumerate(nrefs_per_exp)] st = simple_table(rows, header) print("Number of reflections per experiment") print(st.format()) for cycle in range(params.n_macrocycles): print("MACROCYCLE %02d" % (cycle + 1)) print("=============\n") # first run: multi experiment joint refinement of detector with fixed beam and # crystals print("PHASE 1") # SET THIS TEST TO FALSE TO REFINE WHOLE DETECTOR AS SINGLE JOB if params.detector_phase.refinement.parameterisation.detector.hierarchy_level > 0: experiments = detector_parallel_refiners( params.detector_phase, experiments, reflections) else: experiments = detector_refiner(params.detector_phase, experiments, reflections) # second run print("PHASE 2") experiments = crystals_refiner(params.crystals_phase, experiments, reflections) # Save the refined experiments to file output_experiments_filename = params.output.experiments_filename print('Saving refined experiments to {0}'.format( output_experiments_filename)) from dxtbx.model.experiment_list import ExperimentListDumper dump = ExperimentListDumper(experiments) dump.as_json(output_experiments_filename) # Write out refined reflections, if requested if params.output.reflections_filename: print('Saving refined reflections to {0}'.format( params.output.reflections_filename)) reflections.as_pickle(params.output.reflections_filename) return
def run(self): ''' Perform the integration. ''' from dials.util.command_line import heading from dials.util.options import flatten_reflections, flatten_experiments from dials.util import log from time import time from libtbx.utils import Sorry # Check the number of arguments is correct start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) reference = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(reference) == 0 and len(experiments) == 0: self.parser.print_help() return if len(reference) == 0: reference = None elif len(reference) != 1: raise Sorry('more than 1 reflection file was given') else: reference = reference[0] if len(experiments) == 0: raise Sorry('no experiment list was specified') # Save phil parameters if params.output.phil is not None: with open(params.output.phil, "w") as outfile: outfile.write(self.parser.diff_phil.as_str()) # Configure logging log.config( params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Print if we're using a mask for i, exp in enumerate(experiments): mask = exp.imageset.external_lookup.mask if mask.filename is not None: if mask.data: logger.info('Using external mask: %s' % mask.filename) logger.info(' Mask has %d pixels masked' % mask.data.count(False)) # Print the experimental models for i, exp in enumerate(experiments): logger.debug("Models for experiment %d" % i) logger.debug("") logger.debug(str(exp.beam)) logger.debug(str(exp.detector)) if exp.goniometer: logger.debug(str(exp.goniometer)) if exp.scan: logger.debug(str(exp.scan)) logger.debug(str(exp.crystal)) logger.info("=" * 80) logger.info("") logger.info(heading("Initialising")) logger.info("") # Load the data reference, rubbish = self.process_reference(reference) logger.info("") # Initialise the integrator from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Modify experiment list if scan range is set. experiments, reference = self.split_for_scan_range( experiments, reference, params.scan_range) # Predict the reflections logger.info("") logger.info("=" * 80) logger.info("") logger.info(heading("Predicting reflections")) logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static) # Match reference with predicted if reference: matched, reference, unmatched = predicted.match_with_reference(reference) assert(len(matched) == len(predicted)) assert(matched.count(True) <= len(reference)) if matched.count(True) == 0: raise Sorry(''' Invalid input for reference reflections. Zero reference spots were matched to predictions ''') elif len(unmatched) != 0: logger.info('') logger.info('*' * 80) logger.info('Warning: %d reference spots were not matched to predictions' % ( len(unmatched))) logger.info('*' * 80) logger.info('') rubbish.extend(unmatched) if len(experiments) > 1: # filter out any experiments without matched reference reflections # f_: filtered from dxtbx.model.experiment.experiment_list import ExperimentList f_reference = flex.reflection_table() f_predicted = flex.reflection_table() f_rubbish = flex.reflection_table() f_experiments = ExperimentList() good_expt_count = 0 def refl_extend(src, dest, eid): tmp = src.select(src['id'] == eid) tmp['id'] = flex.int(len(tmp), good_expt_count) dest.extend(tmp) for expt_id, experiment in enumerate(experiments): if len(reference.select(reference['id'] == expt_id)) != 0: refl_extend(reference, f_reference, expt_id) refl_extend(predicted, f_predicted, expt_id) refl_extend(rubbish, f_rubbish, expt_id) f_experiments.append(experiment) good_expt_count += 1 else: logger.info("Removing experiment %d: no reference reflections matched to predictions"%expt_id) reference = f_reference predicted = f_predicted experiments = f_experiments rubbish = f_rubbish # Select a random sample of the predicted reflections if not params.sampling.integrate_all_reflections: predicted = self.sample_predictions(experiments, predicted, params) # Compute the profile model if (params.create_profile_model and reference is not None and "shoebox" in reference): experiments = ProfileModelFactory.create(params, experiments, reference) else: for expr in experiments: if expr.profile is None: raise Sorry('No profile information in experiment list') expr.profile.params = params.profile del reference # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator logger.info("") integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() # Append rubbish data onto the end if rubbish is not None and params.output.include_bad_reference: mask = flex.bool(len(rubbish), True) rubbish.unset_flags(mask, rubbish.flags.integrated_sum) rubbish.unset_flags(mask, rubbish.flags.integrated_prf) rubbish.set_flags(mask, rubbish.flags.bad_reference) reflections.extend(rubbish) # Save the reflections self.save_reflections(reflections, params.output.reflections) self.save_experiments(experiments, params.output.experiments) # Write a report if requested if params.output.report is not None: integrator.report().as_file(params.output.report) # Print the total time taken logger.info("\nTotal time taken: %f" % (time() - start_time))
def run(args): from dials.util.options import OptionParser from dials.util.options import flatten_datablocks from dials.util.options import flatten_experiments from dials.util.options import flatten_reflections from dials.util import log import libtbx.load_env usage = "%s [options] datablock.json reflections.pickle" %( libtbx.env.dispatcher_name) parser = OptionParser( usage=usage, phil=phil_scope, read_datablocks=True, read_experiments=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args() datablocks = flatten_datablocks(params.input.datablock) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if (len(datablocks) == 0 and len(experiments) == 0) or len(reflections) == 0: parser.print_help() exit(0) # Configure the logging log.config(info='dials.rl_png.log') # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) reflections = reflections[0] if len(datablocks) == 0 and len(experiments) > 0: imagesets = experiments.imagesets() else: imagesets = [] for datablock in datablocks: imagesets.extend(datablock.extract_imagesets()) f = ReciprocalLatticePng(settings=params) f.load_models(imagesets, reflections) imageset = imagesets[0] rotation_axis = matrix.col(imageset.get_goniometer().get_rotation_axis()) s0 = matrix.col(imageset.get_beam().get_s0()) e1 = rotation_axis.normalize() e2 = s0.normalize() e3 = e1.cross(e2).normalize() #print e1 #print e2 #print e3 f.viewer.plot('rl_rotation_axis.png', n=e1.elems) f.viewer.plot('rl_beam_vector', n=e2.elems) f.viewer.plot('rl_e3.png', n=e3.elems) n_solutions = params.basis_vector_search.n_solutions if len(experiments): for i, c in enumerate(experiments.crystals()): A = c.get_A() astar = A[:3] bstar = A[3:6] cstar = A[6:9] direct_matrix = A.inverse() a = direct_matrix[:3] b = direct_matrix[3:6] c = direct_matrix[6:9] prefix = '' if len(experiments.crystals()) > 1: prefix = '%i_' %(i+1) f.viewer.plot('rl_%sa.png' %prefix, n=a) f.viewer.plot('rl_%sb.png' %prefix, n=b) f.viewer.plot('rl_%sc.png' %prefix, n=c) elif n_solutions: from dials.command_line.discover_better_experimental_model \ import run_dps, dps_phil_scope hardcoded_phil = dps_phil_scope.extract() hardcoded_phil.d_min = params.d_min result = run_dps((imagesets[0], reflections, hardcoded_phil)) solutions = [matrix.col(v) for v in result['solutions']] for i in range(min(n_solutions, len(solutions))): v = solutions[i] #if i > 0: #for v1 in solutions[:i-1]: #angle = v.angle(v1, deg=True) #print angle f.viewer.plot('rl_solution_%s.png' %(i+1), n=v.elems)
def run(args): import libtbx.load_env from libtbx.utils import Sorry from dials.util import log from logging import info usage = "%s [options] datablock.json strong.pickle" %libtbx.env.dispatcher_name parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_datablocks=True, read_experiments=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=False) # Configure the logging log.config( params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) datablocks = flatten_datablocks(params.input.datablock) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if len(datablocks) == 0: if len(experiments) > 0: imagesets = experiments.imagesets() else: parser.print_help() return #elif len(datablocks) > 1: #raise Sorry("Only one DataBlock can be processed at a time") else: imagesets = [] for datablock in datablocks: imagesets.extend(datablock.extract_imagesets()) if len(experiments): known_crystal_models = experiments.crystals() else: known_crystal_models = None if len(reflections) == 0: raise Sorry("No reflection lists found in input") if len(reflections) > 1: #raise Sorry("Multiple reflections lists provided in input") assert len(reflections) == len(imagesets) from scitbx.array_family import flex for i in range(len(reflections)): reflections[i]['imageset_id'] = flex.int(len(reflections[i]), i) if i > 0: reflections[0].extend(reflections[i]) #assert(len(reflections) == 1) reflections = reflections[0] for imageset in imagesets: if (imageset.get_goniometer() is not None and imageset.get_scan() is not None and imageset.get_scan().get_oscillation()[1] == 0): imageset.set_goniometer(None) imageset.set_scan(None) from dials.algorithms.indexing.indexer import indexer_base idxr = indexer_base.from_parameters( reflections, imagesets, known_crystal_models=known_crystal_models, params=params) refined_experiments = idxr.refined_experiments reflections = copy.deepcopy(idxr.refined_reflections) reflections.extend(idxr.unindexed_reflections) if len(refined_experiments): info("Saving refined experiments to %s" %params.output.experiments) idxr.export_as_json(refined_experiments, file_name=params.output.experiments) info("Saving refined reflections to %s" %params.output.reflections) idxr.export_reflections( reflections, file_name=params.output.reflections) if params.output.unindexed_reflections is not None: info("Saving unindexed reflections to %s" %params.output.unindexed_reflections) idxr.export_reflections(idxr.unindexed_reflections, file_name=params.output.unindexed_reflections) return
def run(self, args): from dials.util.version import dials_version from i19.util.version import i19_version version_information = "%s using %s (%s)" % (i19_version(), dials_version(), time.strftime("%Y-%m-%d %H:%M:%S")) start = timeit.default_timer() if len(args) == 0: print help_message print version_information return # Configure the logging from dials.util import log log.config(1, info='i19.screen.log', debug='i19.screen.debug.log') info(version_information) debug('Run with %s' % str(args)) # FIXME use proper optionparser here. This works for now nproc = None if len(args) >= 1 and args[0].startswith('nproc='): nproc = args[0][6:] args = args[1:] self._count_processors(nproc=nproc) debug('Using %s processors' % self.nproc) if len(args) == 1 and args[0].endswith('.json'): self.json_file = args[0] else: self._import(args) self.json_file = 'datablock.json' self._find_spots() if not self._index(): info("\nRetrying for stronger spots only...") os.rename("strong.pickle", "all_spots.pickle") self._find_spots(['sigma_strong=15']) if not self._index(): warn("Giving up.") info(""" Could not find an indexing solution. You may want to have a look at the reciprocal space by running: dials.reciprocal_lattice_viewer datablock.json all_spots.pickle or, to only include stronger spots: dials.reciprocal_lattice_viewer datablock.json strong.pickle """) sys.exit(1) if not self._create_profile_model(): info("\nRefining model to attempt to increase number of valid spots...") self._refine() if not self._create_profile_model(): warn("Giving up.") info(""" The identified indexing solution may not be correct. You may want to have a look at the reciprocal space by running: dials.reciprocal_lattice_viewer experiments.json indexed.pickle """) sys.exit(1) self._report() self._predict() self._check_intensities() self._refine_bravais() i19screen_runtime = timeit.default_timer() - start debug("Finished at %s, total runtime: %.1f" % (time.strftime("%Y-%m-%d %H:%M:%S"), i19screen_runtime)) info("i19.screen successfully completed (%.1f sec)" % i19screen_runtime)
def run(self): '''Execute the script.''' from dials.algorithms.refinement.two_theta_refiner import \ TwoThetaReflectionManager, TwoThetaTarget, \ TwoThetaPredictionParameterisation start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # set up global experiments and reflections lists from dials.array_family import flex reflections = flex.reflection_table() global_id = 0 from dxtbx.model.experiment.experiment_list import ExperimentList experiments=ExperimentList() # loop through the input, building up the global lists nrefs_per_exp = [] for ref_wrapper, exp_wrapper in zip(params.input.reflections, params.input.experiments): refs = ref_wrapper.data exps = exp_wrapper.data for i, exp in enumerate(exps): sel = refs['id'] == i sub_ref = refs.select(sel) nrefs_per_exp.append(len(sub_ref)) sub_ref['id'] = flex.int(len(sub_ref), global_id) reflections.extend(sub_ref) experiments.append(exp) global_id += 1 # Try to load the models and data nexp = len(experiments) if nexp == 0: print "No Experiments found in the input" self.parser.print_help() return if len(reflections) == 0: print "No reflection data found in the input" self.parser.print_help() return self.check_input(reflections) # Configure the logging log.config(info=params.output.log, debug=params.output.debug_log) logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Convert to P 1? if params.refinement.triclinic: reflections, experiments = self.convert_to_P1(reflections, experiments) # Combine crystals? if params.refinement.combine_crystal_models and len(experiments) > 1: logger.info('Combining {0} crystal models'.format(len(experiments))) experiments = self.combine_crystals(experiments) # Filter integrated centroids? if params.refinement.filter_integrated_centroids: reflections = self.filter_integrated_centroids(reflections) # Get the refiner logger.info('Configuring refiner') refiner = self.create_refiner(params, reflections, experiments) # Refine the geometry if nexp == 1: logger.info('Performing refinement of a single Experiment...') else: logger.info('Performing refinement of {0} Experiments...'.format(nexp)) # Refine and get the refinement history history = refiner.run() # get the refined experiments experiments = refiner.get_experiments() crystals = experiments.crystals() if len(crystals) == 1: # output the refined model for information logger.info('') logger.info('Final refined crystal model:') logger.info(crystals[0]) logger.info(self.cell_param_table(crystals[0])) # Save the refined experiments to file output_experiments_filename = params.output.experiments logger.info('Saving refined experiments to {0}'.format(output_experiments_filename)) from dxtbx.model.experiment.experiment_list import ExperimentListDumper dump = ExperimentListDumper(experiments) dump.as_json(output_experiments_filename) # Correlation plot if params.output.correlation_plot.filename is not None: from os.path import splitext root, ext = splitext(params.output.correlation_plot.filename) if not ext: ext = ".pdf" steps = params.output.correlation_plot.steps if steps is None: steps = [history.get_nrows()-1] # extract individual column names or indices col_select = params.output.correlation_plot.col_select num_plots = 0 for step in steps: fname_base = root if len(steps) > 1: fname_base += "_step%02d" % step plot_fname = fname_base + ext corrmat, labels = refiner.get_parameter_correlation_matrix(step, col_select) if [corrmat, labels].count(None) == 0: from dials.algorithms.refinement.refinement_helpers import corrgram plt = corrgram(corrmat, labels) if plt is not None: logger.info('Saving parameter correlation plot to {}'.format(plot_fname)) plt.savefig(plot_fname) num_plots += 1 mat_fname = fname_base + ".pickle" with open(mat_fname, 'wb') as handle: py_mat = corrmat.as_scitbx_matrix() #convert to pickle-friendly form logger.info('Saving parameter correlation matrix to {0}'.format(mat_fname)) pickle.dump({'corrmat':py_mat, 'labels':labels}, handle) if num_plots == 0: msg = "Sorry, no parameter correlation plots were produced. Please set " \ "track_parameter_correlation=True to ensure correlations are " \ "tracked, and make sure correlation_plot.col_select is valid." logger.info(msg) if params.output.cif is not None: self.generate_cif(crystals[0], refiner, file=params.output.cif) if params.output.p4p is not None: self.generate_p4p(crystals[0], experiments[0].beam, file=params.output.p4p) if params.output.mmcif is not None: self.generate_mmcif(crystals[0], refiner, file=params.output.mmcif) # Log the total time taken logger.info("\nTotal time taken: {0:.2f}s".format(time() - start_time))
def run(self): '''Execute the script.''' from dials.array_family import flex from dials.util.options import flatten_datablocks from time import time from dials.util import log from libtbx.utils import Sorry import datetime start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure the logging log.config( params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Ensure we have a data block datablocks = flatten_datablocks(params.input.datablock) if len(datablocks) == 0: self.parser.print_help() return # Extend the first datablock datablock = datablocks[0] for db in datablocks[1:]: if datablock.format_class() != db.format_class(): raise Sorry("Datablocks must have the same format") datablock.extend(db) # Get the imagesets and sweeps stills = datablock.extract_stills() sweeps = datablock.extract_sweeps() if len(stills) > 0: raise Sorry("Sets of still images are currently unsupported") logger.info("Number of sweeps = %d" % len(sweeps)) # Sort the sweeps by timestamps logger.info("Sorting sweeps based on timestamp") sweeps = sorted(sweeps, key=lambda x: x.get_scan().get_epochs()[0]) # Count the number of datasets from each day from collections import Counter counter = Counter() for s in sweeps: timestamp = s.get_scan().get_epochs()[0] timestamp = datetime.datetime.fromtimestamp(timestamp) timestamp = timestamp.strftime('%Y-%m-%d') counter[timestamp] += 1 # Print the number of datasets on each day for timestamp in sorted(counter.keys()): logger.info("%d datasets collected on %s" % (counter[timestamp], timestamp)) # Loop though and see if any models might be shared b_list = [ s.get_beam() for s in sweeps ] d_list = [ s.get_detector() for s in sweeps ] g_list = [ s.get_goniometer() for s in sweeps ] b_index = [] d_index = [] g_index = [] for i in range(len(sweeps)): b = b_list[i] d = d_list[i] g = g_list[i] bn = i dn = i gn = i if i > 0: bj = b_index[-1] dj = d_index[-1] gj = g_index[-1] if b.is_similar_to(b_list[bj]): bn = bj if d.is_similar_to(d_list[dj]): dn = dj if g.is_similar_to(g_list[gj]): gn = gj b_index.append(bn) d_index.append(dn) g_index.append(gn) # Print a table of possibly shared models from libtbx.table_utils import format as table rows = [["Sweep", "ID", "Beam", "Detector", "Goniometer", "Date", "Time"]] for i in range(len(sweeps)): timestamp = sweeps[i].get_scan().get_epochs()[0] timestamp = datetime.datetime.fromtimestamp(timestamp) date_str = timestamp.strftime('%Y-%m-%d') time_str = timestamp.strftime('%H:%M:%S') row = [ '%s' % sweeps[i].get_template(), '%s' % i, '%s' % b_index[i], '%s' % d_index[i], '%s' % g_index[i], '%s' % date_str, '%s' % time_str] rows.append(row) logger.info(table(rows, has_header=True, justify='left', prefix=' ')) # Print the time logger.info("Time Taken: %f" % (time() - start_time))
def run(self): print "Parsing input" params, options = self.parser.parse_args(show_diff_phil=True) #Configure the logging log.config(params.detector_phase.refinement.verbosity, info='dials.refine.log', debug='dials.refine.debug.log') # Try to obtain the models and data if not params.input.experiments: raise Sorry("No Experiments found in the input") if not params.input.reflections: raise Sorry("No reflection data found in the input") try: assert len(params.input.reflections) == len(params.input.experiments) except AssertionError: raise Sorry("The number of input reflections files does not match the " "number of input experiments") # set up global experiments and reflections lists from dials.array_family import flex reflections = flex.reflection_table() global_id = 0 from dxtbx.model.experiment.experiment_list import ExperimentList experiments=ExperimentList() if params.reference_detector == "first": # Use the first experiment of the first experiment list as the reference detector ref_exp = params.input.experiments[0].data[0] else: # Average all the detectors to generate a reference detector assert params.detector_phase.refinement.parameterisation.detector.hierarchy_level == 0 from scitbx.matrix import col panel_fasts = [] panel_slows = [] panel_oris = [] for exp_wrapper in params.input.experiments: exp = exp_wrapper.data[0] if panel_oris: for i, panel in enumerate(exp.detector): panel_fasts[i] += col(panel.get_fast_axis()) panel_slows[i] += col(panel.get_slow_axis()) panel_oris[i] += col(panel.get_origin()) else: for i, panel in enumerate(exp.detector): panel_fasts.append(col(panel.get_fast_axis())) panel_slows.append(col(panel.get_slow_axis())) panel_oris.append(col(panel.get_origin())) ref_exp = copy.deepcopy(params.input.experiments[0].data[0]) for i, panel in enumerate(ref_exp.detector): # Averaging the fast and slow axes can make them be non-orthagonal. Fix by finding # the vector that goes exactly between them and rotate # around their cross product 45 degrees from that vector in either direction vf = panel_fasts[i]/len(params.input.experiments) vs = panel_slows[i]/len(params.input.experiments) c = vf.cross(vs) angle = vf.angle(vs, deg=True) v45 = vf.rotate(c, angle/2, deg=True) vf = v45.rotate(c, -45, deg=True) vs = v45.rotate(c, 45, deg=True) panel.set_frame(vf, vs, panel_oris[i]/len(params.input.experiments)) print "Reference detector (averaged):", str(ref_exp.detector) # set the experiment factory that combines a crystal with the reference beam # and the reference detector experiment_from_crystal=ExperimentFromCrystal(ref_exp.beam, ref_exp.detector) # keep track of the number of refl per accepted experiment for a table nrefs_per_exp = [] # loop through the input, building up the global lists for ref_wrapper, exp_wrapper in zip(params.input.reflections, params.input.experiments): refs = ref_wrapper.data exps = exp_wrapper.data # there might be multiple experiments already here. Loop through them for i, exp in enumerate(exps): # select the relevant reflections sel = refs['id'] == i sub_ref = refs.select(sel) ## DGW commented out as reflections.minimum_number_of_reflections no longer exists #if len(sub_ref) < params.crystals_phase.refinement.reflections.minimum_number_of_reflections: # print "skipping experiment", i, "in", exp_wrapper.filename, "due to insufficient strong reflections in", ref_wrapper.filename # continue # build an experiment with this crystal plus the reference models combined_exp = experiment_from_crystal(exp.crystal) # next experiment ID in series exp_id = len(experiments) # check this experiment if not check_experiment(combined_exp, sub_ref): print "skipping experiment", i, "in", exp_wrapper.filename, "due to poor RMSDs" continue # set reflections ID sub_ref['id'] = flex.int(len(sub_ref), exp_id) # keep number of reflections for the table nrefs_per_exp.append(len(sub_ref)) # obtain mm positions on the reference detector sub_ref = indexer_base.map_spots_pixel_to_mm_rad(sub_ref, combined_exp.detector, combined_exp.scan) # extend refl and experiments lists reflections.extend(sub_ref) experiments.append(combined_exp) # print number of reflections per accepted experiment from libtbx.table_utils import simple_table header = ["Experiment", "Nref"] rows = [(str(i), str(n)) for (i, n) in enumerate(nrefs_per_exp)] st = simple_table(rows, header) print "Number of reflections per experiment" print st.format() for cycle in range(params.n_macrocycles): print "MACROCYCLE %02d" % (cycle + 1) print "=============\n" # first run: multi experiment joint refinement of detector with fixed beam and # crystals print "PHASE 1" # SET THIS TEST TO FALSE TO REFINE WHOLE DETECTOR AS SINGLE JOB if params.detector_phase.refinement.parameterisation.detector.hierarchy_level > 0: experiments = detector_parallel_refiners(params.detector_phase, experiments, reflections) else: experiments = detector_refiner(params.detector_phase, experiments, reflections) # second run print "PHASE 2" experiments = crystals_refiner(params.crystals_phase, experiments, reflections) # Save the refined experiments to file output_experiments_filename = params.output.experiments_filename print 'Saving refined experiments to {0}'.format(output_experiments_filename) from dxtbx.model.experiment.experiment_list import ExperimentListDumper dump = ExperimentListDumper(experiments) dump.as_json(output_experiments_filename) # Write out refined reflections, if requested if params.output.reflections_filename: print 'Saving refined reflections to {0}'.format( params.output.reflections_filename) reflections.as_pickle(params.output.reflections_filename) return
def run(self): """ Perform the integration. """ from dials.util.command_line import heading from dials.util.options import flatten_reflections, flatten_experiments from dials.util import log from logging import info, debug from time import time from libtbx.utils import Sorry # Check the number of arguments is correct start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) reference = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(reference) == 0 and len(experiments) == 0: self.parser.print_help() return if len(reference) == 0: reference = None elif len(reference) != 1: raise Sorry("more than 1 reflection file was given") else: reference = reference[0] if len(experiments) == 0: raise Sorry("no experiment list was specified") # Save phil parameters if params.output.phil is not None: with open(params.output.phil, "w") as outfile: outfile.write(self.parser.diff_phil.as_str()) # Configure logging log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not "": info("The following parameters have been modified:\n") info(diff_phil) # Print if we're using a mask for i, exp in enumerate(experiments): mask = exp.imageset.external_lookup.mask if mask.filename is not None: info("Using external mask: %s" % mask.filename) info(" Mask has %d pixels masked" % mask.data.count(False)) # Print the experimental models for i, exp in enumerate(experiments): debug("Models for experiment %d" % i) debug("") debug(str(exp.beam)) debug(str(exp.detector)) if exp.goniometer: debug(str(exp.goniometer)) if exp.scan: debug(str(exp.scan)) debug(str(exp.crystal)) info("=" * 80) info("") info(heading("Initialising")) info("") # Load the data reference, rubbish = self.process_reference(reference) info("") # Initialise the integrator from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.algorithms.integration.integrator import IntegratorFactory from dials.array_family import flex # Modify experiment list if scan range is set. experiments, reference = self.split_for_scan_range(experiments, reference, params.scan_range) # Predict the reflections info("") info("=" * 80) info("") info(heading("Predicting reflections")) info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static, ) # Match reference with predicted if reference: matched, reference, unmatched = predicted.match_with_reference(reference) assert len(matched) == len(predicted) assert matched.count(True) <= len(reference) if matched.count(True) == 0: raise Sorry( """ Invalid input for reference reflections. Zero reference spots were matched to predictions """ ) elif len(unmatched) != 0: info("") info("*" * 80) info("Warning: %d reference spots were not matched to predictions" % (len(unmatched))) info("*" * 80) info("") rubbish.extend(unmatched) # Select a random sample of the predicted reflections if not params.sampling.integrate_all_reflections: predicted = self.sample_predictions(experiments, predicted, params) # Compute the profile model if reference is not None and params.create_profile_model: experiments = ProfileModelFactory.create(params, experiments, reference) else: for expr in experiments: expr.profile.params = params.profile del reference # Compute the bounding box predicted.compute_bbox(experiments) # Create the integrator info("") integrator = IntegratorFactory.create(params, experiments, predicted) # Integrate the reflections reflections = integrator.integrate() # Append rubbish data onto the end if rubbish is not None and params.output.include_bad_reference: mask = flex.bool(len(rubbish), True) rubbish.unset_flags(mask, rubbish.flags.integrated_sum) rubbish.unset_flags(mask, rubbish.flags.integrated_prf) rubbish.set_flags(mask, rubbish.flags.bad_reference) reflections.extend(rubbish) # Save the reflections self.save_reflections(reflections, params.output.reflections) self.save_experiments(experiments, params.output.experiments) # Write a report if requested if params.output.report is not None: integrator.report().as_file(params.output.report) # Print the total time taken info("\nTotal time taken: %f" % (time() - start_time))
def run(self): '''Execute the script.''' from time import time import cPickle as pickle from logging import info from dials.util import log from dials.algorithms.refinement import RefinerFactory from dials.util.options import flatten_reflections, flatten_experiments start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) # Try to load the models and data nexp = len(experiments) if nexp == 0: print "No Experiments found in the input" self.parser.print_help() return if len(reflections) == 0: print "No reflection data found in the input" self.parser.print_help() return if len(reflections) > 1: raise Sorry("Only one reflections list can be imported at present") reflections = reflections[0] self.check_input(reflections) # Configure the logging log.config(info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) # Modify options if necessary if params.output.correlation_plot.filename is not None: params.refinement.refinery.track_parameter_correlation = True # Get the refiner info('Configuring refiner') refiner = RefinerFactory.from_parameters_data_experiments(params, reflections, experiments) # Refine the geometry if nexp == 1: info('Performing refinement of a single Experiment...') else: info('Performing refinement of {0} Experiments...'.format(nexp)) # Refine and get the refinement history history = refiner.run() if params.output.centroids: info("Writing table of centroids to '{0}'".format( params.output.centroids)) self.write_centroids_table(refiner, params.output.centroids) # Write scan-varying parameters to file, if there were any if params.output.parameter_table: scan = refiner.get_scan() if scan: text = refiner.get_param_reporter().varying_params_vs_image_number( scan.get_array_range()) if text: info("Writing scan-varying parameter table to {0}".format( params.output.parameter_table)) f = open(params.output.parameter_table,"w") f.write(text) f.close() else: info("No scan-varying parameter table to write") # get the refined experiments experiments = refiner.get_experiments() crystals = experiments.crystals() if len(crystals) == 1: # output the refined model for information info('') info('Final refined crystal model:') info(crystals[0]) # Save the refined experiments to file output_experiments_filename = params.output.experiments info('Saving refined experiments to {0}'.format(output_experiments_filename)) from dxtbx.model.experiment.experiment_list import ExperimentListDumper dump = ExperimentListDumper(experiments) dump.as_json(output_experiments_filename) # Save reflections with updated predictions if requested (allow to switch # this off if it is a time-consuming step) if params.output.reflections: # Update predictions for all indexed reflections info('Updating predictions for indexed reflections') preds = refiner.predict_for_indexed() # just copy over the columns of interest, leaving behind things # added by e.g. scan-varying refinement such as 'block', the # U, B and UB matrices and gradients. reflections['s1'] = preds['s1'] reflections['xyzcal.mm'] = preds['xyzcal.mm'] reflections['xyzcal.px'] = preds['xyzcal.px'] if preds.has_key('entering'): reflections['entering'] = preds['entering'] # set used_in_refinement and centroid_outlier flags assert len(preds) == len(reflections) reflections.unset_flags(flex.size_t_range(len(reflections)), reflections.flags.used_in_refinement | reflections.flags.centroid_outlier) mask = preds.get_flags(preds.flags.centroid_outlier) reflections.set_flags(mask, reflections.flags.centroid_outlier) mask = preds.get_flags(preds.flags.used_in_refinement) reflections.set_flags(mask, reflections.flags.used_in_refinement) info('Saving reflections with updated predictions to {0}'.format( params.output.reflections)) if params.output.include_unused_reflections: reflections.as_pickle(params.output.reflections) else: sel = reflections.get_flags(reflections.flags.used_in_refinement) reflections.select(sel).as_pickle(params.output.reflections) # For debugging, if requested save matches to file if params.output.matches: matches = refiner.get_matches() info('Saving matches (use for debugging purposes) to {0}'.format( params.output.matches)) matches.as_pickle(params.output.matches) # Correlation plot if params.output.correlation_plot.filename is not None: from os.path import splitext root, ext = splitext(params.output.correlation_plot.filename) if not ext: ext = ".pdf" steps = params.output.correlation_plot.steps if steps is None: steps = [history.get_nrows()-1] # extract individual column names or indices col_select = params.output.correlation_plot.col_select num_plots = 0 for step in steps: fname_base = root + "_step%02d" % step plot_fname = fname_base + ext corrmat, labels = refiner.get_parameter_correlation_matrix(step, col_select) if [corrmat, labels].count(None) == 0: from dials.algorithms.refinement.refinement_helpers import corrgram plt = corrgram(corrmat, labels) if plt is not None: info('Saving parameter correlation plot to {}'.format(plot_fname)) plt.savefig(plot_fname) num_plots += 1 mat_fname = fname_base + ".pickle" with open(mat_fname, 'wb') as handle: py_mat = corrmat.as_scitbx_matrix() #convert to pickle-friendly form info('Saving parameter correlation matrix to {0}'.format(mat_fname)) pickle.dump({'corrmat':py_mat, 'labels':labels}, handle) if num_plots == 0: msg = "Sorry, no parameter correlation plots were produced. Please set " \ "track_parameter_correlation=True to ensure correlations are " \ "tracked, and make sure correlation_plot.col_select is valid." info(msg) # Write out refinement history, if requested if params.output.history: with open(params.output.history, 'wb') as handle: info('Saving refinement step history to {0}'.format( params.output.history)) pickle.dump(history, handle) # Log the total time taken info("\nTotal time taken: {0:.2f}s".format(time() - start_time)) return
def run(self): '''Execute the script.''' from dxtbx.datablock import DataBlockTemplateImporter from dials.util.options import flatten_datablocks from dials.util import log from logging import info from time import time from libtbx.utils import Abort # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) datablocks = flatten_datablocks(params.input.datablock) # Check we have some filenames if len(datablocks) == 0: # Check if a template has been set and print help if not, otherwise try to # import the images based on the template input if len(params.input.template) == 0: self.parser.print_help() exit(0) else: importer = DataBlockTemplateImporter( params.input.template, options.verbose) datablocks = importer.datablocks # Save the options self.options = options self.params = params self.load_reference_geometry() st = time() # Import stuff if len(datablocks) == 0: raise Abort('No datablocks specified') elif len(datablocks) > 1: raise Abort('Only 1 datablock can be processed at a time.') datablock = datablocks[0] if self.reference_detector is not None: for imageset in datablock.extract_imagesets(): imageset.set_detector(self.reference_detector) # Configure logging log.config( params.verbosity, info='dials.process.log', debug='dials.process.debug.log') # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) if self.params.output.datablock_filename: from dxtbx.datablock import DataBlockDumper dump = DataBlockDumper(datablock) dump.as_json(self.params.output.datablock_filename) # Do the processing observed = self.find_spots(datablock) experiments, indexed = self.index(datablock, observed) experiments = self.refine(experiments, indexed) integrated = self.integrate(experiments, indexed) # Total Time info("") info("Total Time Taken = %f seconds" % (time() - st))
def run(args): from dials.util.options import OptionParser from dials.util.options import flatten_datablocks from dials.util.options import flatten_experiments from dials.util.options import flatten_reflections from dials.util import log import libtbx.load_env usage = "%s [options] datablock.json reflections.pickle" % (libtbx.env.dispatcher_name) parser = OptionParser( usage=usage, phil=phil_scope, read_datablocks=True, read_experiments=True, read_reflections=True, check_format=False, epilog=help_message, ) params, options = parser.parse_args() datablocks = flatten_datablocks(params.input.datablock) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if (len(datablocks) == 0 and len(experiments) == 0) or len(reflections) == 0: parser.print_help() exit(0) # Configure the logging log.config(info="dials.detect_blanks.log") # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) reflections = reflections[0] if len(datablocks) == 0 and len(experiments) > 0: imagesets = experiments.imagesets() else: imagesets = [] for datablock in datablocks: imagesets.extend(datablock.extract_imagesets()) assert len(imagesets) == 1 imageset = imagesets[0] scan = imageset.get_scan() integrated_sel = reflections.get_flags(reflections.flags.integrated) indexed_sel = reflections.get_flags(reflections.flags.indexed) centroid_outlier_sel = reflections.get_flags(reflections.flags.centroid_outlier) strong_sel = reflections.get_flags(reflections.flags.strong) indexed_sel &= ~centroid_outlier_sel logger.info("Analysis of %i strong reflections:" % strong_sel.count(True)) strong_results = blank_counts_analysis( reflections.select(strong_sel), scan, phi_step=params.phi_step, fractional_loss=params.counts_fractional_loss ) for blank_start, blank_end in strong_results["blank_regions"]: logger.info("Potential blank images: %i -> %i" % (blank_start + 1, blank_end)) indexed_results = None if indexed_sel.count(True) > 0: logger.info("Analysis of %i indexed reflections:" % indexed_sel.count(True)) indexed_results = blank_counts_analysis( reflections.select(indexed_sel), scan, phi_step=params.phi_step, fractional_loss=params.counts_fractional_loss, ) for blank_start, blank_end in indexed_results["blank_regions"]: logger.info("Potential blank images: %i -> %i" % (blank_start + 1, blank_end)) integrated_results = None if integrated_sel.count(True) > 0: logger.info("Analysis of %i integrated reflections:" % integrated_sel.count(True)) integrated_results = blank_integrated_analysis( reflections.select(integrated_sel), scan, phi_step=params.phi_step, fractional_loss=params.misigma_fractional_loss, ) for blank_start, blank_end in integrated_results["blank_regions"]: logger.info("Potential blank images: %i -> %i" % (blank_start + 1, blank_end)) d = {"strong": strong_results, "indexed": indexed_results, "integrated": integrated_results} if params.output.json is not None: import json with open(params.output.json, "wb") as f: json.dump(d, f) if params.output.plot: from matplotlib import pyplot plots = [(strong_results, "-")] if indexed_results: plots.append((indexed_results, "--")) if integrated_results: plots.append((integrated_results, ":")) for results, linestyle in plots: xs = results["data"][0]["x"] ys = results["data"][0]["y"] xmax = max(xs) ymax = max(ys) xs = [x / xmax for x in xs] ys = [y / ymax for y in ys] blanks = results["data"][0]["blank"] pyplot.plot(xs, ys, color="blue", linestyle=linestyle) pyplot.plot( *zip(*[(x, y) for x, y, blank in zip(xs, ys, blanks) if blank]), color="red", linestyle=linestyle ) pyplot.ylim(0) pyplot.show() pyplot.clf()
def run(self): '''Execute the script.''' from dials.util.command_line import heading from dials.array_family import flex from dials.util.options import flatten_experiments from time import time from dials.util import log from libtbx.utils import Sorry from dials.algorithms.background.modeller import BackgroundModeller start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure the logging log.config( params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Ensure we have a data block experiments = flatten_experiments(params.input.experiments) if len(experiments) == 0: self.parser.print_help() return # Only handle a single imageset at once imagesets = set(expr.imageset for expr in experiments) if len(imagesets) != 1: raise Sorry("Can only process a single imageset at a time") # Predict the reflections logger.info("") logger.info("=" * 80) logger.info("") logger.info(heading("Predicting reflections")) logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static) # Create the modeller modeller = BackgroundModeller(experiments, predicted, params) model = modeller.compute() # Save the background model logger.info("Saving background model to %s" % params.output.model) from dials.algorithms.background.gmodel import StaticBackgroundModel static_model = StaticBackgroundModel() for i in range(len(model)): static_model.add(model[i].model) with open(params.output.model, "w") as outfile: import cPickle as pickle pickle.dump(static_model, outfile, protocol=pickle.HIGHEST_PROTOCOL) # Output some diagnostic images image_generator = ImageGenerator(model) image_generator.save_mean(params.output.mean_image_prefix) image_generator.save_variance(params.output.variance_image_prefix) image_generator.save_dispersion(params.output.dispersion_image_prefix) image_generator.save_mask(params.output.mask_image_prefix) image_generator.save_min(params.output.min_image_prefix) image_generator.save_max(params.output.max_image_prefix) image_generator.save_model(params.output.model_image_prefix) #image_generator.save_polar_model(params.output.polar_model_image_prefix) # Print the time logger.info("Time Taken: %f" % (time() - start_time))
def run(self): ''' Perform the integration. ''' from dials.util.command_line import heading from dials.util.options import flatten_datablocks, flatten_experiments from dials.util import log from time import time from libtbx.utils import Sorry from dials.array_family import flex # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) experiments = flatten_experiments(params.input.experiments) datablocks = flatten_datablocks(params.input.datablock) if len(experiments) == 0 and len(datablocks) == 0: self.parser.print_help() return if len(datablocks) > 0: assert len(datablocks) == 1 imagesets = datablocks[0].extract_imagesets() assert len(imagesets) == 1 imageset = imagesets[0] beam = imageset.get_beam() detector = imageset.get_detector() else: assert len(experiments) == 1 imageset = experiments[0].imageset beam = experiments[0].beam detector = experiments[0].detector # Configure logging log.config() # Set the scan range if params.scan_range is None: scan_range = (0, len(imageset)) else: scan_range = params.scan_range i0, i1 = scan_range if i0 < 0 or i1 > len(imageset): raise RuntimeError('Scan range outside image range') if i0 >= i1: raise RuntimeError('Invalid scan range') summed_data = None summed_mask = None # Loop through images for i in range(*scan_range): logger.info("Reading image %d" % i) # Read image data = imageset.get_raw_data(i) mask = imageset.get_mask(i) assert isinstance(data, tuple) assert isinstance(mask, tuple) if summed_data is None: summed_mask = mask summed_data = data else: summed_data = [ sd + d for sd, d in zip(summed_data, data) ] summed_mask = [ sm & m for sm, m in zip(summed_mask, mask) ] # Compute min and max and num if params.num_bins is None: num_bins = sum(sum(p.get_image_size()) for p in detector) if params.d_max is None: vmin = 0 else: vmin = (1.0 / d_max)**2 if params.d_min is None: params.d_min = detector.get_max_resolution(beam.get_s0()) vmax = (1.0 / params.d_min)**2 # Print some info logger.info("Min 1/d^2: %f" % vmin) logger.info("Max 1/d^2: %f" % vmax) logger.info("Num bins: %d" % num_bins) # Compute the radial average from dials.algorithms.background import RadialAverage radial_average = RadialAverage(beam, detector, vmin, vmax, num_bins) for d, m in zip(summed_data, summed_mask): radial_average.add(d.as_double(), m) mean = radial_average.mean() reso = radial_average.inv_d2() logger.info("Writing to %s" % params.output.filename) with open(params.output.filename, "w") as outfile: for r, m in zip(reso, mean): outfile.write("%f, %f\n" % (r, m))
# Create the option parser parser = OptionParser( usage=usage, read_experiments=True, read_reflections=True, read_datablocks=True, phil=phil_scope, epilog=help_message) # Get the parameters params, options = parser.parse_args(show_diff_phil=False) # Configure the logging log.config( info=params.output.log, debug=params.output.debug_log) # Print the version number logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Get the experiments and reflections datablocks = flatten_datablocks(params.input.datablock) experiments = flatten_experiments(params.input.experiments)
def run(self): '''Execute the script.''' from dials.util import log from logging import info from time import time from libtbx.utils import Abort from libtbx import easy_mp import os, copy from dxtbx.datablock import DataBlockFactory # Parse the command line params, options, all_paths = self.parser.parse_args(show_diff_phil=False, return_unhandled=True) # Check we have some filenames if len(all_paths) == 0: self.parser.print_help() return # Save the options self.options = options self.params = params st = time() # Configure logging log.config( params.verbosity, info='dials.process.log', debug='dials.process.debug.log') # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) # Import stuff info("Loading files...") if len(all_paths) == 1: datablocks = DataBlockFactory.from_filenames(all_paths) else: def do_import(filename): info("Loading %s"%os.path.basename(filename)) datablocks = DataBlockFactory.from_filenames([filename]) if len(datablocks) == 0: raise Abort("Could not load %s"%filename) if len(datablocks) > 1: raise Abort("Got multiple datablocks from file %s"%filename) return datablocks[0] datablocks = easy_mp.parallel_map( func=do_import, iterable=all_paths, processes=params.mp.nproc, method=params.mp.method, preserve_order=True, preserve_exception_message=True) if len(datablocks) == 0: raise Abort('No datablocks specified') # Handle still imagesets by breaking them apart into multiple datablocks # Further handle single file still imagesets (like HDF5) by tagging each # frame using its index indices = [] basenames = [] split_datablocks = [] for datablock in datablocks: for imageset in datablock.extract_imagesets(): for i in xrange(len(imageset)): subset = imageset[i:i+1] split_datablocks.append(DataBlockFactory.from_imageset(subset)[0]) indices.append(i) basenames.append(os.path.splitext(os.path.basename(subset.paths()[0]))[0]) tags = [] for i, basename in zip(indices, basenames): if basenames.count(basename) > 1: tags.append("%s_%d"%(basename, i)) else: tags.append(basename) # Wrapper function def do_work(item): Processor(copy.deepcopy(params)).process_datablock(item[0], item[1]) # Process the data easy_mp.parallel_map( func=do_work, iterable=zip(tags, split_datablocks), processes=params.mp.nproc, method=params.mp.method, preserve_order=True, preserve_exception_message=True) # Total Time info("") info("Total Time Taken = %f seconds" % (time() - st))
def run(self): ''' Parse the options. ''' from dials.util import log # Parse the command line arguments in two passes to set up logging early params, options = self.parser.parse_args(show_diff_phil=False, quick_parse=True) # Configure logging log.config( params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Parse the command line arguments completely if params.input.ignore_unhandled: params, options, unhandled = self.parser.parse_args( show_diff_phil=False, return_unhandled=True) else: params, options = self.parser.parse_args(show_diff_phil=False) unhandled = None # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Print a warning if something unhandled if unhandled is not None and len(unhandled) > 0: msg = 'Unable to handle the following arguments:\n' msg += '\n'.join([' %s' % a for a in unhandled]) msg += '\n' logger.warn(msg) # Print help if no input if (len(params.input.datablock) == 0 and not (params.input.template or params.input.directory)): self.parser.print_help() return # Setup the datablock importer datablock_importer = DataBlockImporter(params) # Setup the metadata updater metadata_updater = MetaDataUpdater(params) # Get the datablocks datablock = metadata_updater(datablock_importer()) # Extract any sweeps sweeps = datablock.extract_sweeps() # Extract any stills stills = datablock.extract_stills() if not stills: num_stills = 0 else: num_stills = sum([len(s) for s in stills]) # Print some data block info - override the output of image range # if appropriate image_range = params.geometry.scan.image_range logger.info("-" * 80) logger.info(" format: %s" % str(datablock.format_class())) if image_range is None: logger.info(" num images: %d" % datablock.num_images()) else: logger.info(" num images: %d" % (image_range[1] - image_range[0] + 1)) logger.info(" num sweeps: %d" % len(sweeps)) logger.info(" num stills: %d" % num_stills) # Loop through all the sweeps for j, sweep in enumerate(sweeps): logger.debug("") logger.debug("Sweep %d" % j) logger.debug(" Length %d" % len(sweep)) logger.debug(sweep.get_beam()) logger.debug(sweep.get_goniometer()) logger.debug(sweep.get_detector()) logger.debug(sweep.get_scan()) # Only allow a single sweep if params.input.allow_multiple_sweeps is False: self.assert_single_sweep(sweeps, params) # Write the datablocks to file self.write_datablocks([datablock], params)
def run(self): '''Execute the script.''' from dials.array_family import flex from dials.util.options import flatten_datablocks from time import time from dials.util import log from libtbx.utils import Sorry start_time = time() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure the logging log.config( params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) # Ensure we have a data block datablocks = flatten_datablocks(params.input.datablock) if len(datablocks) == 0: self.parser.print_help() return elif len(datablocks) != 1: raise Sorry('only 1 datablock can be processed at a time') # Loop through all the imagesets and find the strong spots reflections = flex.reflection_table.from_observations( datablocks[0], params) # Delete the shoeboxes if not params.output.shoeboxes: del reflections['shoebox'] # ascii spot count per image plot from dials.util.ascii_art import spot_counts_per_image_plot for i, imageset in enumerate(datablocks[0].extract_imagesets()): ascii_plot = spot_counts_per_image_plot( reflections.select(reflections['id'] == i)) if len(ascii_plot): logger.info('\nHistogram of per-image spot count for imageset %i:' %i) logger.info(ascii_plot) # Save the reflections to file logger.info('\n' + '-' * 80) reflections.as_pickle(params.output.reflections) logger.info('Saved {0} reflections to {1}'.format( len(reflections), params.output.reflections)) # Save the datablock if params.output.datablock: from dxtbx.datablock import DataBlockDumper logger.info('Saving datablocks to {0}'.format( params.output.datablock)) dump = DataBlockDumper(datablocks) dump.as_file(params.output.datablock) # Print some per image statistics if params.per_image_statistics: from dials.algorithms.spot_finding import per_image_analysis from cStringIO import StringIO s = StringIO() for i, imageset in enumerate(datablocks[0].extract_imagesets()): print >> s, "Number of centroids per image for imageset %i:" %i stats = per_image_analysis.stats_imageset( imageset, reflections.select(reflections['id'] == i), resolution_analysis=False) per_image_analysis.print_table(stats, out=s) logger.info(s.getvalue()) # Print the time logger.info("Time Taken: %f" % (time() - start_time))
def run(self, args=None): """ Parse the options. """ # Parse the command line arguments in two passes to set up logging early params, options = self.parser.parse_args(args=args, show_diff_phil=False, quick_parse=True) # Configure logging, if this is the main process if __name__ == "__main__": from dials.util import log log.config(verbosity=options.verbose, logfile=params.output.log) from dials.util.version import dials_version logger.info(dials_version()) # Parse the command line arguments completely if params.input.ignore_unhandled: params, options, unhandled = self.parser.parse_args( args=args, show_diff_phil=False, return_unhandled=True) # Remove any False values from unhandled (eliminate empty strings) unhandled = [x for x in unhandled if x] else: params, options = self.parser.parse_args(args=args, show_diff_phil=False) unhandled = None # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil: logger.info("The following parameters have been modified:\n") logger.info(diff_phil) # Print a warning if something unhandled if unhandled: msg = "Unable to handle the following arguments:\n" msg += "\n".join([" %s" % a for a in unhandled]) msg += "\n" logger.warning(msg) # Print help if no input if len(params.input.experiments) == 0 and not (params.input.template or params.input.directory): self.parser.print_help() return # Setup the experiments importer imageset_importer = ImageSetImporter(params) # Setup the metadata updater metadata_updater = MetaDataUpdater(params) # Extract the experiments and loop through experiments = metadata_updater(imageset_importer()) # Compute some numbers num_sweeps = 0 num_stills = 0 num_images = 0 for e in experiments: if isinstance(e.imageset, ImageSweep): num_sweeps += 1 else: num_stills += 1 num_images += len(e.imageset) format_list = {str(e.imageset.get_format_class()) for e in experiments} # Print out some bulk info logger.info("-" * 80) for f in format_list: logger.info(" format: %s" % f) logger.info(" num images: %d" % num_images) logger.info(" num sweeps: %d" % num_sweeps) logger.info(" num stills: %d" % num_stills) # Print out info for all experiments for experiment in experiments: # Print some experiment info - override the output of image range # if appropriate image_range = params.geometry.scan.image_range if isinstance(experiment.imageset, ImageSweep): imageset_type = "sweep" else: imageset_type = "stills" logger.debug("-" * 80) logger.debug(" format: %s" % str(experiment.imageset.get_format_class())) logger.debug(" imageset type: %s" % imageset_type) if image_range is None: logger.debug(" num images: %d" % len(experiment.imageset)) else: logger.debug(" num images: %d" % (image_range[1] - image_range[0] + 1)) logger.debug("") logger.debug(experiment.imageset.get_beam()) logger.debug(experiment.imageset.get_goniometer()) logger.debug(experiment.imageset.get_detector()) logger.debug(experiment.imageset.get_scan()) # Only allow a single sweep if params.input.allow_multiple_sweeps is False: self.assert_single_sweep(experiments, params) # Write the experiments to file self.write_experiments(experiments, params)
def run(self): ''' Run the script. ''' from dials.algorithms.profile_model.factory import ProfileModelFactory from dials.util.command_line import Command from dials.array_family import flex from dials.util.options import flatten_reflections, flatten_experiments from dxtbx.model.experiment.experiment_list import ExperimentListDumper from libtbx.utils import Sorry from dials.util import log log.config() # Parse the command line params, options = self.parser.parse_args(show_diff_phil=True) reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) if len(reflections) == 0 and len(experiments) == 0: self.parser.print_help() return if len(reflections) != 1: raise Sorry('exactly 1 reflection table must be specified') if len(experiments) == 0: raise Sorry('no experiments were specified') if (not 'background.mean' in reflections[0]) and params.subtract_background: raise Sorry('for subtract_background need background.mean in reflections') reflections, _ = self.process_reference(reflections[0], params) # Predict the reflections logger.info("") logger.info("=" * 80) logger.info("") logger.info("Predicting reflections") logger.info("") predicted = flex.reflection_table.from_predictions_multi( experiments, dmin=params.prediction.d_min, dmax=params.prediction.d_max, margin=params.prediction.margin, force_static=params.prediction.force_static) # Match with predicted matched, reflections, unmatched = predicted.match_with_reference(reflections) assert(len(matched) == len(predicted)) assert(matched.count(True) <= len(reflections)) if matched.count(True) == 0: raise Sorry(''' Invalid input for reference reflections. Zero reference spots were matched to predictions ''') elif len(unmatched) != 0: logger.info('') logger.info('*' * 80) logger.info('Warning: %d reference spots were not matched to predictions' % ( len(unmatched))) logger.info('*' * 80) logger.info('') # Create the profile model experiments = ProfileModelFactory.create(params, experiments, reflections) for model in experiments: sigma_b = model.profile.sigma_b(deg=True) sigma_m = model.profile.sigma_m(deg=True) if type(sigma_b) == type(1.0): logger.info('Sigma B: %f' % sigma_b) logger.info('Sigma M: %f' % sigma_m) else: # scan varying mean_sigma_b = sum(sigma_b) / len(sigma_b) mean_sigma_m = sum(sigma_m) / len(sigma_m) logger.info('Sigma B: %f' % mean_sigma_b) logger.info('Sigma M: %f' % mean_sigma_m) # Wrtie the parameters Command.start("Writing experiments to %s" % params.output) dump = ExperimentListDumper(experiments) with open(params.output, "w") as outfile: outfile.write(dump.as_json()) Command.end("Wrote experiments to %s" % params.output)
def run(args): from dials.util import log usage = "%s [options] datablock.json strong.pickle" % libtbx.env.dispatcher_name parser = OptionParser(usage=usage, phil=phil_scope, read_datablocks=True, read_reflections=True, check_format=False, epilog=help_message) params, options = parser.parse_args(show_diff_phil=False) datablocks = flatten_datablocks(params.input.datablock) reflections = flatten_reflections(params.input.reflections) if len(datablocks) == 0 or len(reflections) == 0: parser.print_help() exit(0) # Configure the logging log.config(info=params.output.log, debug=params.output.debug_log) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) if params.seed is not None: import random flex.set_random_seed(params.seed) random.seed(params.seed) imagesets = [] for datablock in datablocks: imagesets.extend(datablock.extract_imagesets()) assert len(imagesets) > 0 assert len(reflections) == len(imagesets) if params.scan_range is not None and len(params.scan_range) > 0: reflections = [ filter_reflections_by_scan_range(refl, params.scan_range) for refl in reflections ] dps_params = dps_phil_scope.extract() # for development, we want an exhaustive plot of beam probability map: dps_params.indexing.plot_search_scope = params.plot_search_scope dps_params.indexing.mm_search_scope = params.mm_search_scope for i in range(params.n_macro_cycles): if params.n_macro_cycles > 1: logger.info('Starting macro cycle %i' % (i + 1)) new_detector, new_beam = discover_better_experimental_model( imagesets, reflections, params, dps_params, nproc=params.nproc, wide_search_binning=params.wide_search_binning) for imageset in imagesets: imageset.set_detector(new_detector) imageset.set_beam(new_beam) logger.info('') from dxtbx.serialize import dump logger.info("Saving optimized datablock to %s" % params.output.datablock) dump.datablock(datablock, params.output.datablock)
def run(self): ''' Extract the shoeboxes. ''' from dials.util.options import flatten_reflections from dials.util.options import flatten_experiments from dials.util.options import flatten_datablocks from dials.util import log from dials.array_family import flex from libtbx.utils import Sorry from logging import info # Parse the command line params, options = self.parser.parse_args(show_diff_phil=False) # Configure logging log.config() # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) # Get the data reflections = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) datablocks = flatten_datablocks(params.input.datablock) if len(experiments) == 0 and len(datablocks) == 0 and len(reflections) == 0: self.parser.print_help() exit(0) elif (len(experiments) != 0 and len(datablocks) != 0): raise Sorry('Both experiment list and datablocks set') elif len(experiments) > 1: raise Sorry('More than 1 experiment set') elif len(datablocks) > 1: raise Sorry('More than 1 datablock set') elif len(experiments) == 1: imageset = experiments[0].imageset elif len(datablocks) == 1: imagesets = datablocks[0].extract_imagesets() if len(imagesets) != 1: raise Sorry('Need 1 imageset, got %d' % len(imagesets)) imageset = imagesets[0] if len(reflections) != 1: raise Sorry('Need 1 reflection table, got %d' % len(reflections)) else: reflections = reflections[0] # Check the reflections contain the necessary stuff assert("bbox" in reflections) assert("panel" in reflections) # Get some models detector = imageset.get_detector() scan = imageset.get_scan() frame0, frame1 = scan.get_array_range() # Add some padding but limit to image volume if params.padding > 0: info('Adding %d pixels as padding' % params.padding) x0, x1, y0, y1, z0, z1 = reflections['bbox'].parts() x0 -= params.padding x1 += params.padding y0 -= params.padding y1 += params.padding z0 -= params.padding z1 += params.padding panel = reflections['panel'] for i in range(len(reflections)): width, height = detector[panel[i]].get_image_size() if z0[i] < frame0: z0[i] = frame0 if z1[i] > frame1: z1[i] = frame1 reflections['bbox'] = flex.int6(x0, x1, y0, y1, z0, z1) # Save the old shoeboxes if "shoebox" in reflections: old_shoebox = reflections['shoebox'] else: old_shoebox = None # Allocate the shoeboxes reflections["shoebox"] = flex.shoebox( reflections["panel"], reflections["bbox"], allocate=True) # Extract the shoeboxes reflections.extract_shoeboxes(imageset, verbose=True) # Preserve masking if old_shoebox is not None: info("Applying old shoebox mask") new_shoebox = reflections['shoebox'] for i in range(len(reflections)): bbox0 = old_shoebox[i].bbox bbox1 = new_shoebox[i].bbox mask0 = old_shoebox[i].mask mask1 = new_shoebox[i].mask mask2 = flex.int(mask1.accessor(), 0) x0 = bbox0[0] - bbox1[0] x1 = bbox0[1] - bbox0[0] + x0 y0 = bbox0[2] - bbox1[2] y1 = bbox0[3] - bbox0[2] + y0 z0 = bbox0[4] - bbox1[4] z1 = bbox0[5] - bbox0[4] + z0 mask2[z0:z1,y0:y1,x0:x1] = mask0 mask1 = mask1.as_1d() | mask2.as_1d() mask1.reshape(new_shoebox[i].mask.accessor()) new_shoebox[i].mask = mask1 # Saving the reflections to disk filename = params.output.reflections info('Saving %d reflections to %s' % (len(reflections), filename)) reflections.as_pickle(filename)