def __init__(self, args): working_phil = parse(phil_defaults) phil_args = [] for arg in args[1:]: if os.path.exists(arg): working_phil = working_phil.fetch( source = parse(open(arg).read())) else: phil_args.append(arg) for phil_arg in phil_args: interp = working_phil.command_line_argument_interpreter( home_scope = 'resolutionizer') more_phil = interp.process(phil_arg) working_phil = working_phil.fetch(source = more_phil) self._params = working_phil.extract().resolutionizer # look for useful information in the input file mtz_file = args[0] mtz_obj = mtz.object(mtz_file) i_data = None sigi_data = None mi = mtz_obj.extract_miller_indices() unit_cell = None for crystal in mtz_obj.crystals(): unit_cell = crystal.unit_cell() for dataset in crystal.datasets(): for column in dataset.columns(): if column.label() == 'I': i_data = column.extract_values( not_a_number_substitute = 0.0) if column.label() == 'SIGI': sigi_data = column.extract_values( not_a_number_substitute = 0.0) assert(i_data) assert(sigi_data) self._unit_cell = unit_cell self._space_group = mtz_obj.space_group() self._r = x2tbx.ReflectionList() self._r.setup(mi, i_data, sigi_data) self._r.set_unit_cell(unit_cell.parameters()) self._r.merge() return
def phil_scope(cls): ''' Get the phil scope for the interface or extension. :returns: The phil scope for the interface or extension ''' from libtbx.phil import parse if cls == Interface: raise RuntimeError('"Interface has no phil parameters"') elif not issubclass(cls, Interface): raise RuntimeError('%s is not an interface or extension' % str(cls)) if Interface in cls.__bases__: doc = '\n'.join('"%s"' % d for d in cls.__doc__) master_scope = parse('%s .help=%s {}' % (cls.name, doc)) main_scope = master_scope.get_without_substitution(cls.name) assert(len(main_scope) == 1) main_scope = main_scope[0] if 'phil' in cls.__dict__: main_scope.adopt_scope(cls.phil()) if Interface in cls.__bases__: def ext_names(extensions): names = [] default_index = -1 for ext in extensions: if 'default' in ext.__dict__: default_index = len(names) names.append(ext.name) if default_index < 0: default_index = 0 if names: names[default_index] = '*' + names[default_index] return names exts = list(cls.extensions()) if exts: algorithm = parse(''' algorithm = %s .help = "The choice of algorithm" .type = choice ''' % ' '.join(ext_names(exts))) main_scope.adopt_scope(algorithm) for ext in exts: main_scope.adopt_scope(ext.phil_scope()) else: if 'phil' in cls.__dict__: help_str = '\n'.join(['"%s"' % line for line in cls.__doc__.split()]) master_scope = parse('%s .help=%s {}' % (cls.name, help_str)) main_scope = master_scope.get_without_substitution(cls.name) assert(len(main_scope) == 1) main_scope = main_scope[0] main_scope.adopt_scope(cls.phil()) else: master_scope = parse('') return master_scope
def run(args): if '-h' in args or '--help' in args or '-c' in args: print help_str phil_scope.show(attributes_level=2) return user_phil = [] for arg in args: if os.path.isfile(arg): user_phil.append(parse("geom_path=%s"%arg)) else: try: user_phil.append(parse(arg)) except Exception, e: raise Sorry("Unrecognized argument: %s"%arg)
def get_params(args): user_phil = [] for arg in args: if os.path.isfile(arg): try: user_phil.append(parse(file_name=arg)) except Exception, e: print str(e) raise Sorry("Couldn't parse phil file %s"%arg) else: try: user_phil.append(parse(arg)) except Exception, e: print str(e) raise Sorry("Couldn't parse argument %s"%arg)
def __init__(self): '''Initialise the script.''' from dials.util.options import OptionParser from libtbx.phil import parse import libtbx.load_env # The phil scope phil_scope = parse(''' input { experiments_template = None .type = path .multiple = True reflections_template = None .type = path .multiple = True } output { phil_filename = experiments_and_reflections.phil .type = str .help = "The filename for combined experiments and reflections" } ''', process_includes=True) self.output_phil = parse(''' input { experiments = None .type = path .multiple = True .help = "The experiment list file path" reflections = None .type = path .multiple = True .help = "The reflection table file path" } ''') # The script usage usage = "usage: %s [options] [param.phil] " % libtbx.env.dispatcher_name # Create the parser self.parser = OptionParser( usage=usage, phil=phil_scope) return
def run(self): from dials.algorithms.integration.integrator import Integrator3D from dials.algorithms.integration.integrator import phil_scope from dials.algorithms.integration.integrator import Parameters import StringIO import sys from libtbx.phil import parse params = phil_scope.fetch(parse(''' integration.block.size=%d integration.mp.nproc=%d integration.profile_fitting=False ''' % (5, self.nproc))).extract() output = StringIO.StringIO() stdout = sys.stdout sys.stdout = output try: integrator = Integrator3D( self.exlist, self.rlist, Parameters.from_phil(params.integration)) result = integrator.integrate() except Exception: print output raise sys.stdout = stdout print 'OK'
def phil(cls): from libtbx.phil import parse phil = parse(''' exp_spot_dimension = 3 .type = int .help = "The expected spot dimensions" global_threshold = 100 .type = float .help = "The global threshold value." min_blob_score = 0.7 .type = float .help = "The minimum score for a blob" num_passes = 0 .type = int .help = "Number of passes after updating ideal spot" debug = False .type = bool .help = "Write out correlation" ''') return phil
def __init__(self): '''Initialise the script.''' from dials.util.options import OptionParser from libtbx.phil import parse phil_scope = parse(''' key = 'miller_index' .type = str .help = "The chosen sort key. This should be a column of " "the reflection table." reverse = False .type = bool .help = "Reverse the sort direction" output = sorted.pickle .type = str .help = "The output reflection filename" ''') # The script usage usage = """ usage: %s [options] reflections.pickle """ % libtbx.env.dispatcher_name # Initialise the base class self.parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, epilog=help_message)
def master(self, package="iotbx"): libselector = {"libtbx": self.libtbx_defs, "iotbx": self.iotbx_defs + self.libtbx_defs}[package] if package == "libtbx": from libtbx import phil else: from iotbx import phil return phil.parse(input_string=libselector, process_includes=True)
def run(args): import libtbx.load_env from dials.util.options import OptionParser from dials.util.options import flatten_reflections from libtbx.utils import Sorry from libtbx.phil import parse phil_scope = parse( """ hklout = hklout.pickle .type = str .help = "The output pickle file" """ ) usage = "%s integrated.pickle [hklout=hklout.pickle]" % (libtbx.env.dispatcher_name) parser = OptionParser(usage=usage, read_reflections=True, check_format=False, phil=phil_scope) params, options = parser.parse_args(show_diff_phil=True) reflections = flatten_reflections(params.input.reflections) if len(reflections) != 1: raise Sorry("exactly 1 reflection table must be specified") integrated_data = reflections[0] filter_good_reflections(integrated_data).as_pickle(params.hklout)
def __init__(self): '''Initialise the script.''' from dials.util.options import OptionParser from libtbx.phil import parse import libtbx.load_env # The phil scope phil_scope = parse(''' output { experiments_prefix = experiments .type = str .help = "Filename prefix for the split experimental models" reflections_prefix = reflections .type = str .help = "Filename prefix for the split reflections" } ''', process_includes=True) # The script usage usage = "usage: %s [options] [param.phil] " \ "experiments1.json experiments2.json reflections1.pickle " \ "reflections2.pickle..." \ % libtbx.env.dispatcher_name # Create the parser self.parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message)
def phil(cls): from libtbx.phil import parse phil = parse(''' robust { algorithm = False .type = bool .help = "Use the robust algorithm" tuning_constant = 1.345 .type = float .help = "The tuning constant for robust estimation" } min_pixels = 10 .type = int(value_min=1) .help = "The minimum number of pixels required" model = None .type = str .help = "The model filename" ''') return phil
def read_experiment_file(self, experiment_file): ### open DIALS json file phil_scope_str=''' experiments = 'example_refined_experiments.json' ''' phil_scope = parse(phil_scope_str, process_includes=True) parser = OptionParser( phil=phil_scope, check_format=False, read_experiments=True) params, options = parser.parse_args(args=[experiment_file], show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) exp_xtal = experiments.crystals()[0] ### define useful attributes self.crystal = exp_xtal uc = self.crystal.get_unit_cell() uc_nospace = str(uc).replace(" ", "") uc_nospace_noparen = uc_nospace[1:-1] self.unit_cell = uc_nospace_noparen self.space_group = self.crystal.get_space_group() self.laue_group = self.space_group.laue_group_type() # self.a_matrix = crystal.get_A() self.experiments = experiments
def __init__(self, master_phil, local_overrides = "", cmdline_args = None, verbose=False): self._verbose = verbose arg_interpreter = command_line.argument_interpreter( master_phil=master_phil) user_phil = parse(local_overrides) cmdline_phils = [] if cmdline_args: for arg in cmdline_args: cmdline_phils.append(arg_interpreter.process(arg)) working_phil = master_phil.fetch( sources=[user_phil] + cmdline_phils) self._params = working_phil.extract().geometry.parameters self.set_seed() self.build_goniometer() self.build_crystal() self.build_beam() self.build_detector() # write changes back to the PHIL object temp = working_phil.extract() temp.geometry.parameters = self._params self.phil = master_phil.format(python_object = temp)
def run(args): user_phil = [] for arg in args: try: user_phil.append(parse(arg)) except Exception, e: raise Sorry("Unrecognized argument %s"%arg)
def __init__(self): '''Initialise the script.''' from dials.util.options import OptionParser import libtbx.load_env # The phil scope phil_scope = parse(''' print_precision = 4 .type=int(value_min=0) .help="Number of decimal places to print values with" ''', process_includes=True) # The script usage usage = ("usage: %s [options] [param.phil] experiments1.json " "experiments2.json..." % libtbx.env.dispatcher_name) # Create the parser self.parser = OptionParser( usage=usage, phil=phil_scope, read_experiments=True, check_format=False, epilog=help_message)
def create_models(self): # build models, with a larger crystal than default in order to get plenty of # reflections on the 'still' image overrides = """ geometry.parameters.crystal.a.length.range=40 50; geometry.parameters.crystal.b.length.range=40 50; geometry.parameters.crystal.c.length.range=40 50; geometry.parameters.random_seed = 42""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) models = Extract(master_phil, overrides) # keep track of the models self.detector = models.detector self.gonio = models.goniometer self.crystal = models.crystal self.beam = models.beam # Create a stills ExperimentList self.stills_experiments = ExperimentList() self.stills_experiments.append(Experiment(beam=self.beam, detector=self.detector, crystal=self.crystal, imageset=None)) # keep track of the parameterisation of the models self.det_param = DetectorParameterisationSinglePanel(self.detector) self.s0_param = BeamParameterisation(self.beam, self.gonio) self.xlo_param = CrystalOrientationParameterisation(self.crystal) self.xluc_param = CrystalUnitCellParameterisation(self.crystal)
def __init__(self, experiments, sig_mm=0.1, sig_deg=0.05, frac_sig_unitless=0.02, perturb_cell_lengths=True, perturb_cell_angles=True): self._sig_mm = sig_mm self._sig_mrad = sig_deg * pi/0.18 self._frac_sig_unitless = frac_sig_unitless self._perturb_cell_lengths = perturb_cell_lengths self._perturb_cell_angles = perturb_cell_angles self.dummy_reflections = generate_reflections(experiments) # use default options for Refiners built internally phil_scope = parse(''' include scope dials.algorithms.refinement.refiner.phil_scope ''', process_includes=True) self.params = phil_scope.extract() # changes for building internal Refiners self.params.refinement.reflections.outlier.algorithm="null" self.params.refinement.reflections.random_seed=None self.params.refinement.verbosity=0 self.original_experiments = experiments return
def setup_models(args): """setup the experimental models""" # Setup experimental models master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) models = setup_geometry.Extract(master_phil, cmdline_args = args) detector = models.detector goniometer = models.goniometer crystal = models.crystal beam = models.beam # Build a mock scan for a 180 degree sweep sf = scan_factory() scan = sf.make_scan(image_range = (1,180), exposure_times = 0.1, oscillation = (0, 1.0), epochs = range(180), deg = True) sweep_range = scan.get_oscillation_range(deg=False) im_width = scan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 1.0 * pi / 180.) experiments = ExperimentList() experiments.append(Experiment( beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None)) return experiments
def run(self): """ Process all images assigned to this thread """ if len(sys.argv) == 1 or "-h" in sys.argv or "--help" in sys.argv or "-c" in sys.argv: print help_str print "Showing phil parameters:" print phil_scope.as_str(attributes_level = 2) return user_phil = [] for arg in sys.argv[1:]: if (os.path.isfile(arg)): user_phil.append(parse(file_name=arg)) else: try: user_phil.append(parse(arg)) except RuntimeError, e: raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e)))
def OnRestoreMetrology(self, event): print "Not implemented" return dialog = wx.FileDialog( self, defaultDir="", message="Restore metrology file", style=wx.FD_OPEN, wildcard="Phil files (*.eff; *.def)|*.eff;*.def") if dialog.ShowModal() == wx.ID_OK: path = dialog.GetPath() if (path != "") : from xfel.cftbx.detector.metrology import \ master_phil, metrology_as_transformation_matrices from libtbx import phil frame = self.GetParent().GetParent() stream = open(path) metrology_phil = master_phil.fetch(sources=[phil.parse(stream.read())]) stream.close() # Merge restored metrology into the raw image from libtbx.phil import experimental experimental.merge_params_by_key( frame.pyslip.tiles.raw_image._metrology_params, metrology_phil.extract(), 'serial') img = frame.pyslip.tiles.raw_image img.apply_metrology_from_matrices(metrology_as_transformation_matrices( metrology_phil.extract())) # Update the view, trigger redraw. XXX Duplication # w.r.t. OnUpdateQuad(). tiles = frame.pyslip.tiles tiles.flex_image = frame.pyslip.tiles.raw_image.get_flex_image( brightness=tiles.current_brightness / 100) tiles.flex_image.adjust(color_scheme=tiles.current_color_scheme) tiles.reset_the_cache() tiles.tile_cache = tiles.cache[tiles.zoom_level] tiles.tile_list = tiles.lru[tiles.zoom_level] frame.pyslip.Update() # Update the controls, remember to reset the default values # for the spinners. for serial in xrange(4): fast, slow = img.get_panel_fast_slow(serial) name_quadrant = ["Q0", "Q1", "Q2", "Q3"][serial] spinner = getattr(self, "_" + name_quadrant + "_fast_ctrl") spinner.SetDefaultValue(fast) spinner.SetValue(fast) spinner = getattr(self, "_" + name_quadrant + "_slow_ctrl") spinner.SetDefaultValue(slow) spinner.SetValue(slow)
def run(args): if "-c" in args or "-h" in args or "--help" in args: print help_message user_phil = [] for arg in args : try : user_phil.append(phil.parse(arg)) except RuntimeError, e : raise Sorry("Unrecognized argument '%s' (error: %s)" % (arg, str(e)))
def __init__(self): '''Initialise the script.''' from dials.util.options import OptionParser from libtbx.phil import parse import libtbx.load_env from operator import itemgetter flags = list(flex.reflection_table.flags.names.iteritems()) flags.sort(key=itemgetter(0)) self.flag_names, self.flag_values = zip(*flags) phil_str = ''' output { reflections = 'filtered.pickle' .type = str .help = "The filtered reflections output filename" } inclusions { flag = %s .type = choice .help = "Include reflections with this flag to form the working set." .multiple = True } exclusions { flag = %s .type = choice .help = "Exclude reflections from the working set with this flag." .multiple = True } d_min = None .type = float .help = "The maximum resolution" d_max = None .type = float .help = "The minimum resolution" ''' % tuple([' '.join(self.flag_names)] * 2) phil_scope = parse(phil_str) # The script usage usage = "usage: %s [options] experiment.json" % libtbx.env.dispatcher_name # Create the parser self.parser = OptionParser( usage=usage, phil=phil_scope, epilog=help_message, read_reflections=True)
def run(self, argv=None): """ Set up run folder and submit the job. """ if argv is None: argv = sys.argv[1:] if len(argv) == 0 or "-h" in argv or "--help" in argv or "-c" in argv: print help_str print "Showing phil parameters:" print phil_scope.as_str(attributes_level=2) return user_phil = [] dispatcher_args = [] for arg in argv: if os.path.isfile(arg): user_phil.append(parse(file_name=arg)) else: try: user_phil.append(parse(arg)) except RuntimeError, e: dispatcher_args.append(arg)
def read_reflection_file(self, reflection_file): ### open DIALS pickle file phil_scope_str=''' reflections = 'example_refined.pickle' ''' phil_scope = parse(phil_scope_str, process_includes=True) parser = OptionParser( phil=phil_scope, check_format=False, read_reflections=True) params, options = parser.parse_args(args=[reflection_file], show_diff_phil=True) reflections = flatten_reflections(params.input.reflections) self.reflections = reflections
def __init__(self, phil_file = None, selected_filter = None): """The mod_onecolor_spectrum_filter class constructor stores the parameters passed from the psana configuration file in instance variables. @param phil_file Filter parameters. See cxi/spectra_filter.py @param selected_filter Which of the filters in phil_file to apply """ self.params = phil_scope.fetch(parse(file_name = phil_file)).extract() self.filter = spectra_filter(self.params) self.selected_filter = selected_filter self.n_accepted = 0 self.n_total = 0
def phil(cls): from libtbx.phil import parse phil = parse(''' gain = None .type = float(value_min=0.0) .help = "Use a flat gain map for the entire detector. Cannot be used" "in conjunction with lookup.gain_map parameter." kernel_size = 3 3 .help = "The size of the local area around the spot in which" "to calculate the mean and variance. The kernel is" "given as a box of size (2 * nx + 1, 2 * ny + 1) centred" "at the pixel." .type = ints(size=2) .expert_level = 1 sigma_background = 6 .help = "The number of standard deviations of the coefficient of" "variation (variance / mean) in the local area below" "which the pixel will be classified as background." .type = float .expert_level = 1 sigma_strong = 3 .help = "The number of standard deviations above the mean in the" "local area above which the pixel will be classified as" "strong." .type = float .expert_level = 1 min_local = 2 .help = "The minimum number of pixels under the image processing kernel" "that are need to do the thresholding operation. Setting the" "value between 2 and the total number of pixels under the" "kernel will force the algorithm to use that number as the" "minimum. If the value is less than or equal to zero, then" "the algorithm will use all pixels under the kernel. In" "effect this will add a border of pixels which are always" "classed as background around the edge of the image and around" "any masked out pixels." .type = int .expert_level = 1 global_threshold = 0 .type = float .help = "The global threshold value. Consider all pixels less than this" "value to be part of the background." ''') return phil
def __init__(self): ''' Initialise the script. ''' from dials.util.options import OptionParser from libtbx.phil import parse import libtbx.load_env # Create the phil parameters phil_scope = parse(''' input { method = *experiment reflections .type = choice .help = "The input method" xds_file = None .type = str .help = "Explicitly specify the file to use" } output { filename = None .type = str .help = "The output file" xds_datablock = None .type = str .help = "Output filename of data block with xds" } remove_invalid = False .type = bool .help = "Remove non-index reflections (if miller indices are present)" add_standard_columns = False .type = bool .help = "Add empty standard columns to the reflections. Note columns" "for centroid variances are set to contain 1s, not 0s" read_varying_crystal = False .type = bool .help = "Attempt to create a scan-varying crystal model from" "INTEGRATE.LP, if present" ''') # The option parser usage = "usage: %s [options] (SPOT.XDS|INTEGRATE.HKL)" % libtbx.env.dispatcher_name self.parser = OptionParser( usage=usage, phil=phil_scope)
def __init__(self): from dials.util.options import OptionParser from libtbx.phil import parse usage = "usage: %prog [options] [param.phil] " \ "sweep.json crystal.json intensities.mtz" phil_scope = parse(''' output = simulated.pickle .type = str .help = "The filename for the simulated reflections" ''') # Create the parser self.parser = OptionParser( usage=usage, phil=self.phil_scope())
def __init__(self): ''' Initialise the script. ''' from dials.util.options import OptionParser from libtbx.phil import parse import libtbx.load_env # Create the phil parameters phil_scope = parse(''' interfaces=False .type = bool .help = "Only show information about the interfaces" ''') # Create the option parser usage = "usage: %s [options] /path/to/image/files" \ % libtbx.env.dispatcher_name self.parser = OptionParser(usage=usage, phil=phil_scope)
from __future__ import absolute_import, division, print_function # DIALS_ENABLE_COMMAND_LINE_COMPLETION help_message = """ """ # Create the phil scope from libtbx.phil import parse phil_scope = parse( """ """, process_includes=True, ) class Script(object): """ The integration program. """ def __init__(self): """Initialise the script.""" from dials.util.options import OptionParser import libtbx.load_env # The script usage usage = "usage: %s [options] experiment.expt" % libtbx.env.dispatcher_name
phil_scope = parse(''' reference_from_experiment{ beam = None .help = "Take beam model from this experiment to overwrite all other" "beam models in the combined experiments" .type = int(value_min=0) scan = None .help = "Take scan model from this experiment to overwrite all other" "scan models in the combined experiments" .type = int(value_min=0) crystal = None .help = "Take crystal model from this experiment to overwrite all" "other crystal models in the combined experiments" .type = int(value_min=0) goniometer = None .help = "Take goniometer model from this experiment to overwrite all" "other goniometer models in the combined experiments" .type = int(value_min=0) detector = None .help = "Take detector model from this experiment to overwrite all" "other detector models in the combined experiments" .type = int(value_min=0) average_detector = False .help = "Create an average detector model from all the input detector" "models and use it as the reference. Not compatible with" "reference_from_experiment.detector" .type = bool compare_models = True .help = "Whether to compare a model with the reference model before" "replacing it. If the comparison falls outside the tolerance," "the combination will not be allowed. Disable comparison to force" "overwriting of models with the reference" .type = bool average_hierarchy_level = None .help = "For hierarchical detectors, optionally provide a single level" "to do averaging at." .type = int(value_min=0) include scope dials.util.options.tolerance_phil_scope } clustering { use = False .type = bool .help = "Separate experiments into subsets using the clustering" "toolkit. One json per cluster will be saved." dendrogram = False .type = bool .help = "Display dendrogram of the clustering results. Should not" "be used with parallel processing." threshold = 1000 .type = int .help = "Threshold used in the dendrogram to separate into clusters." max_clusters = None .type = int .help = "Maximum number of clusters to save as jsons." max_crystals = None .type = int .help = "Maximum number of crystals to cluster." exclude_single_crystal_clusters = True .type = bool .help = "Don't produce a 'cluster' containing only one crystal." } output { experiments_filename = combined_experiments.json .type = str .help = "The filename for combined experimental models" reflections_filename = combined_reflections.pickle .type = str .help = "The filename for combined reflections" n_subset = None .type = int .help = "If not None, keep a random subset of size n_subset when" "saving the combined experiments" max_batch_size = None .type = int .expert_level = 2 .help = "If not None, split the resultant combined set of experiments" "into seperate files, each at most max_batch_size number of" "experiments. Example, if there were 5500 experiments and" "max_batch_size is 1000, 6 experiment lists will be created," "of sizes 917, 917, 917, 917, 916, 916" delete_shoeboxes = False .type = bool .expert_level = 2 .help = "If true, delete shoeboxes from reflection tables while comb-" "ining them to save on memory." } ''', process_includes=True)
pass logger = logging.getLogger("dials") phil_scope = phil.parse( """ include scope dials.algorithms.scaling.error_model.error_model.phil_scope intensity_choice = *profile sum combine .type = choice .help = "Use profile or summation intensities" combine.Imid = None .type = float .help = "Midpoint value to use when combining profile/summation intensities" output { log = dials.refine_error_model.log .type = str .help = "The log filename" html = error_model.html .type = str .help = "Filename for html report of error model" json = None .type = str .help = "Filename for json export of html report data" } """, process_includes=True, ) def refine_error_model(params, experiments, reflection_tables): """Do error model refinement."""
phil_scope = phil.parse( """ log = dials.cross_validate.log .type = str .help = "The log filename" debug_log = dials.cross_validate.debug.log .type = str .help = "The debug log filename" cross_validation { mode = multi *single .type = choice .help = "Choose the cross validation running mode, for a full description" "see the module docstring. Choice is used for testing a parameter" "that can only have discreet values (a choice or bool phil parameter)." "Variable is used for testing a parameter that can have a float or" "int value (that is also not a 'choice' type). Single just performs" "cross validation on one parameter configuration." parameter = None .type = str .help = "Optimise a command-line parameter. parameter_values must also be" "specified, unless the parameter is a True/False option." parameter_values = None .type = strings .help = "Parameter values to compare, entered as a string of space" "separated values." nfolds = 1 .type = int(value_min=1) .help = "Number of cross-validation folds to perform. If nfolds > 1, the" "minimisation for each option is repeated nfolds times, with an" "incremental offset for the free set. The max number of folds" "allowed is 1/free_set_percentage; if set greater than this then" "the repetition will finish afer 1/free_set_percentage folds." } include scope dials.command_line.scale.phil_scope """, process_includes=True, )
import libtbx.load_env from libtbx.utils import Usage from scitbx.array_family import flex from libtbx.phil import parse from libtbx.utils import Sorry phil_scope = parse(""" show_plots = False .type = bool .help = Show CC gridmap plots multi_angle = True .type = bool .help = If true, compute CC over many angles at each gridpoint (20-70 degrees \ in increments of 2.5 degrees). Otherwise, just compute CC at 45 \ degrees at each grid point (much faster but not as robust) plot_range = None .type = floats(size=2) .help = Min and max CC values for gridmap plots pdf_file = None .type = path .help = If not None and show_plots is True, then save CC plots as multi- \ page cbf file save_cbf = True .type = bool .help = If True, write cbf with best corrections applied """) def run(args): if len(args) == 0 or '-h' in args or '--help' in args or '-c' in args: print("Usage: %s [-p] files" % libtbx.env.dispatcher_name) phil_scope.show(attributes_level=2)
.type = str .help = Mode for reading the xtc data (see LCLS documentation) data_source = None .type = str .help = Complete LCLS data source. Overrides experiment and run. Example: \ exp=mfxo1916:run=20:smd \ More info at https://confluence.slac.stanford.edu/display/PSDM/Manual#Manual-Datasetspecification detector_address = None .type = str .multiple = True .help = detector used for collecting the data at LCLS calib_dir = None .type = str .help = Specifiy path to custom calib directory if needed """ locator_scope = parse(locator_str) class XtcReader(Reader): def nullify_format_instance(self): """ No-op for XTC streams. No issue with multiprocessing. """ pass class FormatXTC(FormatMultiImageLazy, FormatStill, Format): def __init__(self, image_file, **kwargs): from dxtbx import IncorrectFormatError if not self.understand(image_file): raise IncorrectFormatError(self, image_file) FormatMultiImageLazy.__init__(self, **kwargs)
def test_fd_derivatives(): '''Test derivatives of the prediction equation''' from libtbx.phil import parse # Import model builder from dials.test.algorithms.refinement.setup_geometry import Extract # Imports for reflection prediction from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection from dxtbx.model.experiment_list import ExperimentList, Experiment from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor # Create models overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil """, process_includes=True) models = Extract(master_phil, overrides) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam # Build a mock scan for a 72 degree sweep sweep_range = (0., pi/5.) from dxtbx.model import ScanFactory sf = ScanFactory() myscan = sf.make_scan(image_range = (1,720), exposure_times = 0.1, oscillation = (0, 0.1), epochs = range(720), deg = True) # Create a parameterisation of the crystal unit cell from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalUnitCellParameterisation xluc_param = CrystalUnitCellParameterisation(mycrystal) # Create an ExperimentList experiments = ExperimentList() experiments.append(Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None)) # Build a prediction parameterisation for two theta prediction pred_param = TwoThetaPredictionParameterisation(experiments, detector_parameterisations = None, beam_parameterisations = None, xl_orientation_parameterisations = None, xl_unit_cell_parameterisations = [xluc_param]) # Generate some reflections obs_refs, ref_predictor = generate_reflections(experiments) # Build a ReflectionManager with overloads for handling 2theta residuals refman = TwoThetaReflectionManager(obs_refs, experiments, outlier_detector=None) # Build a TwoThetaExperimentsPredictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # Make a target for the least squares 2theta residual target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # Keep only reflections that pass inclusion criteria and have predictions reflections = refman.get_matches() # Get analytical gradients an_grads = pred_param.get_gradients(reflections) # Get finite difference gradients p_vals = pred_param.get_param_vals() deltas = [1.e-7] * len(p_vals) for i in range(len(deltas)): val = p_vals[i] p_vals[i] -= deltas[i] / 2. pred_param.set_param_vals(p_vals) target.predict() reflections = refman.get_matches() rev_state = reflections['2theta_resid'].deep_copy() p_vals[i] += deltas[i] pred_param.set_param_vals(p_vals) target.predict() reflections = refman.get_matches() fwd_state = reflections['2theta_resid'].deep_copy() p_vals[i] = val fd = (fwd_state - rev_state) fd /= deltas[i] # compare with analytical calculation assert approx_equal(fd, an_grads[i]["d2theta_dp"], eps=1.e-6) # return to the initial state pred_param.set_param_vals(p_vals) return
from dials.util import show_mail_on_error help_message = "stage 2 (global) diffBragg refinement" script_phil = """ pandas_table = None .type = str .help = path to an input pandas table (usually output by simtbx.diffBragg.predictions) prep_time = 60 .type = float .help = Time spent optimizing order of input dataframe to better divide shots across ranks .help = Unit is seconds, 1-2 minutes of prep might save a lot of time during refinement! """ philz = script_phil + philz + hopper_phil phil_scope = parse(philz) class Script: def __init__(self): from dials.util.options import OptionParser self.parser = None if COMM.rank == 0: self.parser = OptionParser( usage="", # stage 1 (per-shot) diffBragg refinement", sort_options=True, phil=phil_scope, read_experiments=False, read_reflections=False, check_format=False,
phil_scope = parse(''' output { experiments = 'integrated_experiments.json' .type = str .help = "The experiments output filename" reflections = 'integrated.pickle' .type = str .help = "The integrated output filename" phil = 'dials.integrate.phil' .type = str .help = "The output phil file" log = 'dials.integrate.log' .type = str .help = "The log filename" debug_log = 'dials.integrate.debug.log' .type = str .help = "The debug log filename" report = None .type = str .help = "The integration report filename (*.xml or *.json)" include_bad_reference = False .type = bool .help = "Include bad reference data including unindexed spots," "and reflections whose predictions are messed up in the" "reflection table output. Reflections will have the" "'bad_reference' flag set." } scan_range = None .type = ints(size=2) .help = "Explicitly specify the images to be processed. Only applicable" "when experiment list contains a single imageset." .multiple = True create_profile_model = True .type = bool .help = "Create the profile model" sampling .expert_level = 1 { reflections_per_degree = 50 .help = "The number of predicted reflections per degree of the sweep " "to integrate." .type = float(value_min=0.) minimum_sample_size = 1000 .help = "cutoff that determines whether subsetting of the input " "prediction list is done" .type = int maximum_sample_size = None .help = "The maximum number of predictions to integrate." "Overrides reflections_per_degree if that produces a" "larger sample size." .type = int(value_min=1) integrate_all_reflections = True .help = "Override reflections_per_degree and integrate all predicted" "reflections." .type = bool } exclude_images = None .type = ints .help = "Exclude images from integration (e.g. 1,2,3,4,5 etc)" verbosity = 1 .type = int(value_min=0) .help = "The verbosity level" include scope dials.algorithms.integration.integrator.phil_scope include scope dials.algorithms.profile_model.factory.phil_scope include scope dials.algorithms.spot_prediction.reflection_predictor.phil_scope include scope dials.algorithms.integration.stills_significance_filter.phil_scope include scope dials.algorithms.integration.kapton_correction.absorption_phil_scope ''', process_includes=True)
# Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation # implicit import # Imports for the target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff # implicit import ############################# # Setup experimental models # ############################# args = sys.argv[1:] master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 1") crystal1 = models.crystal models = setup_geometry.Extract( master_phil, cmdline_args=args, local_overrides="geometry.parameters.random_seed = 2")
phil_scope = parse(''' output { model = 'background.pickle' .type = str .help = "The output filename" log = 'dials.model_background.log' .type = str .help = "The log filename" debug_log = 'dials.model_background.debug.log' .type = str .help = "The debug log filename" mean_image_prefix = 'mean' .type = str .help = "The mean background image" variance_image_prefix = 'variance' .type = str .help = "The variance background image" dispersion_image_prefix = 'dispersion' .type = str .help = "The dispersion background image" mask_image_prefix = 'mask' .type = str .help = "The mask background image" min_image_prefix = 'min' .type = str .help = "The min background image" max_image_prefix = 'max' .type = str .help = "The max background image" model_image_prefix = 'model' .type = str .help = "The model background image" polar_model_image_prefix = 'polar' .type = str .help = "The polar model background image" } verbosity = 1 .type = int(value_min=0) .help = "The verbosity level" modeller { min_images = 10 .type = int(value_min=1) .help = "The minimum number of images per pixel" filter_type = *median mean .type = choice .help = "The filter to use on the polar transformed image" kernel_size = 10 .type = int(value_min=0) .help = "The kernel size for the median filter" niter = 100 .type = int(value_min=1) .help = "The number of iterations for filling holes" image_type = min *mean .type = choice .help = "Which image to use" } include scope dials.algorithms.integration.integrator.phil_scope include scope dials.algorithms.spot_prediction.reflection_predictor.phil_scope ''', process_includes=True)
from setup_geometry import Extract from dxtbx.model.experiment.experiment_list import ExperimentList, Experiment # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator from dials.algorithms.refinement.prediction import ScansRayPredictor, \ ExperimentsPredictor from cctbx.sgtbx import space_group, space_group_symbols # We will set up a mock scan from dxtbx.model.scan import scan_factory args = sys.argv[1:] master_phil = parse(""" include scope dials.test.algorithms.refinement.geometry_phil include scope dials.test.algorithms.refinement.minimiser_phil """, process_includes=True) overrides = """geometry.parameters.crystal.a.length.range = 10 50 geometry.parameters.crystal.b.length.range = 10 50 geometry.parameters.crystal.c.length.range = 10 50""" models = Extract(master_phil, local_overrides=overrides, cmdline_args = args) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam ############################# # Generate some reflections #
def test(): # Python and cctbx imports from math import pi from cctbx.sgtbx import space_group, space_group_symbols # Symmetry constrained parameterisation for the unit cell from cctbx.uctbx import unit_cell # We will set up a mock scan and a mock experiment list from dxtbx.model import ScanFactory from dxtbx.model.experiment_list import Experiment, ExperimentList from libtbx.phil import parse from libtbx.test_utils import approx_equal from rstbx.symmetry.constraints.parameter_reduction import symmetrize_reduce_enlarge from scitbx import matrix from scitbx.array_family import flex # Get modules to build models and minimiser using PHIL import dials.tests.algorithms.refinement.setup_geometry as setup_geometry import dials.tests.algorithms.refinement.setup_minimiser as setup_minimiser from dials.algorithms.refinement.parameterisation.beam_parameters import ( BeamParameterisation, ) from dials.algorithms.refinement.parameterisation.crystal_parameters import ( CrystalOrientationParameterisation, CrystalUnitCellParameterisation, ) # Model parameterisations from dials.algorithms.refinement.parameterisation.detector_parameters import ( DetectorParameterisationSinglePanel, ) # Parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import ( XYPhiPredictionParameterisation, ) from dials.algorithms.refinement.prediction.managed_predictors import ( ScansExperimentsPredictor, ScansRayPredictor, ) from dials.algorithms.refinement.reflection_manager import ReflectionManager # Imports for the target function from dials.algorithms.refinement.target import ( LeastSquaresPositionalResidualWithRmsdCutoff, ) # Reflection prediction from dials.algorithms.spot_prediction import IndexGenerator, ray_intersection ############################# # Setup experimental models # ############################# override = """geometry.parameters { beam.wavelength.random=False beam.wavelength.value=1.0 beam.direction.inclination.random=False crystal.a.length.random=False crystal.a.length.value=12.0 crystal.a.direction.method=exactly crystal.a.direction.exactly.direction=1.0 0.002 -0.004 crystal.b.length.random=False crystal.b.length.value=14.0 crystal.b.direction.method=exactly crystal.b.direction.exactly.direction=-0.002 1.0 0.002 crystal.c.length.random=False crystal.c.length.value=13.0 crystal.c.direction.method=exactly crystal.c.direction.exactly.direction=0.002 -0.004 1.0 detector.directions.method=exactly detector.directions.exactly.dir1=0.99 0.002 -0.004 detector.directions.exactly.norm=0.002 -0.001 0.99 detector.centre.method=exactly detector.centre.exactly.value=1.0 -0.5 199.0 }""" master_phil = parse( """ include scope dials.tests.algorithms.refinement.geometry_phil include scope dials.tests.algorithms.refinement.minimiser_phil """, process_includes=True, ) models = setup_geometry.Extract( master_phil, local_overrides=override, verbose=False ) mydetector = models.detector mygonio = models.goniometer mycrystal = models.crystal mybeam = models.beam ########################### # Parameterise the models # ########################### det_param = DetectorParameterisationSinglePanel(mydetector) s0_param = BeamParameterisation(mybeam, mygonio) xlo_param = CrystalOrientationParameterisation(mycrystal) xluc_param = CrystalUnitCellParameterisation(mycrystal) # Fix beam to the X-Z plane (imgCIF geometry), fix wavelength s0_param.set_fixed([True, False, True]) ######################################################################## # Link model parameterisations together into a parameterisation of the # # prediction equation # ######################################################################## # Build a mock scan for a 180 degree sequence sf = ScanFactory() myscan = sf.make_scan( image_range=(1, 1800), exposure_times=0.1, oscillation=(0, 0.1), epochs=list(range(1800)), deg=True, ) # Build an ExperimentList experiments = ExperimentList() experiments.append( Experiment( beam=mybeam, detector=mydetector, goniometer=mygonio, scan=myscan, crystal=mycrystal, imageset=None, ) ) # Create the PredictionParameterisation pred_param = XYPhiPredictionParameterisation( experiments, [det_param], [s0_param], [xlo_param], [xluc_param] ) ################################ # Apply known parameter shifts # ################################ # shift detector by 1.0 mm each translation and 4 mrad each rotation det_p_vals = det_param.get_param_vals() p_vals = [a + b for a, b in zip(det_p_vals, [1.0, 1.0, 1.0, 4.0, 4.0, 4.0])] det_param.set_param_vals(p_vals) # shift beam by 4 mrad in free axis s0_p_vals = s0_param.get_param_vals() p_vals = list(s0_p_vals) p_vals[0] += 4.0 s0_param.set_param_vals(p_vals) # rotate crystal a bit (=3 mrad each rotation) xlo_p_vals = xlo_param.get_param_vals() p_vals = [a + b for a, b in zip(xlo_p_vals, [3.0, 3.0, 3.0])] xlo_param.set_param_vals(p_vals) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # alpha and beta angles) xluc_p_vals = xluc_param.get_param_vals() cell_params = mycrystal.get_unit_cell().parameters() cell_params = [a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])] new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(mycrystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.0e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) ############################# # Generate some reflections # ############################# # All indices in a 2.0 Angstrom sphere resolution = 2.0 index_generator = IndexGenerator( mycrystal.get_unit_cell(), space_group(space_group_symbols(1).hall()).type(), resolution, ) indices = index_generator.to_array() sequence_range = myscan.get_oscillation_range(deg=False) im_width = myscan.get_oscillation(deg=False)[1] assert sequence_range == (0.0, pi) assert approx_equal(im_width, 0.1 * pi / 180.0) # Predict rays within the sequence range ray_predictor = ScansRayPredictor(experiments, sequence_range) obs_refs = ray_predictor(indices) # Take only those rays that intersect the detector intersects = ray_intersection(mydetector, obs_refs) obs_refs = obs_refs.select(intersects) # Make a reflection predictor and re-predict for all these reflections. The # result is the same, but we gain also the flags and xyzcal.px columns ref_predictor = ScansExperimentsPredictor(experiments) obs_refs["id"] = flex.int(len(obs_refs), 0) obs_refs = ref_predictor(obs_refs) # Set 'observed' centroids from the predicted ones obs_refs["xyzobs.mm.value"] = obs_refs["xyzcal.mm"] # Invent some variances for the centroid positions of the simulated data im_width = 0.1 * pi / 180.0 px_size = mydetector[0].get_pixel_size() var_x = flex.double(len(obs_refs), (px_size[0] / 2.0) ** 2) var_y = flex.double(len(obs_refs), (px_size[1] / 2.0) ** 2) var_phi = flex.double(len(obs_refs), (im_width / 2.0) ** 2) obs_refs["xyzobs.mm.variance"] = flex.vec3_double(var_x, var_y, var_phi) # The total number of observations should be 1128 assert len(obs_refs) == 1128 ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ##################################### # Select reflections for refinement # ##################################### refman = ReflectionManager( obs_refs, experiments, outlier_detector=None, close_to_spindle_cutoff=0.1 ) ############################## # Set up the target function # ############################## # The current 'achieved' criterion compares RMSD against 1/3 the pixel size and # 1/3 the image width in radians. For the simulated data, these are just made up mytarget = LeastSquaresPositionalResidualWithRmsdCutoff( experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None ) ###################################### # Set up the LSTBX refinement engine # ###################################### overrides = """minimiser.parameters.engine=GaussNewton minimiser.parameters.logfile=None""" refiner = setup_minimiser.Extract( master_phil, mytarget, pred_param, local_overrides=overrides ).refiner refiner.run() assert mytarget.achieved() assert refiner.get_num_steps() == 1 assert approx_equal( mytarget.rmsds(), (0.00508252354876, 0.00420954552156, 8.97303428289e-05) ) ############################### # Undo known parameter shifts # ############################### s0_param.set_param_vals(s0_p_vals) det_param.set_param_vals(det_p_vals) xlo_param.set_param_vals(xlo_p_vals) xluc_param.set_param_vals(xluc_p_vals) ###################################################### # Set up the LBFGS with curvatures refinement engine # ###################################################### overrides = """minimiser.parameters.engine=LBFGScurvs minimiser.parameters.logfile=None""" refiner = setup_minimiser.Extract( master_phil, mytarget, pred_param, local_overrides=overrides ).refiner refiner.run() assert mytarget.achieved() assert refiner.get_num_steps() == 9 assert approx_equal( mytarget.rmsds(), (0.0558857700305, 0.0333446685335, 0.000347402754278) )
phil_scope = parse( """ refinement .help = "Parameters to configure the refinement" { mp .expert_level = 2 { nproc = 1 .type = int(value_min=1) .help = "The number of processes to use. Not all choices of refinement" "engine support nproc > 1. Where multiprocessing is possible," "it is helpful only in certain circumstances, so this is not" "recommended for typical use." } parameterisation .help = "Parameters to control the parameterisation of experimental models" { %(parameterisation_phil)s } %(refinery_phil)s target .help = "Parameters to configure the target function" .expert_level = 1 { %(target_phil)s } reflections .help = "Parameters used by the reflection manager" { %(reflections_phil)s } } """ % format_data, process_includes=True, )
phil_scope = parse( """ input { mtzfile = None .type = str .help = "We can also import an MTZ file" } mode = *dataset image_group .type = choice .help = "Perform analysis on whole datasets or batch groups" group_size = 10 .type = int(value_min=1) .help = "The number of images to group together when calculating delta" "cchalf in image_group mode" mtz { batch_offset = None .type = int .help = "The batch offset between consecutive datasets in the mtz file." } output { experiments = "filtered.expt" .type = str .help = "The filtered experiments file" reflections = "filtered.refl" .type = str .help = "The filtered reflections file" table = "delta_cchalf.dat" .type = str .help = "A file with delta cchalf values" html = "compute_delta_cchalf.html" .type = str .help = "HTML filename for report of results." } nbins = 10 .type = int(value_min=1) .help = "The number of resolution bins to use" dmin = None .type = float .help = "The maximum resolution" dmax = None .type = float .help = "The minimum resolution" stdcutoff = 4.0 .type = float .help = "Datasets with a ΔCC½ below (mean - stdcutoff*std) are removed" output { log = 'dials.compute_delta_cchalf.log' .type = str .help = "The log filename" } """ )
) from dials.util.multi_dataset_handling import ( assign_unique_identifiers, parse_multiple_datasets, ) help_message = """Command line script which assigns experiment identifiers to reflections and experiments and saves them back to disk. """ phil_scope = phil.parse(""" identifiers = None .type = strings .help = "User specified identifiers to use, must be a list of strings equal" "to the number of datasets." output { reflections = assigned.refl .type = str experiments = assigned.expt .type = str } """) def run(args=None): """Run assign experiment identifiers from the command line.""" usage = ( """Usage: dials.assign_experiment_identifiers observations.refl models.expt""" ) parser = OptionParser( usage=usage, read_experiments=True,
phil_scope = phil.parse( """ include scope dials.algorithms.scaling.model.model.model_phil_scope output { log = dials.scale.log .type = str .help = "The log filename" experiments = "scaled.expt" .type = str .help = "Option to set filepath for output json." reflections = "scaled.refl" .type = str .help = "Option to set filepath for output pickle file of scaled intensities." html = "dials.scale.html" .type = str .help = "Filename for html report." json = None .type = str .help = "Filename to save html report data in json format." unmerged_mtz = None .type = str .help = "Filename to export an unmerged_mtz file using dials.export." merged_mtz = None .type = str .help = "Filename to export a merged_mtz file." crystal_name = XTAL .type = str .help = "The crystal name to be exported in the mtz file metadata" .expert_level = 1 project_name = DIALS .type = str .help = "The project name for the mtz file metadata" use_internal_variance = False .type = bool .help = "Option to use internal spread of the intensities when merging reflection groups and calculating sigI, rather than using the sigmas of the individual reflections." .expert_level = 1 merging.nbins = 20 .type = int .help = "Number of bins to use for calculating and plotting merging stats." .expert_level = 1 delete_integration_shoeboxes = True .type = bool .help = "Discard integration shoebox data from scaling output, to help" "with memory management." .expert_level = 2 } include scope dials.algorithms.scaling.scaling_options.phil_scope include scope dials.algorithms.scaling.cross_validation.cross_validate.phil_scope include scope dials.algorithms.scaling.scaling_refiner.scaling_refinery_phil_scope include scope dials.algorithms.scaling.scale_and_filter.phil_scope include scope dials.util.exclude_images.phil_scope include scope dials.util.multi_dataset_handling.phil_scope """, process_includes=True, )
from __future__ import division # LIBTBX_SET_DISPATCHER_NAME cxi.experiment_json_to_cbf_def # Script to convert the output from a joint refinement using dials.refine to a CSPAD # cbf header file. Note hardcoded distance of 100 isn't relevant for just a cbf header from dials.util.options import OptionParser from dials.util.options import flatten_experiments from xfel.cftbx.detector.cspad_cbf_tbx import write_cspad_cbf, map_detector_to_basis_dict from libtbx import phil phil_scope = phil.parse(""" output_def_file = refined_detector.def .type = str .help = Name of output .def file """) class Script(object): def __init__(self): # Create the parser self.parser = OptionParser(phil=phil_scope, read_experiments=True) def run(self): params, options = self.parser.parse_args(show_diff_phil=True) experiments = flatten_experiments(params.input.experiments) detector = experiments[0].detector metro = map_detector_to_basis_dict(detector) write_cspad_cbf(None,
from dials.util.options import OptionParser, flatten_experiments logger = logging.getLogger("dials.command_line.complete_full_sphere") help_message = """ This program attempts to compute a sample realignment to measure the blind region of reciprocal space, given an already recorded data set. dials.complete_full_sphere [resolution=1.6] models.expt """ phil_scope = parse( """ resolution = 0.0 .type = float .help = "Resolution of diffraction for blind region calculation" shadow = True .type = bool .help = "Consider shadowing in calculating overall completeness" """ ) class Script: """A class for running the script.""" def __init__(self): """Initialise the script.""" # The script usage usage = "usage: dials.complete_full_sphere [options] " # Create the parser
phil_str = """ iqr_ratio = 1.5 .type = float .help = Interquartile range multiplier for outlier rejection. Use None to disable outlier rejection. ranges = None .type = floats(6) .help = Lower and upper bounds for the ranges to display for each of the a, b and c axes extract_tags = False .type = bool .help = Extract tags from the names of multiple combined_experiments.json filenames and use .help = these tags to label multiple groups of experiments. title = None .type = str .help = Title for the plot """ phil_scope = parse(phil_str) class Script(object): '''A class for running the script.''' def __init__(self): '''Initialise the script.''' from dials.util.options import OptionParser import libtbx.load_env # The script usage usage = "usage: %s [options] [param.phil] filenames" % libtbx.env.dispatcher_name self.tag = None self.reference_detector = None
# make dx, dy distortion maps from two detector models, a moving and a reference # from hierarchy_level=0 refinement and =1 respectively - at the moment it # assumes P6M detector with exactly 60 panels from __future__ import absolute_import, division, print_function from scitbx.array_family import flex from scitbx import matrix from libtbx import phil phil_scope = phil.parse(""" map_prefix = '' .type = str .help = 'Prefix for output dx, dy pickle files' """) def offset_dx_dy_p6m(detector0, detector1): """Compute pixel-remap function between detector1 and detector0, assuming panels are arranged [0][1][2][3][4] in the fast direction first etc. This also assumes that the rotation of pixels is locally small, but on the scale of a panel (80 mm ish) the rotation matters. N.B. Also assumes that the panel gap in the map is 7 pixels in fast direction and 17 in slow direction. Offsets are with respect to the fast and slow directions on the original coordinate system. """ assert len(detector0) == 60 assert len(detector1) == 60 dx = flex.double(flex.grid(2527, 2463), 0.0)
This program allows you to combine reflection lists. Note that merge in this case refers to merging lists rather than merging equivalent reflections. Reflection tables can either be extended (i.e. given two reflection tables, "A" and "B", the result would have length = len("A") + len("B")). Or values in one table can be overwritten with values from the other table. Any columns present in "A" but not present in "B" (or vice versa) will be preserved. """ # Create the phil parameters phil_scope = parse(""" output = merged.refl .type = str .help = "The output file" method = *update extend .type = choice .help = "The method of merging" """) class Script(object): """A class to encapsulate the script.""" def __init__(self): """Initialise the script.""" # The script usage usage = "usage: dev.dials.merge_reflection_lists [options] /path/to/image/reflection/files" self.parser = OptionParser(epilog=help_message, usage=usage, phil=phil_scope,
integration { integrator = stills profile.fitting = False background { algorithm = simple simple { outlier.algorithm = plane model.algorithm = linear2d } } } profile.gaussian_rs.min_spots.overall = 0 ''' phil_scope = parse(control_phil_str + dials_phil_str, process_includes=True).fetch( parse(program_defaults_phil_str)) def do_import(filename): logger.info("Loading %s" % os.path.basename(filename)) datablocks = DataBlockFactory.from_filenames([filename]) if len(datablocks) == 0: try: datablocks = DataBlockFactory.from_json_file(filename) except ValueError: raise Abort("Could not load %s" % filename) if len(datablocks) == 0: raise Abort("Could not load %s" % filename) if len(datablocks) > 1:
phil_scope = phil.parse( """ assess_space_group = True .type = bool .help = "Option to assess space group by testing presence of axial reflections" anomalous = True .type = bool .help = "Output anomalous as well as mean intensities." truncate = True .type = bool .help = "Option to perform truncation on merged data." d_min = None .type = float .help = "High resolution limit to apply to the data." d_max = None .type = float .help = "Low resolution limit to apply to the data." combine_partials = True .type = bool .help = "Combine partials that have the same partial id into one reflection, with an updated partiality given by the sum of the individual partialities." partiality_threshold=0.4 .type = float .help = "All reflections with partiality values above the partiality threshold will be retained. This is done after any combination of partials if applicable." n_residues = 200 .type = int .help = "Number of residues to use in Wilson scaling" merging { use_internal_variance = False .type = bool n_bins = 20 .type = int(value_min=5) anomalous = False .type = bool .help = "Option to control whether reported merging stats are anomalous." } reporting { wilson_stats = True .type = bool .help = "Option to turn off reporting of Wilson statistics" merging_stats = True .type = bool .help = "Option to turn off reporting of merging statistics." } output { log = dials.merge.log .type = str mtz = merged.mtz .type = str .help = "Filename to use for mtz output." crystal_names = XTAL .type = strings .help = "Crystal name to be used in MTZ file output (multiple names allowed for MAD datasets)" project_name = AUTOMATIC .type = str .help = "Project name to be used in MTZ file output" dataset_names = NATIVE .type = strings .help = "Dataset name to be used in MTZ file output (multiple names allowed for MAD datasets)" } include scope cctbx.french_wilson.master_phil """, process_includes=True, )
def run(args): if len(args) == 0 or '-h' in args or '--help' in args or '-c' in args: print("Usage: %s [-p] files" % libtbx.env.dispatcher_name) phil_scope.show(attributes_level=2) return files = [arg for arg in args if os.path.isfile(arg)] arguments = [arg for arg in args if not os.path.isfile(arg)] user_phil = [] for arg in arguments: if arg == '-p': user_phil.append(parse("show_plots=True")) else: try: user_phil.append(parse(arg)) except Exception as e: raise Sorry("Unrecognized argument: %s" % arg) params = phil_scope.fetch(sources=user_phil).extract() for file in files: message = """Based on the file %s, this program will compute incremental quadrant translations to circularize powder rings on the inner four sensors. The algorithm treats each quadrant independently and scores based on self-correlation upon 45-degree rotation. """ % file print(message) image = dxtbx.load(file) detector = image.get_detector() beam = image.get_beam() from xfel.metrology.quadrant import one_panel ccs = flex.double() for i_quad, quad in enumerate(detector.hierarchy()): # find panel closest to the beam center panels = [] def recursive_get_panels(group): if group.is_group(): for child in group: recursive_get_panels(child) else: panels.append(group) recursive_get_panels(quad) smallest_dist = float("inf") for panel in panels: p_w, p_h = panel.get_image_size() c = center([ col(panel.get_pixel_lab_coord((0, 0))), col(panel.get_pixel_lab_coord((p_w - 1, 0))), col(panel.get_pixel_lab_coord((p_w - 1, p_h - 1))), col(panel.get_pixel_lab_coord((0, p_h - 1))) ]) beam_center = col(panel.get_beam_centre_lab(beam.get_s0())) dist = (c - beam_center).length() if dist < smallest_dist: smallest_dist = dist key_panel = panel print("Doing cross-correlation on panel", key_panel.get_name()) Q = one_panel(image, key_panel, i_quad, quad, params.show_plots, params.multi_angle, params.plot_range, params.pdf_file is None) delta = panel.pixel_to_millimeter((Q.coordmax[0], Q.coordmax[1])) quad.set_frame( quad.get_fast_axis(), quad.get_slow_axis(), col(quad.get_origin()) - col((delta[0], delta[1], 0))) ccs.append(Q.ccmax) print("Average CC: %7.4f" % flex.mean(ccs)) if params.pdf_file is not None: print("Saving plots to", params.pdf_file) from matplotlib import pyplot as plt from matplotlib.backends.backend_pdf import PdfPages pp = PdfPages(params.pdf_file) for i in plt.get_fignums(): pp.savefig(plt.figure(i), dpi=300) pp.close() if params.save_cbf: import pycbf image.sync_detector_to_cbf() dest_path = os.path.splitext(file)[0] + "_cc.cbf" print("Saving result to", dest_path) image._cbf_handle.write_widefile(dest_path,pycbf.CBF,\ pycbf.MIME_HEADERS|pycbf.MSG_DIGEST|pycbf.PAD_4K,0)
def test_from_phil(): beam = Beam((0, 0, 1)) params = detector_phil_scope.fetch( parse(""" detector { panel { id = 0 origin = (1, 1, 1) pixel_size = (0.001,0.001) image_size = (1000,1000) trusted_range = (-1, 1000) material = "Si" thickness = 0.01 parallax_correction = True } panel { id = 1 origin = (2, 2, 2) pixel_size = (0.001,0.001) image_size = (1000,1000) trusted_range = (-1, 1000) } panel { id = 2 origin = (3, 3, 3) pixel_size = (0.001,0.001) image_size = (1000,1000) trusted_range = (-1, 1000) } panel { id = 3 origin = (4, 4, 4) pixel_size = (0.001,0.001) image_size = (1000,1000) trusted_range = (-1, 1000) } hierarchy { name = "Root" origin = (100, 100, 100) group { id = 0 origin = (10, 10, 10) } group { id = 0,0 origin = (1, 1, 1) panel = 0 } group { id = 0,1 origin = (2, 2, 2) panel = 1 } group { id = 1 origin = (20, 20, 20) } group { id = 1,0 origin = (1, 1, 1) panel = 2 } group { id = 1,1 origin = (2, 2, 2) panel = 3 } } } """)).extract() # Test create model d1 = DetectorFactory.from_phil(params, beam=beam) root = d1.hierarchy() # Check hierarchy origins assert root.get_origin() == (100, 100, 100) assert root[0].get_origin() == (110, 110, 110) assert root[1].get_origin() == (120, 120, 120) assert root[0][0].get_origin() == (111, 111, 111) assert root[0][1].get_origin() == (112, 112, 112) assert root[1][0].get_origin() == (121, 121, 121) assert root[1][1].get_origin() == (122, 122, 122) assert root[0][0][0].get_origin() == (112, 112, 112) assert root[0][1][0].get_origin() == (114, 114, 114) assert root[1][0][0].get_origin() == (124, 124, 124) assert root[1][1][0].get_origin() == (126, 126, 126) # Check panels are correct in hierarchy assert root[0][0][0].is_(d1[0]) assert root[0][1][0].is_(d1[1]) assert root[1][0][0].is_(d1[2]) assert root[1][1][0].is_(d1[3]) # Check panel attributes assert d1[0].get_image_size() == (1000, 1000) assert d1[0].get_pixel_size() == (0.001, 0.001) assert d1[0].get_trusted_range() == (-1, 1000) assert d1[0].get_material() == "Si" assert d1[0].get_thickness() == 0.01 assert isinstance(d1[0].get_px_mm_strategy(), ParallaxCorrectedPxMmStrategy) assert d1[1].get_image_size() == (1000, 1000) assert d1[1].get_pixel_size() == (0.001, 0.001) assert d1[1].get_trusted_range() == (-1, 1000) assert d1[2].get_image_size() == (1000, 1000) assert d1[2].get_pixel_size() == (0.001, 0.001) assert d1[2].get_trusted_range() == (-1, 1000) assert d1[3].get_image_size() == (1000, 1000) assert d1[3].get_pixel_size() == (0.001, 0.001) assert d1[3].get_trusted_range() == (-1, 1000) params = detector_phil_scope.fetch( parse(""" detector { panel { id = 0 parallax_correction = False } panel { id = 1 material = "Si" thickness = 0.01 parallax_correction = True } hierarchy { name = "Root" origin = (200, 200, 200) group { id = 0 origin = (20, 20, 20) } group { id = 0,0 origin = (2, 2, 2) } group { id = 0,1 origin = (3, 3, 3) } group { id = 1 origin = (30, 30, 30) } group { id = 1,0 origin = (2, 2, 2) } group { id = 1,1 origin = (3, 3, 3) } } } """)).extract() # Test overwrite model d2 = DetectorFactory.from_phil(params, reference=d1, beam=beam) root = d2.hierarchy() # Check hierarchy origins assert root.get_origin() == (200, 200, 200) assert root[0].get_origin() == (220, 220, 220) assert root[1].get_origin() == (230, 230, 230) assert root[0][0].get_origin() == (222, 222, 222) assert root[0][1].get_origin() == (223, 223, 223) assert root[1][0].get_origin() == (232, 232, 232) assert root[1][1].get_origin() == (233, 233, 233) assert root[0][0][0].get_origin() == (223, 223, 223) assert root[0][1][0].get_origin() == (225, 225, 225) assert root[1][0][0].get_origin() == (235, 235, 235) assert root[1][1][0].get_origin() == (237, 237, 237) # Check panels are correct in hierarchy assert root[0][0][0].is_(d2[0]) assert root[0][1][0].is_(d2[1]) assert root[1][0][0].is_(d2[2]) assert root[1][1][0].is_(d2[3]) # Check panel attributes assert not isinstance(d2[0].get_px_mm_strategy(), ParallaxCorrectedPxMmStrategy) assert d2[1].get_material() == "Si" assert d2[1].get_thickness() == 0.01 assert isinstance(d2[1].get_px_mm_strategy(), ParallaxCorrectedPxMmStrategy)
def run(): have_dials_regression = libtbx.env.has_module("dials_regression") if not have_dials_regression: print "Skipped: dials_regression not available" return dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) from dials.test.algorithms.indexing.tst_index import run_one_indexing expected_unit_cell = uctbx.unit_cell( (11.624, 13.550, 30.103, 89.964, 93.721, 90.132)) expected_rmsds = (0.039, 0.035, 0.002) datablock_old = os.path.join( dials_regression, "indexing_test_data/phi_scan/datablock_old.json") datablock_new = os.path.join( dials_regression, "indexing_test_data/phi_scan/datablock.json") strong_pickle = os.path.join( dials_regression, "indexing_test_data/phi_scan/strong.pickle") from dxtbx.serialize import load imageset_old = load.datablock( datablock_old, check_format=False)[0].extract_imagesets()[0] imageset_new = load.datablock( datablock_new, check_format=False)[0].extract_imagesets()[0] gonio_old = imageset_old.get_goniometer() gonio_new = imageset_new.get_goniometer() assert approx_equal( gonio_old.get_rotation_axis(), (0.7497646259807715, -0.5517923303436749, 0.36520984351713554)) assert approx_equal( gonio_old.get_setting_rotation(), (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0)) assert approx_equal( gonio_old.get_fixed_rotation(), (0.7497646259807748, -0.20997265900532208, -0.6275065641872948, -0.5517923303436731, 0.3250014637526764, -0.7680490041218182, 0.3652098435171313, 0.9221092836691605, 0.12781329809272568)) assert approx_equal( gonio_new.get_rotation_axis(), gonio_old.get_rotation_axis()) assert approx_equal(gonio_new.get_rotation_axis_datum(), (1,0,0)) assert approx_equal( gonio_new.get_setting_rotation(), (0.7497646259807705, -0.20997265900532142, -0.6275065641873, -0.5517923303436786, 0.3250014637526763, -0.768049004121814, 0.3652098435171315, 0.9221092836691607, 0.12781329809272335)) assert approx_equal( gonio_new.get_fixed_rotation(), (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0)) result_old = run_one_indexing( pickle_path=strong_pickle, sweep_path=datablock_old, extra_args=[], expected_unit_cell=expected_unit_cell, expected_rmsds=expected_rmsds, expected_hall_symbol=' P 1', ) result_new = run_one_indexing( pickle_path=strong_pickle, sweep_path=datablock_new, extra_args=[], expected_unit_cell=expected_unit_cell, expected_rmsds=expected_rmsds, expected_hall_symbol=' P 1', ) assert approx_equal(result_old.rmsds, result_new.rmsds) assert approx_equal(result_old.crystal_model.get_unit_cell().parameters(), result_new.crystal_model.get_unit_cell().parameters()) # Now test refinement gradients are correct from dxtbx.model.experiment_list import ExperimentList, Experiment old_exps=ExperimentList([Experiment(beam=imageset_old.get_beam(), detector=imageset_old.get_detector(), goniometer=gonio_old, scan=imageset_old.get_scan(), crystal=result_old.crystal_model, imageset=None)]) new_exps=ExperimentList([Experiment(beam=imageset_new.get_beam(), detector=imageset_new.get_detector(), goniometer=gonio_new, scan=imageset_new.get_scan(), crystal=result_new.crystal_model, imageset=None)]) from libtbx.phil import parse from dials.algorithms.refinement.refiner import phil_scope params = phil_scope.fetch(source=parse('')).extract() from dials.algorithms.refinement.refiner import RefinerFactory refiner_old = RefinerFactory.from_parameters_data_experiments(params, result_old.indexed_reflections, old_exps, verbosity=0) refiner_new = RefinerFactory.from_parameters_data_experiments(params, result_new.indexed_reflections, new_exps, verbosity=0) # Analytical gradients should be approximately the same in either case an_grads_old = refiner_old._pred_param.get_gradients(refiner_old.get_matches()) an_grads_new = refiner_new._pred_param.get_gradients(refiner_new.get_matches()) for g1, g2 in zip(an_grads_old, an_grads_new): assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=1.e-6) assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=1.e-6) assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=1.e-6) # Analytical gradients should be approximately equal to finite difference # gradients in either case fd_grads_old = calc_fd_grads(refiner_old) for g1, g2 in zip(fd_grads_old, an_grads_old): assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=5.e-6) assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=5.e-6) assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=5.e-6) fd_grads_new = calc_fd_grads(refiner_new) for g1, g2 in zip(fd_grads_new, an_grads_new): assert approx_equal(g1["dX_dp"], g2["dX_dp"], eps=5.e-6) assert approx_equal(g1["dY_dp"], g2["dY_dp"], eps=5.e-6) assert approx_equal(g1["dphi_dp"], g2["dphi_dp"], eps=5.e-6)
input_path=cxic0415/output/debug ''' phil_scope = parse(''' input_path = . .type = str .help = path to where the processing results are. For example path to XXX_rgYYYY num_nodes = 1 .type = int .help = Number of nodes used to do data processing. Used in timing information num_cores_per_node = 72 .type = int .help = Number of cores per node in the machine (default is for Cori KNL) wall_time = 3600 .type = int .help = total wall time (seconds) taken for job to finish. Used for plotting node-partitioning plot_title = Computational weather plot .type = str .help = title of the computational weather plot show_plot = True .type = bool .help = flag to indicate if plot should be displayed on screen pickle_plot = False .type = bool .help = If True, will pickle matplotlib session so that it can be opened later for analysis/viewing \ https://stackoverflow.com/questions/29160177/matplotlib-save-file-to-be-reedited-later-in-ipython pickle_filename = fig_object.pickle .type = str .help = Default name of pickled matplotlib plot saved to disk ''')
fcalc_f = "/Users/dermen/cxid9114_gain/sim/fcalc_slim.pkl" #outdir = "ssirp_cell.beam" #outdir = " img_f = sys.argv[1] # "xtc_102.loc" outdir = os.path.join(sys.argv[2], os.path.basename(img_f).replace(".", "_")) print outdir if not os.path.exists(outdir): os.makedirs(outdir) BEAM = utils.open_flex(sim_utils.beam_f) sad_wave = parameters.ENERGY_CONV / 8950 BEAM.set_wavelength(sad_wave) MULTI_PANEL = True spot_par = find_spots_phil_scope.fetch(source=parse("")).extract() spot_par_moder = deepcopy(spot_par) par1 = 0 mask_f = "../mask/dials_mask_64panels_2.pkl" if par1: spot_par.spotfinder.threshold.dispersion.global_threshold = 60. spot_par.spotfinder.threshold.dispersion.gain = 28. spot_par.spotfinder.threshold.dispersion.kernel_size = [3, 3] spot_par.spotfinder.threshold.dispersion.sigma_strong = 1.5 spot_par.spotfinder.threshold.dispersion.sigma_background = 6. spot_par.spotfinder.filter.min_spot_size = 3 spot_par.spotfinder.force_2d = True spot_par.spotfinder.lookup.mask = mask_f
Example: %s experiment1.expt experiment2.expt reflections1.refl reflections2.refl ''' % libtbx.env.dispatcher_name # Create the phil parameters phil_scope = parse(''' tag = None .type = str .help = Used in the plot titles hierarchy_level=0 .type=int .help=Provide congruence statistics for detector modules at the given hierarchy level. colormap=RdYlGn_r .type=str .help=matplotlib color map. See e.g.: \ http://matplotlib.org/examples/color/colormaps_reference.html show_plots=False .type=bool .help=Whether to show congruence plots draw_normal_arrows=False .type=bool .help=Whether to draw the XY components of each panel group's normal vector. Useful \ for visualizing tilt. ''') def iterate_detector_at_level(item, depth=0, level=0): """ Iterate through all panel groups or panels of a detector object at a given hierarchy level