def run(args): usage = "dials.estimate_resolution [options] (scaled.expt scaled.refl | scaled_unmerged.mtz)" import libtbx.load_env if libtbx.env.dispatcher_name == "dials.resolutionizer": warnings.warn( "dials.resolutionizer is now deprecated, please use dials.estimate_resolution instead" ) parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options, unhandled = parser.parse_args( args=args, return_unhandled=True, show_diff_phil=True ) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments ) if (not reflections or not experiments) and not unhandled: parser.print_help() return if reflections and experiments and unhandled: sys.exit( "Must provide either scaled unmerged mtz OR dials-format scaled reflections and experiments files" ) # Configure the logging log.config(logfile=params.output.log, verbosity=options.verbose) logger.info(dials_version()) if len(unhandled) == 1: scaled_unmerged = unhandled[0] m = resolution_analysis.Resolutionizer.from_unmerged_mtz( scaled_unmerged, params.resolution ) else: reflections = parse_multiple_datasets(reflections) m = resolution_analysis.Resolutionizer.from_reflections_and_experiments( reflections, experiments, params.resolution ) plots = m.resolution_auto() if params.output.html: output_html_report(plots, params.output.html) if params.output.json: with open(params.output.json, "w") as fh: json.dump(plots, fh) return plots
def protk_experiments_and_reflections(dials_data): data_dir = dials_data("multi_crystal_proteinase_k") # Load experiments experiments = ExperimentList() for expt_file in sorted(f.strpath for f in data_dir.listdir("experiments*.json")): experiments.extend(load.experiment_list(expt_file, check_format=False)) # Load reflection tables reflections = [ flex.reflection_table.from_file(refl_file) for refl_file in sorted( f.strpath for f in data_dir.listdir("reflections*.pickle")) ] # Setup experiment identifiers reflections = parse_multiple_datasets(reflections) experiments, reflections = assign_unique_identifiers( experiments, reflections) # Combine into single ExperimentList reflections_all = flex.reflection_table() for i, (expt, refl) in enumerate(zip(experiments, reflections)): reflections_all.extend(refl) reflections_all.assert_experiment_identifiers_are_consistent(experiments) return experiments, reflections_all
def run(args=None): usage = "dials.cosym [options] models.expt observations.refl" parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options, args = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) # Configure the logging log.config(verbosity=options.verbose, logfile=params.output.log) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.seed is not None: flex.set_random_seed(params.seed) np.random.seed(params.seed) random.seed(params.seed) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): raise Sorry( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) try: experiments, reflections = assign_unique_identifiers( experiments, reflections) cosym_instance = cosym(experiments=experiments, reflections=reflections, params=params) except ValueError as e: raise Sorry(e) if params.output.html or params.output.json: register_default_cosym_observers(cosym_instance) cosym_instance.run() cosym_instance.export()
def run(args): usage = "dials.symmetry [options] models.expt observations.refl" parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, _, args = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) # Configure the logging log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.seed is not None: import random flex.set_random_seed(params.seed) random.seed(params.seed) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): raise Sorry( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) try: experiments, reflections = assign_unique_identifiers( experiments, reflections) symmetry(experiments, reflections, params=params) except ValueError as e: raise Sorry(e)
def run(args=None): usage = "dials.estimate_resolution [options] (scaled.expt scaled.refl | scaled_unmerged.mtz)" parser = ArgumentParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=__doc__, ) params, options, unhandled = parser.parse_args(args=args, return_unhandled=True, show_diff_phil=True) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) if (not reflections or not experiments) and not unhandled: parser.print_help() return if reflections and experiments and unhandled: sys.exit( "Must provide either scaled unmerged mtz OR dials-format scaled reflections and experiments files" ) # Configure the logging log.config(logfile=params.output.log, verbosity=options.verbose) logger.info(dials_version()) if len(unhandled) == 1: scaled_unmerged = unhandled[0] m = resolution_analysis.Resolutionizer.from_unmerged_mtz( scaled_unmerged, params.resolution) else: reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): sys.exit( f"Mismatched number of experiments and reflection tables found: {len(experiments)} & {len(reflections)}." ) m = resolution_analysis.Resolutionizer.from_reflections_and_experiments( reflections, experiments, params.resolution) plots = m.resolution_auto() if params.output.html: output_html_report(plots, params.output.html) if params.output.json: with open(params.output.json, "w") as fh: json.dump(plots, fh) return plots
def run(args=None): """Run symmetry analysis from the command-line.""" usage = "dials.symmetry [options] models.expt observations.refl" parser = ArgumentParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options, args = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) # Configure the logging log.config(verbosity=options.verbose, logfile=params.output.log) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.seed is not None: flex.set_random_seed(params.seed) random.seed(params.seed) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): sys.exit( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) try: experiments, reflections = assign_unique_identifiers( experiments, reflections) symmetry(experiments, reflections, params=params) except ValueError as e: sys.exit(e)
def test_parse_multiple_datasets(): """Test the namesake function. This expects a list of reflection tables, and selects on the column named 'id'.""" rt1 = flex.reflection_table() rt1["id"] = flex.int([0, 0, 0]) rt1.experiment_identifiers()[0] = "0" rt2 = flex.reflection_table() rt2["id"] = flex.int([2, 2, 4, 4]) rt2.experiment_identifiers()[2] = "2" rt2.experiment_identifiers()[4] = "4" single_tables = parse_multiple_datasets([rt2]) assert len(single_tables) == 2 assert list(set(single_tables[0]["id"])) == [2] assert list(set(single_tables[1]["id"])) == [4] single_tables = parse_multiple_datasets([rt1, rt2]) assert list(set(single_tables[0]["id"])) == [0] assert list(set(single_tables[1]["id"])) == [2] assert list(set(single_tables[2]["id"])) == [4] assert len(single_tables) == 3 single_tables = parse_multiple_datasets([rt1]) assert len(single_tables) == 1 assert list(set(single_tables[0]["id"])) == [0] # if a duplicate id is given, then this should be detected and new ids # determined for all datasets. rt3 = flex.reflection_table() rt3["id"] = flex.int([2, 2]) rt3.experiment_identifiers()[2] = "5" single_tables = parse_multiple_datasets([rt1, rt2, rt3]) assert len(single_tables) == 4 assert list(set(single_tables[0]["id"])) == [0] assert single_tables[0].experiment_identifiers()[0] == "0" assert list(set(single_tables[1]["id"])) == [1] assert single_tables[1].experiment_identifiers()[1] == "2" assert list(set(single_tables[2]["id"])) == [2] assert single_tables[2].experiment_identifiers()[2] == "4" assert list(set(single_tables[3]["id"])) == [3] assert single_tables[3].experiment_identifiers()[3] == "5"
def delta_cc_half_analysis(self): # transform models into miller arrays intensities, batches = filtered_arrays_from_experiments_reflections( self._data_manager.experiments, parse_multiple_datasets([self._data_manager.reflections]), outlier_rejection_after_filter=False, partiality_threshold=0.99, return_batches=True, ) result = DeltaCcHalf(intensities, batches) d = {} d.update(result.histogram()) d.update(result.normalised_scores()) return d, result.get_table(html=True)
def run(args): usage = ( "dials.resolutionizer [options] (scaled.expt scaled.refl | scaled_unmerged.mtz)" ) parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options, unhandled = parser.parse_args(return_unhandled=True, show_diff_phil=True) reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) if (not reflections or not experiments) and not unhandled: parser.print_help() return if reflections and experiments and unhandled: sys.exit( "Must provide either scaled unmerged mtz OR dials-format scaled reflections and experiments files" ) # Configure the logging log.config(logfile=params.output.log, verbosity=options.verbose) logger.info(dials_version()) if len(unhandled) == 1: scaled_unmerged = unhandled[0] m = resolutionizer.Resolutionizer.from_unmerged_mtz( scaled_unmerged, params.resolutionizer) else: reflections = parse_multiple_datasets(reflections) m = resolutionizer.Resolutionizer.from_reflections_and_experiments( reflections, experiments, params.resolutionizer) m.resolution_auto()
def run(args=None): """Run assign experiment identifiers from the command line.""" usage = ( """Usage: dials.assign_experiment_identifiers observations.refl models.expt""" ) parser = ArgumentParser( usage=usage, read_experiments=True, read_reflections=True, phil=phil_scope, check_format=False, epilog=help_message, ) params, _ = parser.parse_args(args=args, show_diff_phil=False) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): raise Sorry( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) try: experiments, reflections = assign_unique_identifiers( experiments, reflections, params.identifiers) except ValueError as e: raise Sorry(e) print(f"assigned identifiers: {list(experiments.identifiers())}") experiments.as_file(params.output.experiments) joint_table = flex.reflection_table() for reflection_table in reflections: joint_table.extend(reflection_table) joint_table.as_file(params.output.reflections)
def run(): # The script usage usage = ("usage: xia2.multi_crystal_analysis [options] [param.phil] " "models.expt observations.refl") # Create the parser parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) # Parse the command line params, options = parser.parse_args(show_diff_phil=False) # Configure the logging for name in ("xia2", "dials"): log.config(verbosity=options.verbose, logfile=params.output.log, name=name) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) # Try to load the models and data if len(params.input.experiments) == 0: logger.info("No Experiments found in the input") parser.print_help() return if len(params.input.reflections) == 0: logger.info("No reflection data found in the input") parser.print_help() return try: assert len(params.input.reflections) == len(params.input.experiments) except AssertionError: raise Sorry("The number of input reflections files does not match the " "number of input experiments") if params.seed is not None: flex.set_random_seed(params.seed) random.seed(params.seed) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) reflections = parse_multiple_datasets(reflections) joint_table = flex.reflection_table() for i in range(len(reflections)): joint_table.extend(reflections[i]) reflections = joint_table MultiCrystalReport(params, experiments=experiments, reflections=reflections).report()
def prepare_input(params, experiments, reflections): """Perform checks on the data and prepare the data for scaling. Raises: ValueError - a range of checks are made, a ValueError may be raised for a number of reasons. """ #### First exclude any datasets, before the dataset is split into #### individual reflection tables and expids set. if (params.dataset_selection.exclude_datasets or params.dataset_selection.use_datasets): experiments, reflections = select_datasets_on_ids( experiments, reflections, params.dataset_selection.exclude_datasets, params.dataset_selection.use_datasets, ) ids = flex.size_t() for r in reflections: ids.extend(r.experiment_identifiers().keys()) logger.info( "\nDataset ids for retained datasets are: %s \n", ",".join(str(i) for i in ids), ) #### Split the reflections tables into a list of reflection tables, #### with one table per experiment. logger.info("Checking for the existence of a reflection table \n" "containing multiple datasets \n") reflections = parse_multiple_datasets(reflections) logger.info( "Found %s reflection tables & %s experiments in total.", len(reflections), len(experiments), ) if len(experiments) != len(reflections): raise ValueError( "Mismatched number of experiments and reflection tables found.") #### Assign experiment identifiers. experiments, reflections = assign_unique_identifiers( experiments, reflections) ids = itertools.chain.from_iterable(r.experiment_identifiers().keys() for r in reflections) logger.info("\nDataset ids are: %s \n", ",".join(str(i) for i in ids)) for r in reflections: r.unset_flags(flex.bool(len(r), True), r.flags.bad_for_scaling) r.unset_flags(flex.bool(r.size(), True), r.flags.scaled) reflections, experiments = exclude_image_ranges_for_scaling( reflections, experiments, params.exclude_images) #### Allow checking of consistent indexing, useful for #### targeted / incremental scaling. if params.scaling_options.check_consistent_indexing: logger.info("Running dials.cosym to check consistent indexing:\n") cosym_params = cosym_phil_scope.extract() cosym_params.nproc = params.scaling_options.nproc cosym_instance = cosym(experiments, reflections, cosym_params) cosym_instance.run() experiments = cosym_instance.experiments reflections = cosym_instance.reflections logger.info("Finished running dials.cosym, continuing with scaling.\n") #### Make sure all experiments in same space group sgs = [ expt.crystal.get_space_group().type().number() for expt in experiments ] if len(set(sgs)) > 1: raise ValueError("""The experiments have different space groups: space group numbers found: %s Please reanalyse the data so that space groups are consistent, (consider using dials.reindex, dials.symmetry or dials.cosym) or remove incompatible experiments (using the option exclude_datasets=)""" % ", ".join(map(str, set(sgs)))) logger.info( "Space group being used during scaling is %s", experiments[0].crystal.get_space_group().info(), ) #### If doing targeted scaling, extract data and append an experiment #### and reflection table to the lists if params.scaling_options.target_model: logger.info("Extracting data from structural model.") exp, reflection_table = create_datastructures_for_structural_model( reflections, experiments, params.scaling_options.target_model) experiments.append(exp) reflections.append(reflection_table) elif params.scaling_options.target_mtz: logger.info("Extracting data from merged mtz.") exp, reflection_table = create_datastructures_for_target_mtz( experiments, params.scaling_options.target_mtz) experiments.append(exp) reflections.append(reflection_table) #### Perform any non-batch cutting of the datasets, including the target dataset best_unit_cell = params.reflection_selection.best_unit_cell if best_unit_cell is None: best_unit_cell = determine_best_unit_cell(experiments) for reflection in reflections: if params.cut_data.d_min or params.cut_data.d_max: d = best_unit_cell.d(reflection["miller_index"]) if params.cut_data.d_min: sel = d < params.cut_data.d_min reflection.set_flags(sel, reflection.flags.user_excluded_in_scaling) if params.cut_data.d_max: sel = d > params.cut_data.d_max reflection.set_flags(sel, reflection.flags.user_excluded_in_scaling) if params.cut_data.partiality_cutoff and "partiality" in reflection: reflection.set_flags( reflection["partiality"] < params.cut_data.partiality_cutoff, reflection.flags.user_excluded_in_scaling, ) return params, experiments, reflections
def export_mtz(integrated_data, experiment_list, params): """Export data from integrated_data corresponding to experiment_list to an MTZ file hklout.""" # if mtz filename is auto, then choose scaled.mtz or integrated.mtz if params.mtz.hklout in (None, Auto, "auto"): if ("intensity.scale.value" in integrated_data) and ("intensity.scale.variance" in integrated_data): params.mtz.hklout = "scaled.mtz" logger.info( "Data appears to be scaled, setting mtz.hklout = 'scaled.mtz'") else: params.mtz.hklout = "integrated.mtz" logger.info( "Data appears to be unscaled, setting mtz.hklout = 'integrated.mtz'" ) # First get the experiment identifier information out of the data expids_in_table = integrated_data.experiment_identifiers() if not list(expids_in_table.keys()): reflection_tables = parse_multiple_datasets([integrated_data]) experiment_list, refl_list = assign_unique_identifiers( experiment_list, reflection_tables) integrated_data = flex.reflection_table() for reflections in refl_list: integrated_data.extend(reflections) expids_in_table = integrated_data.experiment_identifiers() integrated_data.assert_experiment_identifiers_are_consistent( experiment_list) expids_in_list = list(experiment_list.identifiers()) # Convert experiment_list to a real python list or else identity assumptions # fail like: # assert experiment_list[0] is experiment_list[0] # And assumptions about added attributes break experiment_list = list(experiment_list) # Validate multi-experiment assumptions if len(experiment_list) > 1: # All experiments should match crystals, or else we need multiple crystals/datasets if not all(x.crystal == experiment_list[0].crystal for x in experiment_list[1:]): logger.warning( "Experiment crystals differ. Using first experiment crystal for file-level data." ) wavelengths = match_wavelengths(experiment_list) if len(wavelengths.keys()) > 1: logger.info( "Multiple wavelengths found: \n%s", "\n".join(" Wavlength: %.5f, experiment numbers: %s " % (k, ",".join(map(str, v))) for k, v in wavelengths.items()), ) else: wavelengths = OrderedDict( {experiment_list[0].beam.get_wavelength(): [0]}) # also only work correctly with one panel (for the moment) if any(len(experiment.detector) != 1 for experiment in experiment_list): logger.warning("Ignoring multiple panels in output MTZ") best_unit_cell = params.mtz.best_unit_cell if best_unit_cell is None: best_unit_cell = determine_best_unit_cell(experiment_list) integrated_data["d"] = best_unit_cell.d(integrated_data["miller_index"]) # Clean up the data with the passed in options integrated_data = filter_reflection_table( integrated_data, intensity_choice=params.intensity, partiality_threshold=params.mtz.partiality_threshold, combine_partials=params.mtz.combine_partials, min_isigi=params.mtz.min_isigi, filter_ice_rings=params.mtz.filter_ice_rings, d_min=params.mtz.d_min, ) # get batch offsets and image ranges - even for scanless experiments batch_offsets = [ expt.scan.get_batch_offset() for expt in experiment_list if expt.scan is not None ] unique_offsets = set(batch_offsets) if len(set(unique_offsets)) <= 1: logger.debug("Calculating new batches") batch_offsets = calculate_batch_offsets(experiment_list) batch_starts = [ e.scan.get_image_range()[0] if e.scan else 0 for e in experiment_list ] effective_offsets = [ o + s for o, s in zip(batch_offsets, batch_starts) ] unique_offsets = set(effective_offsets) else: logger.debug("Keeping existing batches") image_ranges = get_image_ranges(experiment_list) if len(unique_offsets) != len(batch_offsets): raise ValueError("Duplicate batch offsets detected: %s" % ", ".join( str(item) for item, count in Counter(batch_offsets).items() if count > 1)) # Create the mtz file mtz_writer = UnmergedMTZWriter( experiment_list[0].crystal.get_space_group()) # FIXME TODO for more than one experiment into an MTZ file: # # - add an epoch (or recover an epoch) from the scan and add this as an extra # column to the MTZ file for scaling, so we know that the two lattices were # integrated at the same time # ✓ decide a sensible BATCH increment to apply to the BATCH value between # experiments and add this for id_ in expids_in_table.keys(): # Grab our subset of the data loc = expids_in_list.index( expids_in_table[id_]) # get strid and use to find loc in list experiment = experiment_list[loc] if len(list(wavelengths.keys())) > 1: for i, (wl, exps) in enumerate(wavelengths.items()): if loc in exps: wavelength = wl dataset_id = i + 1 break else: wavelength = list(wavelengths.keys())[0] dataset_id = 1 reflections = integrated_data.select(integrated_data["id"] == id_) batch_offset = batch_offsets[loc] image_range = image_ranges[loc] reflections = assign_batches_to_reflections([reflections], [batch_offset])[0] experiment.data = dict(reflections) s0n = matrix.col(experiment.beam.get_s0()).normalize().elems logger.debug("Beam vector: %.4f %.4f %.4f" % s0n) mtz_writer.add_batch_list( image_range, experiment, wavelength, dataset_id, batch_offset=batch_offset, force_static_model=params.mtz.force_static_model, ) # Create the batch offset array. This gives us an experiment (id)-dependent # batch offset to calculate the correct batch from image number. experiment.data["batch_offset"] = flex.int(len(experiment.data["id"]), batch_offset) # Calculate whether we have a ROT value for this experiment, and set the column _, _, z = experiment.data["xyzcal.px"].parts() if experiment.scan: experiment.data[ "ROT"] = experiment.scan.get_angle_from_array_index(z) else: experiment.data["ROT"] = z mtz_writer.add_crystal( crystal_name=params.mtz.crystal_name, project_name=params.mtz.project_name, unit_cell=best_unit_cell, ) # Note: add unit cell here as may have changed basis since creating mtz. # For multi-wave unmerged mtz, we add an empty dataset for each wavelength, # but only write the data into the final dataset (for unmerged the batches # link the unmerged data to the individual wavelengths). for wavelength in wavelengths: mtz_writer.add_empty_dataset(wavelength) # Combine all of the experiment data columns before writing combined_data = { k: v.deep_copy() for k, v in experiment_list[0].data.items() } for experiment in experiment_list[1:]: for k, v in experiment.data.items(): combined_data[k].extend(v) # ALL columns must be the same length assert len({len(v) for v in combined_data.values() }) == 1, "Column length mismatch" assert len(combined_data["id"]) == len( integrated_data["id"]), "Lost rows in split/combine" # Write all the data and columns to the mtz file mtz_writer.write_columns(combined_data) logger.info("Saving {} integrated reflections to {}".format( len(combined_data["id"]), params.mtz.hklout)) mtz_file = mtz_writer.mtz_file mtz_file.write(params.mtz.hklout) return mtz_file
def run(args=None, phil=phil_scope): # type: (List[str], libtbx.phil.scope) -> None usage = "dials.missing_reflections [options] scaled.expt scaled.refl" parser = OptionParser( usage=usage, phil=phil, read_reflections=True, read_experiments=True, check_format=False, epilog=__doc__, ) params, options = parser.parse_args(args=args, show_diff_phil=False) # Configure the logging. dials.util.log.config(options.verbose) logger.info(dials_version()) # Log the difference between the PHIL scope definition and the active PHIL scope, # which will include the parsed user inputs. diff_phil = parser.diff_phil.as_str() if diff_phil: logger.info("The following parameters have been modified:\n%s", diff_phil) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if not experiments or not reflections: parser.print_help() return if len(reflections) != 1 and len(experiments) != len(reflections): sys.exit( "Number of experiments must equal the number of reflection tables") from dials.util.multi_dataset_handling import ( assign_unique_identifiers, parse_multiple_datasets, ) reflections = parse_multiple_datasets(reflections) experiments, reflections = assign_unique_identifiers( experiments, reflections) if all("inverse_scale_factor" in refl for refl in reflections): # Assume all arrays have been scaled miller_array = scaled_data_as_miller_array(reflections, experiments, anomalous_flag=False) else: # Else get the integrated intensities miller_arrays = filtered_arrays_from_experiments_reflections( experiments, reflections, ) miller_array = miller_arrays[0] for ma in miller_arrays[1:]: miller_array = miller_array.concatenate(ma) if params.d_min or params.d_max: miller_array = miller_array.resolution_filter(d_min=params.d_min, d_max=params.d_max) # Print overall summary of input miller array s = io.StringIO() ma_unique = miller_array.unique_under_symmetry() ma_unique.show_comprehensive_summary(f=s) logger.info(f"\n{s.getvalue()}") # Get the regions of missing reflections complete_set, unique_ms = missing_reflections.connected_components( miller_array) unique_ms = [ ms for ms in unique_ms if ms.size() >= params.min_component_size ] # Print some output for user if len(unique_ms): logger.info( "The following connected regions of missing reflections have been identified:" ) n_expected = complete_set.size() rows = [] for ms in unique_ms: d_max, d_min = (uctbx.d_star_sq_as_d(ds2) for ds2 in ms.min_max_d_star_sq()) rows.append([ ms.size(), f"{100 * ms.size() / n_expected:.1f}", f"{d_max:.2f}-{d_min:.2f}", ]) logger.info( tabulate( rows, headers=["# reflections", "% missing", "Resolution range (Å)"])) else: logger.info("No connected regions of missing reflections identified")
def run(self): """Run cycles of scaling and filtering.""" with ScalingHTMLContextManager(self): start_time = time.time() results = AnalysisResults() for counter in range( 1, self.params.filtering.deltacchalf.max_cycles + 1): self.run_scaling_cycle() if counter == 1: results.initial_expids_and_image_ranges = [ (exp.identifier, exp.scan.get_image_range()) if exp.scan else None for exp in self.experiments ] delta_cc_params = deltacc_phil_scope.extract() delta_cc_params.mode = self.params.filtering.deltacchalf.mode delta_cc_params.group_size = ( self.params.filtering.deltacchalf.group_size) delta_cc_params.stdcutoff = self.params.filtering.deltacchalf.stdcutoff logger.info("\nPerforming a round of filtering.\n") # need to reduce to single table. joined_reflections = flex.reflection_table() for table in self.reflections: joined_reflections.extend(table) script = deltaccscript(delta_cc_params, self.experiments, joined_reflections) script.run() valid_image_ranges = get_valid_image_ranges(self.experiments) results.expids_and_image_ranges = [ (exp.identifier, valid_image_ranges[i]) if exp.scan else None for i, exp in enumerate(self.experiments) ] self.experiments = script.experiments self.params.dataset_selection.use_datasets = None self.params.dataset_selection.exclude_datasets = None results = log_cycle_results(results, self, script) logger.info( "Cycle %s of filtering, n_reflections removed this cycle: %s", counter, results.get_last_cycle_results()["n_removed"], ) # Test termination conditions latest_results = results.get_last_cycle_results() if latest_results["n_removed"] == 0: logger.info( "Finishing scaling and filtering as no data removed in this cycle." ) if self.params.scaling_options.full_matrix: self.reflections = parse_multiple_datasets( [script.filtered_reflection_table]) results = self._run_final_scale_cycle(results) else: self.reflections = [script.filtered_reflection_table] results.finish(termination_reason="no_more_removed") break # Need to split reflections for further processing. self.reflections = parse_multiple_datasets( [script.filtered_reflection_table]) if (latest_results["cumul_percent_removed"] > self.params.filtering.deltacchalf.max_percent_removed): logger.info( "Finishing scale and filtering as have now removed more than the limit." ) results = self._run_final_scale_cycle(results) results.finish(termination_reason="max_percent_removed") break if self.params.filtering.deltacchalf.min_completeness: if (latest_results["merging_stats"]["completeness"] < self.params.filtering.deltacchalf.min_completeness ): logger.info( "Finishing scaling and filtering as completeness now below cutoff." ) results = self._run_final_scale_cycle(results) results.finish( termination_reason="below_completeness_limit") break if counter == self.params.filtering.deltacchalf.max_cycles: logger.info("Finishing as reached max number of cycles.") results = self._run_final_scale_cycle(results) results.finish(termination_reason="max_cycles") break # If not finished then need to create new scaler to try again self._create_model_and_scaler() self.filtering_results = results # Print summary of results logger.info(results) with open(self.params.filtering.output.scale_and_filter_results, "w") as f: json.dump(self.filtering_results.to_dict(), f, indent=2) # All done! logger.info("\nTotal time taken: %.4fs ", time.time() - start_time) logger.info("%s%s%s", "\n", "=" * 80, "\n")
def run(args=sys.argv[1:]): # Create the parser parser = OptionParser( # usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, # epilog=help_message, ) # Parse the command line params, options, args = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) # Configure the logging xia2.Handlers.Streams.setup_logging(logfile=params.output.log, verbose=options.verbose) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.unit_cell is not None: unit_cell = params.unit_cell crystal_symmetry = crystal.symmetry(unit_cell=unit_cell) else: crystal_symmetry = None if len(params.input.experiments): experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): raise sys.exit( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) experiments, reflections = assign_unique_identifiers( experiments, reflections) # transform models into miller arrays intensities, batches = filtered_arrays_from_experiments_reflections( experiments, reflections, outlier_rejection_after_filter=False, partiality_threshold=0.99, return_batches=True, ) if args and os.path.isfile(args[0]): result = any_reflection_file(args[0]) unmerged_intensities = None batches_all = None for ma in result.as_miller_arrays(merge_equivalents=False, crystal_symmetry=crystal_symmetry): if ma.info().labels == ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]: assert ma.anomalous_flag() unmerged_intensities = ma elif ma.info().labels == ["I", "SIGI"]: assert not ma.anomalous_flag() unmerged_intensities = ma elif ma.info().labels == ["BATCH"]: batches_all = ma assert batches_all is not None assert unmerged_intensities is not None sel = unmerged_intensities.sigmas() > 0 unmerged_intensities = unmerged_intensities.select(sel).set_info( unmerged_intensities.info()) batches_all = batches_all.select(sel) id_to_batches = None if len(params.batch) > 0: id_to_batches = {} for b in params.batch: assert b.id is not None assert b.range is not None assert b.id not in id_to_batches, "Duplicate batch id: %s" % b.id id_to_batches[b.id] = b.range separate = separate_unmerged(unmerged_intensities, batches_all, id_to_batches=id_to_batches) batches = list(separate.batches.values()) intensities = list(separate.intensities.values()) result = DeltaCcHalf( intensities, batches, n_bins=params.n_bins, d_min=params.d_min, cc_one_half_method=params.cc_one_half_method, group_size=params.group_size, ) logger.info(tabulate(result.get_table(), headers="firstrow")) hist_filename = "delta_cc_hist.png" logger.info("Saving histogram to %s" % hist_filename) result.plot_histogram(hist_filename) normalised_scores_filename = "normalised_scores.png" logger.info("Saving normalised scores to %s" % normalised_scores_filename) result.plot_normalised_scores(normalised_scores_filename) Citations.cite("delta_cc_half") for citation in Citations.get_citations_acta(): logger.info(citation)
def run(args=None, phil=phil_scope): # type: (List[str], libtbx.phil.scope) -> None """ Check command-line input and call other functions to do the legwork. Run the script, parsing arguments found in 'args' and using the PHIL scope defined in 'phil'. Args: args: The arguments supplied by the user (default: sys.argv[1:]) phil: The PHIL scope definition (default: phil_scope, the master PHIL scope for this program). """ usage = "dials.command_name [options] imported.expt strong.refl" parser = OptionParser( usage=usage, phil=phil, read_reflections=True, read_experiments=True, check_format=False, epilog=__doc__, ) params, options = parser.parse_args(args=args, show_diff_phil=False) # Configure the logging. dials.util.log.config(options.verbose) # Log the difference between the PHIL scope definition and the active PHIL scope, # which will include the parsed user inputs. diff_phil = parser.diff_phil.as_str() if diff_phil: logger.info("The following parameters have been modified:\n%s", diff_phil) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if not experiments or not reflections: parser.print_help() return if len(reflections) != 1 and len(experiments) != len(reflections): sys.exit( "Number of experiments must equal the number of reflection tables") from dials.util.multi_dataset_handling import ( assign_unique_identifiers, parse_multiple_datasets, ) reflections = parse_multiple_datasets(reflections) experiments, reflections = assign_unique_identifiers( experiments, reflections) # Do whatever this program is supposed to do. do_connected_components( experiments, reflections, min_component_size=params.min_component_size, )
def run(self): """Execute the script.""" start_time = time() # Parse the command line params, _ = self.parser.parse_args(show_diff_phil=False) # set up global reflections list reflections = flex.reflection_table() # loop through the input, building up the global lists reflections_list = flatten_reflections(params.input.reflections) experiments = flatten_experiments(params.input.experiments) reflections_list = parse_multiple_datasets(reflections_list) for refs in reflections_list: reflections.extend(refs) # Try to load the models and data nexp = len(experiments) if nexp == 0: print("No Experiments found in the input") self.parser.print_help() return if not reflections: print("No reflection data found in the input") self.parser.print_help() return self.check_input(reflections) # Configure the logging log.config(logfile=params.output.log) logger.info(dials_version()) # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) # Convert to P 1? if params.refinement.triclinic: reflections, experiments = self.convert_to_P1(reflections, experiments) # Combine crystals? if params.refinement.combine_crystal_models and len(experiments) > 1: logger.info("Combining {0} crystal models".format(len(experiments))) experiments = self.combine_crystals(experiments) # Filter integrated centroids? if params.refinement.filter_integrated_centroids: reflections = self.filter_integrated_centroids(reflections) # Filter data if scaled to remove outliers if "inverse_scale_factor" in reflections: try: reflections = filter_reflection_table(reflections, ["scale"]) except ValueError as e: logger.warn(e) logger.info( "Filtering on scaled data failed, proceeding with integrated data." ) # Get the refiner logger.info("Configuring refiner") refiner = self.create_refiner(params, reflections, experiments) # Refine the geometry if nexp == 1: logger.info("Performing refinement of a single Experiment...") else: logger.info("Performing refinement of {} Experiments...".format(nexp)) refiner.run() # get the refined experiments experiments = refiner.get_experiments() crystals = experiments.crystals() if len(crystals) == 1: # output the refined model for information logger.info("") logger.info("Final refined crystal model:") logger.info(crystals[0]) logger.info(self.cell_param_table(crystals[0])) # Save the refined experiments to file output_experiments_filename = params.output.experiments logger.info( "Saving refined experiments to {}".format(output_experiments_filename) ) experiments.as_file(output_experiments_filename) # Create correlation plots if params.output.correlation_plot.filename is not None: create_correlation_plots(refiner, params.output) if params.output.cif is not None: self.generate_cif(crystals[0], refiner, filename=params.output.cif) if params.output.p4p is not None: self.generate_p4p( crystals[0], experiments[0].beam, filename=params.output.p4p ) if params.output.mmcif is not None: self.generate_mmcif(crystals[0], refiner, filename=params.output.mmcif) # Log the total time taken logger.info("\nTotal time taken: {:.2f}s".format(time() - start_time))
def run(args: List[str] = None, phil: libtbx.phil.scope = phil_scope) -> None: """ Run dials.anvil_correction as from the command line. Take integrated experiment lists and reflection tables and correct the all the integrated intensities for the estimated attenuation by the diamond anvils. Args: args: The arguments supplied by the user (default: sys.argv[1:]). phil: The PHIL scope definition (default: phil_scope, the master PHIL scope for this program). """ usage = "dials.anvil_correction [options] integrated.expt integrated.refl" parser = OptionParser( usage=usage, phil=phil, read_reflections=True, read_experiments=True, check_format=False, epilog=__doc__, ) params, options = parser.parse_args(args=args, show_diff_phil=False) # Log the difference between the PHIL scope definition and the active PHIL scope, # which will include the parsed user inputs. diff_phil = parser.diff_phil.as_str() if diff_phil: logger.info("The following parameters have been modified:\n%s", diff_phil) # Check that at least one reflection table and experiment list have been provided. input_errors = [] if not params.input.experiments: input_errors.append( "Please provide at least one valid experiment list (.expt) file.") if not params.input.reflections: input_errors.append( "Please provide at least one valid reflection table (.refl) file.") if input_errors: sys.exit("\n".join([parser.format_help()] + input_errors)) if not np.linalg.norm(params.anvil.normal): sys.exit( "It seems you have provided a surface normal vector with zero length." ) # Check that the anvil surface normal really is normalised. dac_norm = params.anvil.normal / np.linalg.norm(params.anvil.normal) # Configure the logging. dials.util.log.config(options.verbose, logfile=params.output.log) # These functions are commonly used to collate the input. experiments = flatten_experiments(params.input.experiments) reflections_list = flatten_reflections(params.input.reflections) # Work around parse_multiple_datasets dropping unindexed reflections. unindexed = flex.reflection_table() for r_table in reflections_list: unindexed.extend(r_table.select(r_table["id"] == -1)) # Get a single reflection table per experiment object. reflections_list = parse_multiple_datasets(reflections_list) reflections_list = sort_tables_to_experiments_order( reflections_list, experiments) # Record the density of diamond in g·cm⁻³ (for consistency with NIST tables, # https://doi.org/10.18434/T4D01F). density = params.anvil.density / 1000 # g·cm⁻³ # Correct for the attenuation of the incident and diffracted beams by the diamond # anvil cell. logger.info( "Correcting integrated reflection intensities for attenuation by the diamond " "anvil cell.") for experiment, reflections in zip(experiments, reflections_list): correct_intensities_for_dac_attenuation(experiment, reflections, dac_norm, params.anvil.thickness, density) logger.info("Done.") # Do optional experiment list file output here. if params.output.experiments: logger.info("Writing the experiment list to %s", params.output.experiments) experiments.as_file(params.output.experiments) logger.info("Writing the reflection table to %s", params.output.reflections) # Collate reflections into a single reflection table and save it to file. reflections = unindexed for r_table in reflections_list: reflections.extend(r_table) del reflections_list reflections.as_file(params.output.reflections)
def run(args): usage = ("xia2.multiplex [options] [param.phil] " "models1.expt models2.expt observations1.refl " "observations2.refl...") # Create the parser parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) # Parse the command line params, options = parser.parse_args(args=args, show_diff_phil=False) # Configure the logging xia2.Handlers.Streams.setup_logging(logfile=params.output.log, verbose=options.verbose) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) # Try to load the models and data if len(params.input.experiments) == 0: logger.info("No Experiments found in the input") parser.print_help() return if len(params.input.reflections) == 0: logger.info("No reflection data found in the input") parser.print_help() return try: assert len(params.input.reflections) == len(params.input.experiments) except AssertionError: raise sys.exit( "The number of input reflections files does not match the " "number of input experiments") if params.seed is not None: flex.set_random_seed(params.seed) random.seed(params.seed) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if len(experiments) < 2: sys.exit("xia2.multiplex requires a minimum of two experiments") reflections = parse_multiple_datasets(reflections) experiments, reflections = assign_unique_identifiers( experiments, reflections) reflections, experiments = exclude_image_ranges_for_scaling( reflections, experiments, params.exclude_images) reflections_all = flex.reflection_table() assert len(reflections) == 1 or len(reflections) == len(experiments) for i, (expt, refl) in enumerate(zip(experiments, reflections)): reflections_all.extend(refl) reflections_all.assert_experiment_identifiers_are_consistent(experiments) if params.identifiers is not None: identifiers = [] for identifier in params.identifiers: identifiers.extend(identifier.split(",")) params.identifiers = identifiers try: ScaleAndMerge.MultiCrystalScale(experiments, reflections_all, params) except ValueError as e: sys.exit(str(e))