def _make_input_for_exclude_tests(exclude_images=True): """Generate input data, that upon exclusion should leave only the first reflection.""" if exclude_images: exclude_images = [["0:360:720"], ["1:360:720"]] expt1 = Experiment(scan=Scan(image_range=(0, 720), oscillation=(0.0, 1.0))) expt2 = Experiment(scan=Scan(image_range=(0, 720), oscillation=(0.0, -1.0))) refls1 = flex.reflection_table() refls2 = flex.reflection_table() refls1["xyzobs.mm.value"] = flex.vec3_double([ (0.0, 0.0, 10.0 * math.pi / 180.0), (0.0, 0.0, 370.0 * math.pi / 180.0) ]) refls1["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 10.0), (0.0, 0.0, 370.0)]) refls1["i"] = flex.int([0, 1]) refls2["xyzobs.mm.value"] = flex.vec3_double([ (0.0, 0.0, -10.0 * math.pi / 180.0), (0.0, 0.0, -370.0 * math.pi / 180.0) ]) refls2["xyzobs.px.value"] = flex.vec3_double([(0.0, 0.0, 10.0), (0.0, 0.0, 370.0)]) refls2["i"] = flex.int([0, 1]) expts = ExperimentList([expt1, expt2]) tables = [refls1, refls2] expts, tables = assign_unique_identifiers(expts, tables) return exclude_images, expts, tables
def test_assigning_specified_identifiers(experiments, reflections): reflections[0].experiment_identifiers()[0] = "5" reflections[1].experiment_identifiers()[1] = "6" reflections[1].experiment_identifiers()[4] = "7" input_identifiers = ["0", "1", "10"] exp, rts = assign_unique_identifiers(experiments, reflections, input_identifiers) assert list(exp.identifiers()) == input_identifiers assert rts[0].experiment_identifiers()[0] == "0" assert rts[0]["id"][0] == 0 assert rts[1].experiment_identifiers()[1] == "1" assert rts[1]["id"][0] == 1 assert rts[2].experiment_identifiers()[2] == "10" assert rts[2]["id"][0] == 2 # Test raises ValueError when wrong number of identifiers given with pytest.raises(ValueError): exp, rts = assign_unique_identifiers(experiments, reflections, ["0", "1"])
def test_assign_identifiers_where_none_are_set_but_refl_table_ids_have_duplicates( experiments, reflections): reflections[2]["id"] = flex.int([0, 0]) expts, rts = assign_unique_identifiers(experiments, reflections) for i, (expt, refl) in enumerate(zip(expts, rts)): assert set(refl["id"]) == {i} assert expt.identifier != "" assert refl.experiment_identifiers()[i] == expt.identifier
def test_assignment_of_unique_identifiers_when_refl_table_ids_are_present( experiments, reflections): assert list(experiments.identifiers()) == ["", "", ""] expts, rts = assign_unique_identifiers(experiments, reflections) for i, (expt, refl) in enumerate(zip(expts, rts)): assert set(refl["id"]) == {i} assert expt.identifier != "" assert refl.experiment_identifiers()[i] == expt.identifier
def run(args=None): usage = "dials.cosym [options] models.expt observations.refl" parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options, args = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) # Configure the logging log.config(verbosity=options.verbose, logfile=params.output.log) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.seed is not None: flex.set_random_seed(params.seed) np.random.seed(params.seed) random.seed(params.seed) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): raise Sorry( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) try: experiments, reflections = assign_unique_identifiers( experiments, reflections) cosym_instance = cosym(experiments=experiments, reflections=reflections, params=params) except ValueError as e: raise Sorry(e) if params.output.html or params.output.json: register_default_cosym_observers(cosym_instance) cosym_instance.run() cosym_instance.export()
def test_raise_exception_if_unequal_experiments_and_reflections(experiments_024): reflections_multi = [flex.reflection_table()] reflections_multi[0]["id"] = flex.int([0, 1, 4]) reflections_multi[0].experiment_identifiers()[0] = "0" reflections_multi[0].experiment_identifiers()[1] = "2" reflections_multi[0].experiment_identifiers()[4] = "4" del reflections_multi[0].experiment_identifiers()[4] del reflections_multi[0]["id"][2] with pytest.raises(ValueError): exp, rts = assign_unique_identifiers(experiments_024, reflections_multi)
def run(args): usage = "dials.symmetry [options] models.expt observations.refl" parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, _, args = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) # Configure the logging log.config(params.verbosity, info=params.output.log, debug=params.output.debug_log) from dials.util.version import dials_version logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.seed is not None: import random flex.set_random_seed(params.seed) random.seed(params.seed) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): raise Sorry( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) try: experiments, reflections = assign_unique_identifiers( experiments, reflections) symmetry(experiments, reflections, params=params) except ValueError as e: raise Sorry(e)
def test_assigned_identifiers_are_kept_when_assigning_rest( experiments, reflections): # Now test that if some are set, these are maintained and unique ids are # set for the rest experiments[0].identifier = "1" reflections[0].experiment_identifiers()[0] = "1" expts, rts = assign_unique_identifiers(experiments, reflections) assert expts.identifiers()[0] == "1" for i, (expt, refl) in enumerate(zip(expts, rts)): assert set(refl["id"]) == {i} assert expt.identifier != "" assert refl.experiment_identifiers()[i] == expt.identifier
def test_assign_identifiers_where_none_are_set_but_refl_table_ids_have_duplicates( experiments, reflections ): reflections[2]["id"] = flex.int([0, 0]) exp, rts = assign_unique_identifiers(experiments, reflections) expected_identifiers = ["0", "1", "2"] # Check that identifiers are set in experiments and reflection table. assert list(exp.identifiers()) == expected_identifiers expected_ids = [0, 1, 2] for i, refl in enumerate(rts): assert refl.experiment_identifiers()[i] == expected_identifiers[i] assert set(refl["id"]) == {i} assert i == expected_ids[i]
def test_assignment_of_unique_identifiers_when_refl_table_ids_are_present( experiments, reflections ): assert list(experiments.identifiers()) == ["", "", ""] exp, rts = assign_unique_identifiers(experiments, reflections) expected_identifiers = ["0", "1", "2"] expected_ids = [0, 1, 2] # Check that identifiers are set in experiments and reflection table. assert list(exp.identifiers()) == expected_identifiers for i, refl in enumerate(rts): assert refl.experiment_identifiers()[i] == expected_identifiers[i] assert set(refl["id"]) == {i} assert i == expected_ids[i]
def test_mtz_multi_wavelength(dials_data, run_in_tmpdir): """Test multi-wavelength mtz export""" # First make suitable input - multi datasets experiment list and reflection # table with different wavelengths mcp = dials_data("multi_crystal_proteinase_k") exp_1 = load.experiment_list( mcp.join("experiments_1.json").strpath, check_format=False ) exp_2 = load.experiment_list( mcp.join("experiments_2.json").strpath, check_format=False ) refl_1 = flex.reflection_table.from_file(mcp.join("reflections_1.pickle").strpath) refl_2 = flex.reflection_table.from_file(mcp.join("reflections_2.pickle").strpath) exp_1[0].beam.set_wavelength(0.5) exp_2[0].beam.set_wavelength(1.0) exp_1.extend(exp_2) reflection_list = [refl_1, refl_2] exps, refls = assign_unique_identifiers(exp_1, reflection_list) joint_refl = flex.reflection_table() for r in refls: joint_refl.extend(r) exps.as_json("tmp_exp.expt") joint_refl.as_file("tmp_refl.refl") # Now run result = procrunner.run( [ "dials.export", "experiments=tmp_exp.expt", "reflections=tmp_refl.refl", "format=mtz", "mtz.hklout=unmerged.mtz", ], environment_override={"DIALS_EXPORT_DO_NOT_CHECK_FORMAT": "True"}, working_directory=run_in_tmpdir.strpath, ) assert not result.returncode and not result.stderr assert os.path.exists("unmerged.mtz") # Inspect output m = mtz.object("unmerged.mtz").crystals() n_batches = [] wavelengths = [] for crystal in m: for dataset in crystal.datasets(): wavelengths.append(dataset.wavelength()) n_batches.append(dataset.n_batches()) assert n_batches == [0, 25, 25] # base, dataset1, dataset2 assert wavelengths == [0, 0.5, 1.0] # base, dataset1, dataset2
def run(args=None): """Run symmetry analysis from the command-line.""" usage = "dials.symmetry [options] models.expt observations.refl" parser = ArgumentParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) params, options, args = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) # Configure the logging log.config(verbosity=options.verbose, logfile=params.output.log) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.seed is not None: flex.set_random_seed(params.seed) random.seed(params.seed) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): sys.exit( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) try: experiments, reflections = assign_unique_identifiers( experiments, reflections) symmetry(experiments, reflections, params=params) except ValueError as e: sys.exit(e)
def test_assigned_identifiers_are_kept_when_assigning_rest(experiments, reflections): # Now test that if some are set, these are maintained and unique ids are # set for the rest experiments[0].identifier = "1" reflections[0].experiment_identifiers()[0] = "1" exp, rts = assign_unique_identifiers(experiments, reflections) expected_identifiers = ["1", "0", "2"] assert list(exp.identifiers()) == expected_identifiers expected_ids = [0, 1, 2] for i, refl in enumerate(rts): id_ = refl["id"][0] assert refl.experiment_identifiers()[id_] == expected_identifiers[i] assert set(refl["id"]) == {id_} assert id_ == expected_ids[i]
def test_cases_where_all_set_whether_reflection_table_is_split_or_not( experiments_024, reflections_024): # should pass experiments back if identifiers all already set exp, rts = assign_unique_identifiers(experiments_024, reflections_024) expected_identifiers = ["0", "2", "4"] # Check that identifiers are set in experiments and reflection table. assert exp is experiments_024 assert list(exp.identifiers()) == expected_identifiers expected_ids = [0, 1, 4] for i, refl in enumerate(rts): id_ = refl["id"][0] assert refl.experiment_identifiers()[id_] == expected_identifiers[i] assert set(refl["id"]) == {id_} assert id_ == expected_ids[i]
def test_synthetic( space_group, unit_cell, dimensions, sample_size, use_known_space_group, use_known_lattice_group, tmpdir, ): os.chdir(tmpdir.strpath) space_group = sgtbx.space_group_info(space_group).group() if unit_cell is not None: unit_cell = uctbx.unit_cell(unit_cell) experiments, reflections, reindexing_ops = generate_experiments_reflections( space_group=space_group, unit_cell=unit_cell, unit_cell_volume=10000, sample_size=sample_size, map_to_p1=True, d_min=1.5, ) params = phil_scope.extract() if use_known_space_group: params.space_group = space_group.info() if use_known_lattice_group: params.lattice_group = space_group.info() if dimensions is not None: params.dimensions = dimensions experiments, reflections = assign_unique_identifiers( experiments, reflections) cosym_instance = cosym(experiments, reflections, params=params) register_default_cosym_observers(cosym_instance) cosym_instance.run() cosym_instance.export() assert os.path.exists(params.output.experiments) assert os.path.exists(params.output.reflections) assert os.path.exists(params.output.html) assert os.path.exists(params.output.json) cosym_expts = load.experiment_list(params.output.experiments, check_format=False) assert len(cosym_expts) == len(experiments) for expt in cosym_expts: if unit_cell is not None: assert expt.crystal.get_unit_cell().parameters() == pytest.approx( unit_cell.parameters()) assert str(expt.crystal.get_space_group().info()) == str( space_group.info()) assert expt.crystal.get_space_group() == space_group
def run(args=None): """Run assign experiment identifiers from the command line.""" usage = ( """Usage: dials.assign_experiment_identifiers observations.refl models.expt""" ) parser = ArgumentParser( usage=usage, read_experiments=True, read_reflections=True, phil=phil_scope, check_format=False, epilog=help_message, ) params, _ = parser.parse_args(args=args, show_diff_phil=False) if not params.input.experiments or not params.input.reflections: parser.print_help() sys.exit() reflections, experiments = reflections_and_experiments_from_files( params.input.reflections, params.input.experiments) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): raise Sorry( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) try: experiments, reflections = assign_unique_identifiers( experiments, reflections, params.identifiers) except ValueError as e: raise Sorry(e) print(f"assigned identifiers: {list(experiments.identifiers())}") experiments.as_file(params.output.experiments) joint_table = flex.reflection_table() for reflection_table in reflections: joint_table.extend(reflection_table) joint_table.as_file(params.output.reflections)
def export_mtz(integrated_data, experiment_list, params): """Export data from integrated_data corresponding to experiment_list to an MTZ file hklout.""" # if mtz filename is auto, then choose scaled.mtz or integrated.mtz if params.mtz.hklout in (None, Auto, "auto"): if ("intensity.scale.value" in integrated_data) and ("intensity.scale.variance" in integrated_data): params.mtz.hklout = "scaled.mtz" logger.info( "Data appears to be scaled, setting mtz.hklout = 'scaled.mtz'") else: params.mtz.hklout = "integrated.mtz" logger.info( "Data appears to be unscaled, setting mtz.hklout = 'integrated.mtz'" ) # First get the experiment identifier information out of the data expids_in_table = integrated_data.experiment_identifiers() if not list(expids_in_table.keys()): reflection_tables = parse_multiple_datasets([integrated_data]) experiment_list, refl_list = assign_unique_identifiers( experiment_list, reflection_tables) integrated_data = flex.reflection_table() for reflections in refl_list: integrated_data.extend(reflections) expids_in_table = integrated_data.experiment_identifiers() integrated_data.assert_experiment_identifiers_are_consistent( experiment_list) expids_in_list = list(experiment_list.identifiers()) # Convert experiment_list to a real python list or else identity assumptions # fail like: # assert experiment_list[0] is experiment_list[0] # And assumptions about added attributes break experiment_list = list(experiment_list) # Validate multi-experiment assumptions if len(experiment_list) > 1: # All experiments should match crystals, or else we need multiple crystals/datasets if not all(x.crystal == experiment_list[0].crystal for x in experiment_list[1:]): logger.warning( "Experiment crystals differ. Using first experiment crystal for file-level data." ) wavelengths = match_wavelengths(experiment_list) if len(wavelengths.keys()) > 1: logger.info( "Multiple wavelengths found: \n%s", "\n".join(" Wavlength: %.5f, experiment numbers: %s " % (k, ",".join(map(str, v))) for k, v in wavelengths.items()), ) else: wavelengths = OrderedDict( {experiment_list[0].beam.get_wavelength(): [0]}) # also only work correctly with one panel (for the moment) if any(len(experiment.detector) != 1 for experiment in experiment_list): logger.warning("Ignoring multiple panels in output MTZ") best_unit_cell = params.mtz.best_unit_cell if best_unit_cell is None: best_unit_cell = determine_best_unit_cell(experiment_list) integrated_data["d"] = best_unit_cell.d(integrated_data["miller_index"]) # Clean up the data with the passed in options integrated_data = filter_reflection_table( integrated_data, intensity_choice=params.intensity, partiality_threshold=params.mtz.partiality_threshold, combine_partials=params.mtz.combine_partials, min_isigi=params.mtz.min_isigi, filter_ice_rings=params.mtz.filter_ice_rings, d_min=params.mtz.d_min, ) # get batch offsets and image ranges - even for scanless experiments batch_offsets = [ expt.scan.get_batch_offset() for expt in experiment_list if expt.scan is not None ] unique_offsets = set(batch_offsets) if len(set(unique_offsets)) <= 1: logger.debug("Calculating new batches") batch_offsets = calculate_batch_offsets(experiment_list) batch_starts = [ e.scan.get_image_range()[0] if e.scan else 0 for e in experiment_list ] effective_offsets = [ o + s for o, s in zip(batch_offsets, batch_starts) ] unique_offsets = set(effective_offsets) else: logger.debug("Keeping existing batches") image_ranges = get_image_ranges(experiment_list) if len(unique_offsets) != len(batch_offsets): raise ValueError("Duplicate batch offsets detected: %s" % ", ".join( str(item) for item, count in Counter(batch_offsets).items() if count > 1)) # Create the mtz file mtz_writer = UnmergedMTZWriter( experiment_list[0].crystal.get_space_group()) # FIXME TODO for more than one experiment into an MTZ file: # # - add an epoch (or recover an epoch) from the scan and add this as an extra # column to the MTZ file for scaling, so we know that the two lattices were # integrated at the same time # ✓ decide a sensible BATCH increment to apply to the BATCH value between # experiments and add this for id_ in expids_in_table.keys(): # Grab our subset of the data loc = expids_in_list.index( expids_in_table[id_]) # get strid and use to find loc in list experiment = experiment_list[loc] if len(list(wavelengths.keys())) > 1: for i, (wl, exps) in enumerate(wavelengths.items()): if loc in exps: wavelength = wl dataset_id = i + 1 break else: wavelength = list(wavelengths.keys())[0] dataset_id = 1 reflections = integrated_data.select(integrated_data["id"] == id_) batch_offset = batch_offsets[loc] image_range = image_ranges[loc] reflections = assign_batches_to_reflections([reflections], [batch_offset])[0] experiment.data = dict(reflections) s0n = matrix.col(experiment.beam.get_s0()).normalize().elems logger.debug("Beam vector: %.4f %.4f %.4f" % s0n) mtz_writer.add_batch_list( image_range, experiment, wavelength, dataset_id, batch_offset=batch_offset, force_static_model=params.mtz.force_static_model, ) # Create the batch offset array. This gives us an experiment (id)-dependent # batch offset to calculate the correct batch from image number. experiment.data["batch_offset"] = flex.int(len(experiment.data["id"]), batch_offset) # Calculate whether we have a ROT value for this experiment, and set the column _, _, z = experiment.data["xyzcal.px"].parts() if experiment.scan: experiment.data[ "ROT"] = experiment.scan.get_angle_from_array_index(z) else: experiment.data["ROT"] = z mtz_writer.add_crystal( crystal_name=params.mtz.crystal_name, project_name=params.mtz.project_name, unit_cell=best_unit_cell, ) # Note: add unit cell here as may have changed basis since creating mtz. # For multi-wave unmerged mtz, we add an empty dataset for each wavelength, # but only write the data into the final dataset (for unmerged the batches # link the unmerged data to the individual wavelengths). for wavelength in wavelengths: mtz_writer.add_empty_dataset(wavelength) # Combine all of the experiment data columns before writing combined_data = { k: v.deep_copy() for k, v in experiment_list[0].data.items() } for experiment in experiment_list[1:]: for k, v in experiment.data.items(): combined_data[k].extend(v) # ALL columns must be the same length assert len({len(v) for v in combined_data.values() }) == 1, "Column length mismatch" assert len(combined_data["id"]) == len( integrated_data["id"]), "Lost rows in split/combine" # Write all the data and columns to the mtz file mtz_writer.write_columns(combined_data) logger.info("Saving {} integrated reflections to {}".format( len(combined_data["id"]), params.mtz.hklout)) mtz_file = mtz_writer.mtz_file mtz_file.write(params.mtz.hklout) return mtz_file
def run(args): usage = ("xia2.multiplex [options] [param.phil] " "models1.expt models2.expt observations1.refl " "observations2.refl...") # Create the parser parser = OptionParser( usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, epilog=help_message, ) # Parse the command line params, options = parser.parse_args(args=args, show_diff_phil=False) # Configure the logging xia2.Handlers.Streams.setup_logging(logfile=params.output.log, verbose=options.verbose) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) # Try to load the models and data if len(params.input.experiments) == 0: logger.info("No Experiments found in the input") parser.print_help() return if len(params.input.reflections) == 0: logger.info("No reflection data found in the input") parser.print_help() return try: assert len(params.input.reflections) == len(params.input.experiments) except AssertionError: raise sys.exit( "The number of input reflections files does not match the " "number of input experiments") if params.seed is not None: flex.set_random_seed(params.seed) random.seed(params.seed) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if len(experiments) < 2: sys.exit("xia2.multiplex requires a minimum of two experiments") reflections = parse_multiple_datasets(reflections) experiments, reflections = assign_unique_identifiers( experiments, reflections) reflections, experiments = exclude_image_ranges_for_scaling( reflections, experiments, params.exclude_images) reflections_all = flex.reflection_table() assert len(reflections) == 1 or len(reflections) == len(experiments) for i, (expt, refl) in enumerate(zip(experiments, reflections)): reflections_all.extend(refl) reflections_all.assert_experiment_identifiers_are_consistent(experiments) if params.identifiers is not None: identifiers = [] for identifier in params.identifiers: identifiers.extend(identifier.split(",")) params.identifiers = identifiers try: ScaleAndMerge.MultiCrystalScale(experiments, reflections_all, params) except ValueError as e: sys.exit(str(e))
def prepare_input(params, experiments, reflections): """Perform checks on the data and prepare the data for scaling. Raises: ValueError - a range of checks are made, a ValueError may be raised for a number of reasons. """ #### First exclude any datasets, before the dataset is split into #### individual reflection tables and expids set. if (params.dataset_selection.exclude_datasets or params.dataset_selection.use_datasets): experiments, reflections = select_datasets_on_ids( experiments, reflections, params.dataset_selection.exclude_datasets, params.dataset_selection.use_datasets, ) ids = flex.size_t() for r in reflections: ids.extend(r.experiment_identifiers().keys()) logger.info( "\nDataset ids for retained datasets are: %s \n", ",".join(str(i) for i in ids), ) #### Split the reflections tables into a list of reflection tables, #### with one table per experiment. logger.info("Checking for the existence of a reflection table \n" "containing multiple datasets \n") reflections = parse_multiple_datasets(reflections) logger.info( "Found %s reflection tables & %s experiments in total.", len(reflections), len(experiments), ) if len(experiments) != len(reflections): raise ValueError( "Mismatched number of experiments and reflection tables found.") #### Assign experiment identifiers. experiments, reflections = assign_unique_identifiers( experiments, reflections) ids = itertools.chain.from_iterable(r.experiment_identifiers().keys() for r in reflections) logger.info("\nDataset ids are: %s \n", ",".join(str(i) for i in ids)) for r in reflections: r.unset_flags(flex.bool(len(r), True), r.flags.bad_for_scaling) r.unset_flags(flex.bool(r.size(), True), r.flags.scaled) reflections, experiments = exclude_image_ranges_for_scaling( reflections, experiments, params.exclude_images) #### Allow checking of consistent indexing, useful for #### targeted / incremental scaling. if params.scaling_options.check_consistent_indexing: logger.info("Running dials.cosym to check consistent indexing:\n") cosym_params = cosym_phil_scope.extract() cosym_params.nproc = params.scaling_options.nproc cosym_instance = cosym(experiments, reflections, cosym_params) cosym_instance.run() experiments = cosym_instance.experiments reflections = cosym_instance.reflections logger.info("Finished running dials.cosym, continuing with scaling.\n") #### Make sure all experiments in same space group sgs = [ expt.crystal.get_space_group().type().number() for expt in experiments ] if len(set(sgs)) > 1: raise ValueError("""The experiments have different space groups: space group numbers found: %s Please reanalyse the data so that space groups are consistent, (consider using dials.reindex, dials.symmetry or dials.cosym) or remove incompatible experiments (using the option exclude_datasets=)""" % ", ".join(map(str, set(sgs)))) logger.info( "Space group being used during scaling is %s", experiments[0].crystal.get_space_group().info(), ) #### If doing targeted scaling, extract data and append an experiment #### and reflection table to the lists if params.scaling_options.target_model: logger.info("Extracting data from structural model.") exp, reflection_table = create_datastructures_for_structural_model( reflections, experiments, params.scaling_options.target_model) experiments.append(exp) reflections.append(reflection_table) elif params.scaling_options.target_mtz: logger.info("Extracting data from merged mtz.") exp, reflection_table = create_datastructures_for_target_mtz( experiments, params.scaling_options.target_mtz) experiments.append(exp) reflections.append(reflection_table) #### Perform any non-batch cutting of the datasets, including the target dataset best_unit_cell = params.reflection_selection.best_unit_cell if best_unit_cell is None: best_unit_cell = determine_best_unit_cell(experiments) for reflection in reflections: if params.cut_data.d_min or params.cut_data.d_max: d = best_unit_cell.d(reflection["miller_index"]) if params.cut_data.d_min: sel = d < params.cut_data.d_min reflection.set_flags(sel, reflection.flags.user_excluded_in_scaling) if params.cut_data.d_max: sel = d > params.cut_data.d_max reflection.set_flags(sel, reflection.flags.user_excluded_in_scaling) if params.cut_data.partiality_cutoff and "partiality" in reflection: reflection.set_flags( reflection["partiality"] < params.cut_data.partiality_cutoff, reflection.flags.user_excluded_in_scaling, ) return params, experiments, reflections
def run(args=None, phil=phil_scope): # type: (List[str], libtbx.phil.scope) -> None usage = "dials.missing_reflections [options] scaled.expt scaled.refl" parser = OptionParser( usage=usage, phil=phil, read_reflections=True, read_experiments=True, check_format=False, epilog=__doc__, ) params, options = parser.parse_args(args=args, show_diff_phil=False) # Configure the logging. dials.util.log.config(options.verbose) logger.info(dials_version()) # Log the difference between the PHIL scope definition and the active PHIL scope, # which will include the parsed user inputs. diff_phil = parser.diff_phil.as_str() if diff_phil: logger.info("The following parameters have been modified:\n%s", diff_phil) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if not experiments or not reflections: parser.print_help() return if len(reflections) != 1 and len(experiments) != len(reflections): sys.exit( "Number of experiments must equal the number of reflection tables") from dials.util.multi_dataset_handling import ( assign_unique_identifiers, parse_multiple_datasets, ) reflections = parse_multiple_datasets(reflections) experiments, reflections = assign_unique_identifiers( experiments, reflections) if all("inverse_scale_factor" in refl for refl in reflections): # Assume all arrays have been scaled miller_array = scaled_data_as_miller_array(reflections, experiments, anomalous_flag=False) else: # Else get the integrated intensities miller_arrays = filtered_arrays_from_experiments_reflections( experiments, reflections, ) miller_array = miller_arrays[0] for ma in miller_arrays[1:]: miller_array = miller_array.concatenate(ma) if params.d_min or params.d_max: miller_array = miller_array.resolution_filter(d_min=params.d_min, d_max=params.d_max) # Print overall summary of input miller array s = io.StringIO() ma_unique = miller_array.unique_under_symmetry() ma_unique.show_comprehensive_summary(f=s) logger.info(f"\n{s.getvalue()}") # Get the regions of missing reflections complete_set, unique_ms = missing_reflections.connected_components( miller_array) unique_ms = [ ms for ms in unique_ms if ms.size() >= params.min_component_size ] # Print some output for user if len(unique_ms): logger.info( "The following connected regions of missing reflections have been identified:" ) n_expected = complete_set.size() rows = [] for ms in unique_ms: d_max, d_min = (uctbx.d_star_sq_as_d(ds2) for ds2 in ms.min_max_d_star_sq()) rows.append([ ms.size(), f"{100 * ms.size() / n_expected:.1f}", f"{d_max:.2f}-{d_min:.2f}", ]) logger.info( tabulate( rows, headers=["# reflections", "% missing", "Resolution range (Å)"])) else: logger.info("No connected regions of missing reflections identified")
def run(args=sys.argv[1:]): # Create the parser parser = OptionParser( # usage=usage, phil=phil_scope, read_reflections=True, read_experiments=True, check_format=False, # epilog=help_message, ) # Parse the command line params, options, args = parser.parse_args(args=args, show_diff_phil=False, return_unhandled=True) # Configure the logging xia2.Handlers.Streams.setup_logging(logfile=params.output.log, verbose=options.verbose) logger.info(dials_version()) # Log the diff phil diff_phil = parser.diff_phil.as_str() if diff_phil != "": logger.info("The following parameters have been modified:\n") logger.info(diff_phil) if params.unit_cell is not None: unit_cell = params.unit_cell crystal_symmetry = crystal.symmetry(unit_cell=unit_cell) else: crystal_symmetry = None if len(params.input.experiments): experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) reflections = parse_multiple_datasets(reflections) if len(experiments) != len(reflections): raise sys.exit( "Mismatched number of experiments and reflection tables found: %s & %s." % (len(experiments), len(reflections))) experiments, reflections = assign_unique_identifiers( experiments, reflections) # transform models into miller arrays intensities, batches = filtered_arrays_from_experiments_reflections( experiments, reflections, outlier_rejection_after_filter=False, partiality_threshold=0.99, return_batches=True, ) if args and os.path.isfile(args[0]): result = any_reflection_file(args[0]) unmerged_intensities = None batches_all = None for ma in result.as_miller_arrays(merge_equivalents=False, crystal_symmetry=crystal_symmetry): if ma.info().labels == ["I(+)", "SIGI(+)", "I(-)", "SIGI(-)"]: assert ma.anomalous_flag() unmerged_intensities = ma elif ma.info().labels == ["I", "SIGI"]: assert not ma.anomalous_flag() unmerged_intensities = ma elif ma.info().labels == ["BATCH"]: batches_all = ma assert batches_all is not None assert unmerged_intensities is not None sel = unmerged_intensities.sigmas() > 0 unmerged_intensities = unmerged_intensities.select(sel).set_info( unmerged_intensities.info()) batches_all = batches_all.select(sel) id_to_batches = None if len(params.batch) > 0: id_to_batches = {} for b in params.batch: assert b.id is not None assert b.range is not None assert b.id not in id_to_batches, "Duplicate batch id: %s" % b.id id_to_batches[b.id] = b.range separate = separate_unmerged(unmerged_intensities, batches_all, id_to_batches=id_to_batches) batches = list(separate.batches.values()) intensities = list(separate.intensities.values()) result = DeltaCcHalf( intensities, batches, n_bins=params.n_bins, d_min=params.d_min, cc_one_half_method=params.cc_one_half_method, group_size=params.group_size, ) logger.info(tabulate(result.get_table(), headers="firstrow")) hist_filename = "delta_cc_hist.png" logger.info("Saving histogram to %s" % hist_filename) result.plot_histogram(hist_filename) normalised_scores_filename = "normalised_scores.png" logger.info("Saving normalised scores to %s" % normalised_scores_filename) result.plot_normalised_scores(normalised_scores_filename) Citations.cite("delta_cc_half") for citation in Citations.get_citations_acta(): logger.info(citation)
def test_raise_exception_when_existing_identifiers_are_inconsistent( experiments_024, reflections): reflections[1].experiment_identifiers()[0] = "5" # should raise an assertion error for inconsistent identifiers with pytest.raises(ValueError): _, __ = assign_unique_identifiers(experiments_024, reflections)
def run(args=None, phil=phil_scope): # type: (List[str], libtbx.phil.scope) -> None """ Check command-line input and call other functions to do the legwork. Run the script, parsing arguments found in 'args' and using the PHIL scope defined in 'phil'. Args: args: The arguments supplied by the user (default: sys.argv[1:]) phil: The PHIL scope definition (default: phil_scope, the master PHIL scope for this program). """ usage = "dials.command_name [options] imported.expt strong.refl" parser = OptionParser( usage=usage, phil=phil, read_reflections=True, read_experiments=True, check_format=False, epilog=__doc__, ) params, options = parser.parse_args(args=args, show_diff_phil=False) # Configure the logging. dials.util.log.config(options.verbose) # Log the difference between the PHIL scope definition and the active PHIL scope, # which will include the parsed user inputs. diff_phil = parser.diff_phil.as_str() if diff_phil: logger.info("The following parameters have been modified:\n%s", diff_phil) experiments = flatten_experiments(params.input.experiments) reflections = flatten_reflections(params.input.reflections) if not experiments or not reflections: parser.print_help() return if len(reflections) != 1 and len(experiments) != len(reflections): sys.exit( "Number of experiments must equal the number of reflection tables") from dials.util.multi_dataset_handling import ( assign_unique_identifiers, parse_multiple_datasets, ) reflections = parse_multiple_datasets(reflections) experiments, reflections = assign_unique_identifiers( experiments, reflections) # Do whatever this program is supposed to do. do_connected_components( experiments, reflections, min_component_size=params.min_component_size, )