Esempio n. 1
0
def test_random_split():
    """Test the reflection_table.random_split() method.

    A random seed is set, so the result is reproducible.
    """
    flex.set_random_seed(0)
    table = flex.reflection_table()
    table["id"] = flex.int(range(0, 10))

    # first test splitting into 2 or 3 tables.
    split_tables = table.random_split(2)
    expected = [[5, 7, 3, 0, 8], [2, 9, 6, 1, 4]]
    for t, e in zip(split_tables, expected):
        assert list(t["id"]) == e

    split_tables = table.random_split(3)
    expected = [[5, 7, 0], [4, 3, 8], [1, 9, 2, 6]]
    for t, e in zip(split_tables, expected):
        assert list(t["id"]) == e

    # test that a float is handled
    split_tables = table.random_split(1.0)
    assert split_tables[0] is table

    # values below 1 should just return the table
    split_tables = table.random_split(0)
    assert split_tables[0] is table

    # if n > len(table), the table should split into n=len(table) tables with
    # one entry.
    split_tables = table.random_split(20)
    assert len(split_tables) == 10
    for t in split_tables:
        assert len(t) == 1
Esempio n. 2
0
def test_unique_hkl():
    '''tests the uniqueness of hkl values associated with each experiment for
  100 simulated randomly oriented thermolysin diffraction images prior to indexing.'''
    flex.set_random_seed(42)  # the meaning of life
    known_symmetry = crystal.symmetry("78,78,37,90,90,90", "P43212")

    detector = detector_factory.simple('SENSOR_UNKNOWN', 125, (97.075, 97.075),
                                       '+x', '-y', (0.11, 0.11), (1765, 1765))
    wavelength1 = 12398 / 7400  #wavelength for 2 color experiment in Angstroms
    wavelength2 = 12398 / 7500  #wavelength for 2 color experiment in Angstroms

    beam1 = beam_factory.simple_directional((0, 0, 1), wavelength1)
    beam2 = beam_factory.simple_directional((0, 0, 1), wavelength2)
    beams = [beam1, beam2]

    for i in range(100):
        data = two_color_still_sim.two_color_still_sim(known_symmetry,
                                                       rot=None)
        hkl_list = data.hkl_list
        A = data.A

        sim_data = data.ewald_proximity_test(beams, hkl_list, A, detector)
        refl = sim_data.get('reflection_table')
        refl1 = refl.select(refl['set_id'] == 0)
        refl2 = refl.select(refl['set_id'] == 1)

        # unit test to make sure all miller indices are unique for each experiment id
        assert len(refl1['miller_index']) == len(set(refl1['miller_index']))
        assert len(refl2['miller_index']) == len(set(refl2['miller_index']))
    print "ok"
Esempio n. 3
0
    def _crystal_factory(space_group_symbol):
        # set random seeds so tests more reliable
        seed = 54321
        random.seed(seed)
        flex.set_random_seed(seed)

        space_group_info = sgtbx.space_group_info(symbol=space_group_symbol)
        space_group = space_group_info.group()
        unit_cell = space_group_info.any_compatible_unit_cell(
            volume=random.uniform(1e4, 1e6))

        crystal_symmetry = crystal.symmetry(unit_cell=unit_cell,
                                            space_group=space_group)
        crystal_symmetry.show_summary()

        # the reciprocal matrix
        B = matrix.sqr(unit_cell.fractionalization_matrix()).transpose()
        U = random_rotation()
        A = U * B

        direct_matrix = A.inverse()
        return Crystal(
            direct_matrix[0:3],
            direct_matrix[3:6],
            direct_matrix[6:9],
            space_group=space_group,
        )
Esempio n. 4
0
def run(args=None):
    usage = "dials.cosym [options] models.expt observations.refl"

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_reflections=True,
        read_experiments=True,
        check_format=False,
        epilog=help_message,
    )

    params, options, args = parser.parse_args(args=args,
                                              show_diff_phil=False,
                                              return_unhandled=True)

    # Configure the logging
    log.config(verbosity=options.verbose, logfile=params.output.log)

    logger.info(dials_version())

    # Log the diff phil
    diff_phil = parser.diff_phil.as_str()
    if diff_phil != "":
        logger.info("The following parameters have been modified:\n")
        logger.info(diff_phil)

    if params.seed is not None:
        flex.set_random_seed(params.seed)
        np.random.seed(params.seed)
        random.seed(params.seed)

    if not params.input.experiments or not params.input.reflections:
        parser.print_help()
        sys.exit()

    reflections, experiments = reflections_and_experiments_from_files(
        params.input.reflections, params.input.experiments)

    reflections = parse_multiple_datasets(reflections)
    if len(experiments) != len(reflections):
        raise Sorry(
            "Mismatched number of experiments and reflection tables found: %s & %s."
            % (len(experiments), len(reflections)))
    try:
        experiments, reflections = assign_unique_identifiers(
            experiments, reflections)
        cosym_instance = cosym(experiments=experiments,
                               reflections=reflections,
                               params=params)
    except ValueError as e:
        raise Sorry(e)

    if params.output.html or params.output.json:
        register_default_cosym_observers(cosym_instance)
    cosym_instance.run()
    cosym_instance.export()
Esempio n. 5
0
    def set_seed(self):

        if self._params.random_seed is not None:
            random.seed(self._params.random_seed)
            # set the flex random seed too
            from dials.array_family import flex
            flex.set_random_seed(self._params.random_seed)
            if self._verbose:
                msg = "Random seed set to %d while building models"
                print(msg % self._params.random_seed)
def run(args):
    usage = "dials.symmetry [options] models.expt observations.refl"

    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_reflections=True,
        read_experiments=True,
        check_format=False,
        epilog=help_message,
    )

    params, _, args = parser.parse_args(args=args,
                                        show_diff_phil=False,
                                        return_unhandled=True)

    # Configure the logging
    log.config(params.verbosity,
               info=params.output.log,
               debug=params.output.debug_log)

    from dials.util.version import dials_version

    logger.info(dials_version())

    # Log the diff phil
    diff_phil = parser.diff_phil.as_str()
    if diff_phil != "":
        logger.info("The following parameters have been modified:\n")
        logger.info(diff_phil)

    if params.seed is not None:
        import random

        flex.set_random_seed(params.seed)
        random.seed(params.seed)

    if not params.input.experiments or not params.input.reflections:
        parser.print_help()
        sys.exit()

    experiments = flatten_experiments(params.input.experiments)
    reflections = flatten_reflections(params.input.reflections)

    reflections = parse_multiple_datasets(reflections)
    if len(experiments) != len(reflections):
        raise Sorry(
            "Mismatched number of experiments and reflection tables found: %s & %s."
            % (len(experiments), len(reflections)))
    try:
        experiments, reflections = assign_unique_identifiers(
            experiments, reflections)
        symmetry(experiments, reflections, params=params)
    except ValueError as e:
        raise Sorry(e)
Esempio n. 7
0
  def set_seed(self):

    if not self._params.random_seed:
      self._params.random_seed = random.randint(0, sys.maxint)
    random.seed(self._params.random_seed)
    # set the flex random seed too
    from dials.array_family import flex
    flex.set_random_seed(self._params.random_seed)
    if self._verbose:
      msg = "Random seed set to %d while building models"
      print msg % self._params.random_seed
Esempio n. 8
0
def run(args=None):
    """Run symmetry analysis from the command-line."""
    usage = "dials.symmetry [options] models.expt observations.refl"

    parser = ArgumentParser(
        usage=usage,
        phil=phil_scope,
        read_reflections=True,
        read_experiments=True,
        check_format=False,
        epilog=help_message,
    )

    params, options, args = parser.parse_args(args=args,
                                              show_diff_phil=False,
                                              return_unhandled=True)

    # Configure the logging
    log.config(verbosity=options.verbose, logfile=params.output.log)

    logger.info(dials_version())

    # Log the diff phil
    diff_phil = parser.diff_phil.as_str()
    if diff_phil != "":
        logger.info("The following parameters have been modified:\n")
        logger.info(diff_phil)

    if params.seed is not None:
        flex.set_random_seed(params.seed)
        random.seed(params.seed)

    if not params.input.experiments or not params.input.reflections:
        parser.print_help()
        sys.exit()

    reflections, experiments = reflections_and_experiments_from_files(
        params.input.reflections, params.input.experiments)

    reflections = parse_multiple_datasets(reflections)

    if len(experiments) != len(reflections):
        sys.exit(
            "Mismatched number of experiments and reflection tables found: %s & %s."
            % (len(experiments), len(reflections)))
    try:
        experiments, reflections = assign_unique_identifiers(
            experiments, reflections)
        symmetry(experiments, reflections, params=params)
    except ValueError as e:
        sys.exit(e)
Esempio n. 9
0
def sample_predictions(experiments, predicted, params):
    """
    Select a random sample of the predicted reflections to integrate.

    Args:
        experiments: The experiment list
        predicted: A reflection table of predicted reflections
        params: The integration phil parameters

    Returns:
        A subset of the original predicted table.
    """

    if params.sampling.random_seed:
        flex.set_random_seed(params.sampling.random_seed)

    nref_per_degree = params.sampling.reflections_per_degree
    min_sample_size = params.sampling.minimum_sample_size
    max_sample_size = params.sampling.maximum_sample_size

    # this code is very similar to David's code in algorithms/refinement/reflection_manager.py!

    working_isel = flex.size_t()
    for iexp, exp in enumerate(experiments):

        sel = predicted["id"] == iexp
        isel = sel.iselection()
        nrefs = sample_size = len(isel)

        # set sample size according to nref_per_degree (per experiment)
        if exp.scan and nref_per_degree:
            sequence_range_rad = exp.scan.get_oscillation_range(deg=False)
            width = math.degrees(abs(sequence_range_rad[1] - sequence_range_rad[0]))
            sample_size = int(nref_per_degree * width)
        else:
            sequence_range_rad = None

        # adjust sample size if below the chosen limit
        sample_size = max(sample_size, min_sample_size)

        # set maximum sample size if requested
        if max_sample_size:
            sample_size = min(sample_size, max_sample_size)

        # determine subset and collect indices
        if sample_size < nrefs:
            isel = isel.select(flex.random_selection(nrefs, sample_size))
        working_isel.extend(isel)

    # create subset
    return predicted.select(working_isel)
Esempio n. 10
0
def test_centroid_outlier(dials_regression, method, colnames, expected_nout):

    flex.set_random_seed(42)
    data_dir = os.path.join(dials_regression, "refinement_test_data",
                            "centroid_outlier")
    residuals = flex.reflection_table.from_file(
        os.path.join(data_dir, "residuals.refl"))
    params = phil_scope.extract()
    params.outlier.algorithm = method
    params.outlier.sauter_poon.px_sz = (0.1, 0.1)  # must be set for SauterPoon
    outlier_detector = CentroidOutlierFactory.from_parameters_and_colnames(
        params, colnames)
    outlier_detector(residuals)
    outliers = residuals.get_flags(residuals.flags.centroid_outlier)

    assert outliers.count(True) == expected_nout
Esempio n. 11
0
def get_basis():
    '''gets the input basis vectors of 100 simulated diffraction images'''

    flex.set_random_seed(42)
    known_symmetry = crystal.symmetry("78,78,37,90,90,90", "P43212")

    detector = detector_factory.simple('SENSOR_UNKNOWN', 125, (97.075, 97.075),
                                       '+x', '-y', (0.11, 0.11), (1765, 1765))
    wavelength1 = 12398 / 7400  #wavelength for 2 color experiment in Angstroms
    wavelength2 = 12398 / 7500  #wavelength for 2 color experiment in Angstroms

    beam1 = beam_factory.simple_directional((0, 0, 1), wavelength1)
    beam2 = beam_factory.simple_directional((0, 0, 1), wavelength2)

    a_basis = []
    b_basis = []
    c_basis = []

    unique_vectors = []

    # refiner resets random number seed so in order to get the same 100 images
    #generated each time the random seed is set
    # the implementation is as follows
    sims = [merge_close_spots.merge_close_spots() for i in range(2)]

    for data in sims:
        A = data.A
        A_inv = A.inverse()
        a = col(A_inv[:3])
        b = col(A_inv[3:6])
        c = col(A_inv[6:])
        a_basis.append(a)
        b_basis.append(b)
        c_basis.append(c)

        res = data.two_color_sim
        info = data.spot_proximity(res)
        refl = info[0]
        candidate_basis_vectors = index(refl, detector, known_symmetry,
                                        [beam1, beam2])
        unique_vectors.append(candidate_basis_vectors)
    return a_basis, b_basis, c_basis, unique_vectors
Esempio n. 12
0
    def run(self):

        self.params, options = self.parser.parse_args()
        if len(self.params.input.experiments) == 0:
            self.parser.print_help()
            raise Sorry("No experiments found in the input")
        if len(self.params.input.reflections) == 0:
            self.parser.print_help()
            raise Sorry("No reflection data found in the input")

        import random

        random.seed(self.params.random_seed)
        flex.set_random_seed(self.params.random_seed)

        from dials.util.options import flatten_reflections

        self.original_reflections = flatten_reflections(
            self.params.input.reflections)[0]

        print("Number of reflections loaded: {0}".format(
            len(self.original_reflections)))

        # make errors to add to observed centroids
        self.set_random_error()

        obs_sets = []
        for wrapper in self.params.input.experiments:
            obs_sets.append(
                self.create_indexed(experiments=wrapper.data,
                                    filename=wrapper.filename))

        # Keep only predictions that are possible for all experiments
        obs_sets = self.select_intersection(obs_sets)

        # Write out reflections
        import os

        for refs, wrapper in zip(obs_sets, self.params.input.experiments):
            outname = os.path.splitext(wrapper.filename)[0] + ".pickle"
            print("Saving reflections to {0}".format(outname))
            refs.as_pickle(outname)
Esempio n. 13
0
def test_close_spot_res():
    '''from 100 simulated diffraction images the resolution of spots < 2 pixels
  apart is calculated and asserted to be < 14 angstroms'''
    flex.set_random_seed(42)

    #unit test average close spots should be in the low resolution region >=15 angstroms
    for j in range(100):
        data = merge_close_spots.merge_close_spots()
        res = data.two_color_sim
        info = data.spot_proximity(res)
        detector = data.detector
        beams = data.beams
        spots_a = res['wavelength_1_spots']
        spots_b = res['wavelength_2_spots']
        mapping = info[1]
        resolution, distances = data.spot_resolution(detector, beams, spots_a,
                                                     spots_b, mapping)
        for i in range(len(distances)):
            if resolution[i] >= 14:
                test = distances[i]
                assert distances[i] <= 2, test
    print "ok"
def run(args):

    from libtbx.phil import command_line

    from dials.util.command_line import Importer
    from dials.array_family import flex

    print(args)
    importer = Importer(args, check_format=False)
    assert len(importer.datablocks) == 1
    sweeps = importer.datablocks[0].extract_imagesets()
    assert len(sweeps) == 1
    sweep = sweeps[0]

    cmd_line = command_line.argument_interpreter(master_params=master_phil_scope)
    working_phil = cmd_line.process_and_fetch(args=importer.unhandled_arguments)
    working_phil.show()

    params = working_phil.extract()
    assert params.unit_cell is not None
    assert params.space_group is not None
    unit_cell = params.unit_cell
    space_group = params.space_group.group()

    import random
    from dxtbx.model.crystal import crystal_model
    from cctbx import crystal, miller
    from scitbx import matrix

    flex.set_random_seed(params.random_seed)
    random.seed(params.random_seed)

    crystal_symmetry = crystal.symmetry(unit_cell=unit_cell, space_group=space_group)

    # the reciprocal matrix
    B = matrix.sqr(unit_cell.fractionalization_matrix()).transpose()

    n_predicted = flex.double()

    def predict_once(args):
        from dxtbx.model.experiment.experiment_list import Experiment

        U = args[0]
        A = U * B
        direct_matrix = A.inverse()
        cryst_model = crystal_model(
            direct_matrix[0:3],
            direct_matrix[3:6],
            direct_matrix[6:9],
            space_group=space_group,
        )
        experiment = Experiment(
            imageset=sweep,
            beam=sweep.get_beam(),
            detector=sweep.get_detector(),
            goniometer=sweep.get_goniometer(),
            scan=sweep.get_scan(),
            crystal=cryst_model,
        )
        predicted_reflections = flex.reflection_table.from_predictions(experiment)
        miller_indices = predicted_reflections["miller_index"]
        miller_set = miller.set(crystal_symmetry, miller_indices, anomalous_flag=True)
        if params.d_min is not None:
            resolution_sel = miller_set.d_spacings().data() > params.d_min
            predicted_reflections = predicted_reflections.select(resolution_sel)
        return len(predicted_reflections)

    from libtbx import easy_mp

    args = [(random_rotation(),) for i in range(params.n_samples)]
    results = easy_mp.parallel_map(
        func=predict_once,
        iterable=args,
        processes=params.nproc,
        preserve_order=True,
        preserve_exception_message=True,
    )
    n_predicted = flex.double(results)

    print("Basic statistics:")
    from scitbx.math import basic_statistics

    stats = basic_statistics(n_predicted)
    stats.show()

    print("Histogram:")
    hist = flex.histogram(n_predicted, n_slots=20)
    hist.show()

    print("Raw spot counts:")
    print(list(n_predicted))

    if params.plot:
        from matplotlib import pyplot
        from matplotlib.backends.backend_pdf import PdfPages

        pyplot.rc("font", family="serif")
        pyplot.rc("font", serif="Times New Roman")

        red, blue = "#B2182B", "#2166AC"
        fig = pyplot.figure()
        ax = fig.add_subplot(1, 1, 1)
        ax.bar(
            hist.slot_centers(),
            hist.slots(),
            width=0.75 * hist.slot_width(),
            color=blue,
            edgecolor=blue,
        )
        ax.set_xlabel("Spot count")
        ax.set_ylabel("Frequency")
        pdf = PdfPages("predicted_count_histogram.pdf")
        pdf.savefig(fig)
        pdf.close()
Esempio n. 15
0
def run():
    # The script usage
    usage = ("usage: xia2.multi_crystal_analysis [options] [param.phil] "
             "models.expt observations.refl")

    # Create the parser
    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_reflections=True,
        read_experiments=True,
        check_format=False,
        epilog=help_message,
    )

    # Parse the command line
    params, options = parser.parse_args(show_diff_phil=False)

    # Configure the logging

    for name in ("xia2", "dials"):
        log.config(verbosity=options.verbose,
                   logfile=params.output.log,
                   name=name)
    logger.info(dials_version())

    # Log the diff phil
    diff_phil = parser.diff_phil.as_str()
    if diff_phil != "":
        logger.info("The following parameters have been modified:\n")
        logger.info(diff_phil)

    # Try to load the models and data
    if len(params.input.experiments) == 0:
        logger.info("No Experiments found in the input")
        parser.print_help()
        return
    if len(params.input.reflections) == 0:
        logger.info("No reflection data found in the input")
        parser.print_help()
        return
    try:
        assert len(params.input.reflections) == len(params.input.experiments)
    except AssertionError:
        raise Sorry("The number of input reflections files does not match the "
                    "number of input experiments")

    if params.seed is not None:
        flex.set_random_seed(params.seed)
        random.seed(params.seed)

    experiments = flatten_experiments(params.input.experiments)
    reflections = flatten_reflections(params.input.reflections)
    reflections = parse_multiple_datasets(reflections)

    joint_table = flex.reflection_table()
    for i in range(len(reflections)):
        joint_table.extend(reflections[i])
    reflections = joint_table

    MultiCrystalReport(params,
                       experiments=experiments,
                       reflections=reflections).report()
Esempio n. 16
0
def test_assign_indices(dials_regression, space_group_symbol):
    experiments_json = os.path.join(dials_regression, "indexing_test_data",
                                    "i04_weak_data", "datablock_orig.json")

    experiments = load.experiment_list(experiments_json, check_format=False)
    sweep = experiments.imagesets()[0]

    sweep = sweep[:20]

    # set random seeds so tests more reliable
    seed = 54321
    random.seed(seed)
    flex.set_random_seed(seed)

    space_group_info = sgtbx.space_group_info(symbol=space_group_symbol)
    space_group = space_group_info.group()
    unit_cell = space_group_info.any_compatible_unit_cell(
        volume=random.uniform(1e4, 1e6))

    crystal_symmetry = crystal.symmetry(unit_cell=unit_cell,
                                        space_group=space_group)
    crystal_symmetry.show_summary()

    # the reciprocal matrix
    B = matrix.sqr(unit_cell.fractionalization_matrix()).transpose()
    U = random_rotation()
    A = U * B

    direct_matrix = A.inverse()
    cryst_model = Crystal(
        direct_matrix[0:3],
        direct_matrix[3:6],
        direct_matrix[6:9],
        space_group=space_group,
    )
    experiment = Experiment(
        imageset=sweep,
        beam=sweep.get_beam(),
        detector=sweep.get_detector(),
        goniometer=sweep.get_goniometer(),
        scan=sweep.get_scan(),
        crystal=cryst_model,
    )
    predicted_reflections = flex.reflection_table.from_predictions(experiment)
    use_fraction = 0.3
    use_sel = flex.random_selection(
        len(predicted_reflections),
        int(use_fraction * len(predicted_reflections)))
    predicted_reflections = predicted_reflections.select(use_sel)
    miller_indices = predicted_reflections["miller_index"]
    predicted_reflections["xyzobs.mm.value"] = predicted_reflections[
        "xyzcal.mm"]
    predicted_reflections["id"] = flex.int(len(predicted_reflections), 0)
    predicted_reflections.map_centroids_to_reciprocal_space(
        sweep.get_detector(), sweep.get_beam(), sweep.get_goniometer())

    # check that local and global indexing worked equally well in absence of errors
    result = CompareGlobalLocal(experiment, predicted_reflections,
                                miller_indices)
    assert result.misindexed_local == 0
    assert result.misindexed_global == 0

    a, b, c = map(matrix.col, cryst_model.get_real_space_vectors())
    relative_error = 0.02
    a *= 1 + relative_error
    b *= 1 + relative_error
    c *= 1 + relative_error

    cryst_model2 = Crystal(a, b, c, space_group=space_group)
    experiment.crystal = cryst_model2

    result = CompareGlobalLocal(experiment, predicted_reflections,
                                miller_indices)

    # check that the local indexing did a better job given the errors in the basis vectors
    # assert result.misindexed_local < result.misindexed_global
    assert result.misindexed_local == 0
    assert result.correct_local > result.correct_global
    # usually the number misindexed is much smaller than this
    assert result.misindexed_local < (0.001 * len(result.reflections_local))

    # the reciprocal matrix
    A = matrix.sqr(cryst_model.get_A())
    A = random_rotation(angle_max=0.5) * A

    direct_matrix = A.inverse()
    cryst_model2 = Crystal(
        direct_matrix[0:3],
        direct_matrix[3:6],
        direct_matrix[6:9],
        space_group=space_group,
    )
    experiment.crystal = cryst_model2

    result = CompareGlobalLocal(experiment, predicted_reflections,
                                miller_indices)

    # check that the local indexing did a better job given the errors in the basis vectors
    assert result.misindexed_local <= result.misindexed_global, (
        result.misindexed_local,
        result.misindexed_global,
    )
    assert result.misindexed_local < 0.01 * result.correct_local
    assert result.correct_local >= result.correct_global
    # usually the number misindexed is much smaller than this
    assert result.misindexed_local < (0.001 * len(result.reflections_local))
Esempio n. 17
0
def run():
    random_seed = 35
    flex.set_random_seed(random_seed)
    random.seed(random_seed)
    data = merge_close_spots.merge_close_spots()
    # unit cell and space group for lysozyme
    known_symmetry = crystal.symmetry("78,78,37,90,90,90", "P43212")

    sim = data.two_color_sim
    merged_spot_info = data.spot_proximity(sim)
    merged_refl = merged_spot_info[0]

    detector = data.detector
    beams = data.beams

    # to make sure input crystal and indexed crystal model are the same
    # orientation before using the refiner
    A = sim['input_orientation']
    A_inv = A.inverse()
    a = A_inv[:3]
    b = A_inv[3:6]
    c = A_inv[6:]
    crystinp = Crystal(a, b, c, space_group=known_symmetry.space_group())
    result = index(merged_refl, detector, known_symmetry, beams)
    print("RESULTS ARE IN")
    cm = result.refined_experiments.crystals()[0]
    R, best_axis, best_angle, change_of_basis = difference_rotation_matrix_axis_angle(
        crystal_a=cm, crystal_b=crystinp)
    euler_angles = euler.xyz_angles(R)

    print "input crystal: %s" % crystinp
    print "Indexed crystal: %s" % cm
    print "rotation of: %s" % R
    print "euler angles:", euler_angles
    print "change of basis:", change_of_basis.as_hkl()

    # cmd_line = command_line.argument_interpreter(master_params=master_phil_scope)
    # working_phil = cmd_line.process_and_fetch(args=[])
    params = master_phil_scope.extract()
    params.refinement.parameterisation.beam.fix = "all"
    params.refinement.parameterisation.detector.fix = "all"
    params.indexing.known_symmetry.space_group = known_symmetry.space_group_info(
    )
    params.refinement.verbosity = 3
    params.indexing.refinement_protocol.d_min_start = 3
    params.indexing.refinement_protocol.n_macro_cycles = 1
    params.indexing.known_symmetry.unit_cell = known_symmetry.unit_cell()
    params.indexing.multiple_lattice_search.max_lattices = 1
    params.indexing.debug = True
    params.indexing.known_symmetry.absolute_angle_tolerance = 5.0
    params.indexing.known_symmetry.relative_length_tolerance = 0.3

    expts = copy.deepcopy(result.refined_experiments)
    expts.crystals()[0].change_basis(change_of_basis)
    print(expts.crystals()[0])

    refined = refine(params, result.reflections, expts, verbosity=1)
    print(refined[0].get_experiments().crystals()[0])

    # from annlib_ext import AnnAdaptor
    # ann = AnnAdaptor(merged_refl['xyzobs.px.value'].as_double(), dim=3, k=1)
    # ann.query(result.reflections['xyzobs.px.value'].as_double()+1e-6)
    indices_sim = change_of_basis.apply(merged_refl['set_miller_index'])
    id_sim = merged_refl['set_id']
    # only get those that refined:
    idx = align_merged_and_refined_refl(merged_refl,
                                        result.refined_reflections)
    indices_sim = flex.miller_index([indices_sim[i] for i in idx])
    id_sim = flex.int([id_sim[i] for i in idx])
    indices_result = result.refined_reflections['miller_index']
    id_result = result.refined_reflections['id']

    correct_ind = (indices_sim == indices_result)
    wrong_wavelength = (id_sim != id_result) & (id_sim != 2)
    wrong_index = (indices_sim != indices_result)
    correct_wavelength = (id_sim == id_result) | (id_sim == 2)
    correct = correct_ind & correct_wavelength
    print "Correct index and wavelength: %i/%i" % (correct.count(True),
                                                   len(correct))
    print "Correct index but wrong wavelength: %i/%i" % (
        wrong_wavelength.count(True), len(wrong_wavelength))
    print "Wrong index but correct wavelength: %i/%i" % (
        wrong_index.count(True), len(correct_wavelength))
Esempio n. 18
0
def test_indexed_hkl():
    '''tests the uniqueness of hkl values associated with each experiment for
  100 simulated randomly oriented thermolysin diffraction images indexed using
  two color indexer'''
    flex.set_random_seed(42)
    known_symmetry = crystal.symmetry("78,78,37,90,90,90", "P43212")

    detector = detector_factory.simple('SENSOR_UNKNOWN', 125, (97.075, 97.075),
                                       '+x', '-y', (0.11, 0.11), (1765, 1765))
    wavelength1 = 12398 / 7400  #wavelength for 2 color experiment in Angstroms
    wavelength2 = 12398 / 7500  #wavelength for 2 color experiment in Angstroms

    beam1 = beam_factory.simple_directional((0, 0, 1), wavelength1)
    beam2 = beam_factory.simple_directional((0, 0, 1), wavelength2)

    a_basis = []
    b_basis = []
    c_basis = []

    # refiner resets random number seed so in order to get the same 100 images
    #generated each time the random seed is set
    # the implementation is as follows
    # gets simulated images
    sims = [merge_close_spots.merge_close_spots() for i in range(2)]

    for data in sims:
        A = data.A
        A_inv = A.inverse()

        a = col(A_inv[:3])
        b = col(A_inv[3:6])
        c = col(A_inv[6:])
        crystinp = Crystal(a, b, c, space_group=known_symmetry.space_group())
        a_basis.append(a)
        b_basis.append(b)
        c_basis.append(c)

        res = data.two_color_sim
        info = data.spot_proximity(res)

        refl = info[0]
        result = index(refl, detector, known_symmetry, [beam1, beam2])
        cm = result.refined_experiments.crystals()[0]
        R, best_axis, best_angle, change_of_basis = difference_rotation_matrix_axis_angle(
            crystal_a=cm, crystal_b=crystinp)

        # cmd_line = command_line.argument_interpreter(master_params=master_phil_scope)
        # working_phil = cmd_line.process_and_fetch(args=[])
        params = master_phil_scope.extract()
        params.refinement.parameterisation.beam.fix = "all"
        params.refinement.parameterisation.detector.fix = "all"
        params.indexing.known_symmetry.space_group = known_symmetry.space_group_info(
        )
        params.refinement.verbosity = 3
        params.indexing.refinement_protocol.d_min_start = 3
        params.indexing.refinement_protocol.n_macro_cycles = 1
        params.indexing.known_symmetry.unit_cell = known_symmetry.unit_cell()
        params.indexing.multiple_lattice_search.max_lattices = 1
        params.indexing.debug = True
        params.indexing.known_symmetry.absolute_angle_tolerance = 5.0
        params.indexing.known_symmetry.relative_length_tolerance = 0.3
        params.indexing.stills.rmsd_min_px = 3.5

        expts = copy.deepcopy(result.refined_experiments)
        expts.crystals()[0].change_basis(change_of_basis)

        reflections_exp0 = result.refined_reflections.select(
            result.refined_reflections['id'] == 0)
        reflections_exp1 = result.refined_reflections.select(
            result.refined_reflections['id'] == 1)

        assert len(reflections_exp0['miller_index']) == len(
            set(reflections_exp0['miller_index']))
        assert len(reflections_exp1['miller_index']) == len(
            set(reflections_exp1['miller_index']))
    print "OK"
Esempio n. 19
0
def run(args):
    usage = ("xia2.multiplex [options] [param.phil] "
             "models1.expt models2.expt observations1.refl "
             "observations2.refl...")

    # Create the parser
    parser = OptionParser(
        usage=usage,
        phil=phil_scope,
        read_reflections=True,
        read_experiments=True,
        check_format=False,
        epilog=help_message,
    )

    # Parse the command line
    params, options = parser.parse_args(args=args, show_diff_phil=False)

    # Configure the logging
    xia2.Handlers.Streams.setup_logging(logfile=params.output.log,
                                        verbose=options.verbose)

    logger.info(dials_version())

    # Log the diff phil
    diff_phil = parser.diff_phil.as_str()
    if diff_phil != "":
        logger.info("The following parameters have been modified:\n")
        logger.info(diff_phil)

    # Try to load the models and data
    if len(params.input.experiments) == 0:
        logger.info("No Experiments found in the input")
        parser.print_help()
        return
    if len(params.input.reflections) == 0:
        logger.info("No reflection data found in the input")
        parser.print_help()
        return
    try:
        assert len(params.input.reflections) == len(params.input.experiments)
    except AssertionError:
        raise sys.exit(
            "The number of input reflections files does not match the "
            "number of input experiments")

    if params.seed is not None:
        flex.set_random_seed(params.seed)
        random.seed(params.seed)

    experiments = flatten_experiments(params.input.experiments)
    reflections = flatten_reflections(params.input.reflections)
    if len(experiments) < 2:
        sys.exit("xia2.multiplex requires a minimum of two experiments")
    reflections = parse_multiple_datasets(reflections)
    experiments, reflections = assign_unique_identifiers(
        experiments, reflections)

    reflections, experiments = exclude_image_ranges_for_scaling(
        reflections, experiments, params.exclude_images)

    reflections_all = flex.reflection_table()
    assert len(reflections) == 1 or len(reflections) == len(experiments)
    for i, (expt, refl) in enumerate(zip(experiments, reflections)):
        reflections_all.extend(refl)
    reflections_all.assert_experiment_identifiers_are_consistent(experiments)

    if params.identifiers is not None:
        identifiers = []
        for identifier in params.identifiers:
            identifiers.extend(identifier.split(","))
        params.identifiers = identifiers

    try:
        ScaleAndMerge.MultiCrystalScale(experiments, reflections_all, params)
    except ValueError as e:
        sys.exit(str(e))
Esempio n. 20
0
from __future__ import absolute_import, division

import os
import libtbx.load_env
from cctbx import crystal, miller, sgtbx
from scitbx import matrix
from dxtbx.serialize import load
from dxtbx.model.experiment_list import Experiment, ExperimentList
from dxtbx.model import Crystal
from dials.array_family import flex

# set random seeds so tests more reliable
seed = 54321
import random
random.seed(seed)
flex.set_random_seed(seed)

have_dials_regression = libtbx.env.has_module("dials_regression")
if have_dials_regression:
    dials_regression = libtbx.env.find_in_repositories(
        relative_path="dials_regression", test=os.path.isdir)
else:
    print 'SKIP: dials_regression not configured'
    exit(0)


def random_rotation(angle_min=0, angle_max=360):
    import random
    from scitbx.math import euler_angles_as_matrix
    return euler_angles_as_matrix(
        [random.uniform(angle_min, angle_max) for i in xrange(3)])
Esempio n. 21
0
def run():

  # The script usage
  usage  = "usage: xia2.multi_crystal_scale_and_merge [options] [param.phil] " \
           "experiments1.json experiments2.json reflections1.pickle " \
           "reflections2.pickle..."

  # Create the parser
  parser = OptionParser(
    usage=usage,
    phil=phil_scope,
    read_reflections=True,
    read_experiments=True,
    check_format=False,
    epilog=help_message)

  # Parse the command line
  params, options = parser.parse_args(show_diff_phil=True)

  # Configure the logging

  for name in ('xia2', 'dials'):
    log.config(
      info=params.output.log,
      debug=params.output.debug_log,
      name=name)
  from dials.util.version import dials_version
  logger.info(dials_version())

  # Try to load the models and data
  if len(params.input.experiments) == 0:
    logger.info("No Experiments found in the input")
    parser.print_help()
    return
  if len(params.input.reflections) == 0:
    logger.info("No reflection data found in the input")
    parser.print_help()
    return
  try:
    assert len(params.input.reflections) == len(params.input.experiments)
  except AssertionError:
    raise Sorry("The number of input reflections files does not match the "
      "number of input experiments")

  if params.seed is not None:
    import random
    flex.set_random_seed(params.seed)
    random.seed(params.seed)

  expt_filenames = OrderedDict((e.filename, e.data) for e in params.input.experiments)
  refl_filenames = OrderedDict((r.filename, r.data) for r in params.input.reflections)

  experiments = flatten_experiments(params.input.experiments)
  reflections = flatten_reflections(params.input.reflections)

  reflections_all = flex.reflection_table()
  assert len(reflections) == 1 or len(reflections) == len(experiments)
  if len(reflections) > 1:
    for i, (expt, refl) in enumerate(zip(experiments, reflections)):
      expt.identifier = '%i' % i
      refl['identifier'] = flex.std_string(refl.size(), expt.identifier)
      refl['id'] = flex.int(refl.size(), i)
      reflections_all.extend(refl)
      reflections_all.experiment_identifiers()[i] = expt.identifier
  else:
    reflections_all = reflections[0]
    assert 'identifier' in reflections_all
    assert len(set(reflections_all['identifier'])) == len(experiments)

  assert reflections_all.are_experiment_identifiers_consistent(experiments)

  if params.identifiers is not None:
    identifiers = []
    for identifier in params.identifiers:
      identifiers.extend(identifier.split(','))
    params.identifiers = identifiers
  scaled = ScaleAndMerge.MultiCrystalScale(experiments, reflections_all, params)
Esempio n. 22
0
def generate_test_data(
    space_group,
    lattice_group=None,
    unit_cell=None,
    unit_cell_volume=1000,
    seed=0,
    d_min=1,
    sigma=0.1,
    sample_size=100,
    map_to_p1=False,
    twin_fractions=None,
    map_to_minimum=True,
):

    import random

    if seed is not None:
        flex.set_random_seed(seed)
        random.seed(seed)

    assert [unit_cell, lattice_group].count(None) > 0

    sgi = space_group.info()

    if unit_cell is not None:
        cs = crystal.symmetry(unit_cell=unit_cell, space_group_info=sgi)
    elif lattice_group is not None:
        subgrps = subgroups(lattice_group).groups_parent_setting()
        assert space_group in subgrps
        cs = lattice_group.any_compatible_crystal_symmetry(
            volume=unit_cell_volume).customized_copy(space_group_info=sgi)
    else:
        cs = sgi.any_compatible_crystal_symmetry(volume=unit_cell_volume)

    if map_to_minimum:
        cs = cs.minimum_cell()
    intensities = generate_intensities(cs, d_min=d_min)
    intensities.show_summary()

    twin_ops = generate_twin_operators(intensities)
    twin_ops = [
        sgtbx.change_of_basis_op(op.operator.as_xyz()) for op in twin_ops
    ]

    if twin_fractions is not None:
        assert len(twin_fractions) == len(twin_ops)
        assert len(
            twin_fractions) == 1, "Only 1 twin component currently supported"
        twin_op = twin_ops[0]
        twin_fraction = twin_fractions[0]
        intensities, intensities_twin = intensities.common_sets(
            intensities.change_basis(twin_op).map_to_asu())
        twinned_miller = intensities.customized_copy(
            data=(1.0 - twin_fraction) * intensities.data() +
            twin_fraction * intensities_twin.data(),
            sigmas=flex.sqrt(
                flex.pow2((1.0 - twin_fraction) * intensities.sigmas()) +
                flex.pow2(twin_fraction * intensities_twin.sigmas())),
        )
        intensities = twinned_miller

    cb_ops = twin_ops
    cb_ops.insert(0, sgtbx.change_of_basis_op())

    reindexing_ops = {}

    datasets = []
    rand_norm = scitbx.random.normal_distribution(mean=0, sigma=sigma)
    g = scitbx.random.variate(rand_norm)
    for i in range(sample_size):
        cb_op = random.choice(cb_ops)
        if cb_op.as_xyz() not in reindexing_ops:
            reindexing_ops[cb_op.as_xyz()] = set()
        reindexing_ops[cb_op.as_xyz()].add(i)
        d = intensities.change_basis(cb_op).customized_copy(
            crystal_symmetry=intensities.crystal_symmetry())

        if map_to_p1:
            cb_op_to_primitive = d.change_of_basis_op_to_primitive_setting()
            d = d.change_basis(cb_op_to_primitive)
            d = d.expand_to_p1()

        d = d.customized_copy(data=d.data() + g(d.size()))
        datasets.append(d)

    return datasets, reindexing_ops
def run(args):

  from libtbx.phil import command_line

  from dials.util.command_line import Importer
  from dials.array_family import flex
  print args
  importer = Importer(args, check_format=False)
  assert len(importer.datablocks) == 1
  sweeps = importer.datablocks[0].extract_imagesets()
  assert len(sweeps) == 1
  sweep = sweeps[0]

  cmd_line = command_line.argument_interpreter(master_params=master_phil_scope)
  working_phil = cmd_line.process_and_fetch(args=importer.unhandled_arguments)
  working_phil.show()

  params = working_phil.extract()
  assert params.unit_cell is not None
  assert params.space_group is not None
  unit_cell = params.unit_cell
  space_group = params.space_group.group()

  import random
  from dxtbx.model.crystal import crystal_model
  from cctbx import crystal, miller
  from scitbx import matrix

  flex.set_random_seed(params.random_seed)
  random.seed(params.random_seed)

  crystal_symmetry = crystal.symmetry(unit_cell=unit_cell,
                                      space_group=space_group)

  # the reciprocal matrix
  B = matrix.sqr(unit_cell.fractionalization_matrix()).transpose()

  n_predicted = flex.double()

  def predict_once(args):
    from dxtbx.model.experiment.experiment_list import Experiment
    U = args[0]
    A = U * B
    direct_matrix = A.inverse()
    cryst_model = crystal_model(direct_matrix[0:3],
                                direct_matrix[3:6],
                                direct_matrix[6:9],
                                space_group=space_group)
    experiment = Experiment(imageset=sweep,
                            beam=sweep.get_beam(),
                            detector=sweep.get_detector(),
                            goniometer=sweep.get_goniometer(),
                            scan=sweep.get_scan(),
                            crystal=cryst_model)
    predicted_reflections = flex.reflection_table.from_predictions(
      experiment)
    miller_indices = predicted_reflections['miller_index']
    miller_set = miller.set(
      crystal_symmetry, miller_indices, anomalous_flag=True)
    if params.d_min is not None:
      resolution_sel = miller_set.d_spacings().data() > params.d_min
      predicted_reflections = predicted_reflections.select(resolution_sel)
    return len(predicted_reflections)

  from libtbx import easy_mp
  args = [(random_rotation(),) for i in range(params.n_samples)]
  results = easy_mp.parallel_map(
    func=predict_once,
    iterable=args,
    processes=params.nproc,
    preserve_order=True,
    preserve_exception_message=True)
  n_predicted = flex.double(results)

  print "Basic statistics:"
  from scitbx.math import basic_statistics
  stats = basic_statistics(n_predicted)
  stats.show()

  print "Histogram:"
  hist = flex.histogram(n_predicted, n_slots=20)
  hist.show()

  print "Raw spot counts:"
  print list(n_predicted)

  if params.plot:
    from matplotlib import pyplot
    from matplotlib.backends.backend_pdf import PdfPages

    pyplot.rc('font', family='serif')
    pyplot.rc('font', serif='Times New Roman')

    red, blue = '#B2182B', '#2166AC'
    fig = pyplot.figure()
    ax = fig.add_subplot(1,1,1)
    ax.bar(hist.slot_centers(), hist.slots(), width=0.75*hist.slot_width(),
           color=blue, edgecolor=blue)
    ax.set_xlabel('Spot count')
    ax.set_ylabel('Frequency')
    pdf = PdfPages("predicted_count_histogram.pdf")
    pdf.savefig(fig)
    pdf.close()
Esempio n. 24
0
from numpy.linalg import det, inv, norm

from dxtbx import flumpy
from scitbx import linalg, matrix

from dials.algorithms.profile_model.ellipsoid import mosaicity_from_eigen_decomposition
from dials.algorithms.profile_model.ellipsoid.model import (
    compute_change_of_basis_operation, )
from dials.algorithms.profile_model.ellipsoid.parameterisation import (
    ReflectionModelState, )
from dials.array_family import flex
from dials.util import tabulate

logger = logging.getLogger("dials")

flex.set_random_seed(0)
random.seed(0)


def compute_dSbar(S: np.array, dS: np.array) -> np.array:
    # dS & S are 3x3 arrays. Returns a 2x2 array
    S12 = S[0:2, 2].reshape(2, 1)
    S21 = S[2, 0:2].reshape(1, 2)
    S22 = S[2, 2]

    dS11 = dS[0:2, 0:2]
    dS12 = dS[0:2, 2].reshape(2, 1)
    dS21 = dS[2, 0:2].reshape(1, 2)
    dS22 = dS[2, 2]

    S22_inv = 1 / S22
Esempio n. 25
0
from __future__ import absolute_import, division, print_function

import pytest
import random

from cctbx import sgtbx
from dials.algorithms.symmetry.cosym._generate_test_data import generate_intensities
from dials.array_family import flex
from dxtbx.model.experiment_list import ExperimentList
from dxtbx.model import Crystal, Scan, Beam, Experiment
from dxtbx.serialize import dump, load


flex.set_random_seed(42)
random.seed(42)


@pytest.fixture
def helper(ccp4, run_in_tmpdir):
    """Initialise a DialsScalerHelper, ensure CCP4 is available for test"""
    from xia2.Modules.Scaler.DialsScaler import DialsScalerHelper

    helper = DialsScalerHelper()
    helper.set_pname_xname("AUTOMATIC", "DEFAULT")
    helper.set_working_directory(run_in_tmpdir.strpath)
    return helper


def generated_exp(n=1, space_group="P 2", assign_ids=False, id_=None):
    """Generate an experiment list with two experiments."""
    experiments = ExperimentList()
Esempio n. 26
0
from __future__ import division

import os
import libtbx.load_env
from cctbx import crystal, miller, sgtbx
from scitbx import matrix
from dxtbx.serialize import load
from dxtbx.model.experiment.experiment_list import Experiment, ExperimentList
from dxtbx.model.crystal import crystal_model
from dials.array_family import flex

# set random seeds so tests more reliable
seed = 54321
import random
random.seed(seed)
flex.set_random_seed(seed)

have_dials_regression = libtbx.env.has_module("dials_regression")
if have_dials_regression:
  dials_regression = libtbx.env.find_in_repositories(
    relative_path="dials_regression",
    test=os.path.isdir)
else:
  print 'SKIP: dials_regression not configured'
  exit(0)


def random_rotation(angle_min=0, angle_max=360):
  import random
  from scitbx.math import euler_angles_as_matrix
  return euler_angles_as_matrix(
Esempio n. 27
0
    def from_parameters_reflections_experiments(
        params, reflections, experiments, do_stills=False
    ):
        """Given a set of parameters and models, build a reflection manager

        Params:
            params The input parameters

        Returns:
            The reflection manager instance
        """

        # While a random subset of reflections is used, continue to
        # set random.seed to get consistent behaviour
        if params.random_seed is not None:
            random.seed(params.random_seed)
            flex.set_random_seed(params.random_seed)
            logger.debug("Random seed set to %d", params.random_seed)

        # check whether we deal with stills or scans
        if do_stills:
            refman = StillsReflectionManager
            # check incompatible weighting strategy
            if params.weighting_strategy.override == "statistical":
                raise DialsRefineConfigError(
                    'The "statistical" weighting strategy is not compatible '
                    "with stills refinement"
                )
        else:
            refman = ReflectionManager
            # check incompatible weighting strategy
            if params.weighting_strategy.override in ["stills", "external_deltapsi"]:
                msg = (
                    'The "{0}" weighting strategy is not compatible with '
                    "scan refinement"
                ).format(params.weighting_strategy.override)
                raise DialsRefineConfigError(msg)

        # set automatic outlier rejection options
        if params.outlier.algorithm in ("auto", libtbx.Auto):
            if do_stills:
                params.outlier.algorithm = "sauter_poon"
            else:
                params.outlier.algorithm = "mcd"

        if params.outlier.separate_panels is libtbx.Auto:
            if do_stills:
                params.outlier.separate_panels = False
            else:
                params.outlier.separate_panels = True

        if params.outlier.algorithm == "sauter_poon":
            if params.outlier.sauter_poon.px_sz is libtbx.Auto:
                # get this from the first panel of the first detector
                params.outlier.sauter_poon.px_sz = experiments.detectors()[0][
                    0
                ].get_pixel_size()

        # do outlier rejection?
        if params.outlier.algorithm in ("null", None):
            outlier_detector = None
        else:
            if do_stills:
                colnames = ["x_resid", "y_resid"]
                params.outlier.block_width = None
            else:
                colnames = ["x_resid", "y_resid", "phi_resid"]
            from dials.algorithms.refinement.outlier_detection import (
                CentroidOutlierFactory,
            )

            outlier_detector = CentroidOutlierFactory.from_parameters_and_colnames(
                params, colnames
            )

        # override default weighting strategy?
        weighting_strategy = None
        if params.weighting_strategy.override == "statistical":
            from dials.algorithms.refinement.weighting_strategies import (
                StatisticalWeightingStrategy,
            )

            weighting_strategy = StatisticalWeightingStrategy()
        elif params.weighting_strategy.override == "stills":
            from dials.algorithms.refinement.weighting_strategies import (
                StillsWeightingStrategy,
            )

            weighting_strategy = StillsWeightingStrategy(
                params.weighting_strategy.delpsi_constant
            )
        elif params.weighting_strategy.override == "external_deltapsi":
            from dials.algorithms.refinement.weighting_strategies import (
                ExternalDelPsiWeightingStrategy,
            )

            weighting_strategy = ExternalDelPsiWeightingStrategy()
        elif params.weighting_strategy.override == "constant":
            from dials.algorithms.refinement.weighting_strategies import (
                ConstantWeightingStrategy,
            )

            weighting_strategy = ConstantWeightingStrategy(
                *params.weighting_strategy.constants, stills=do_stills
            )

        # Check for deprecated parameter
        if params.trim_scan_edges is not None:
            warnings.warn(
                "The parameter trim_scan_edges is deprecated and will be removed shortly",
                FutureWarning,
            )
            params.scan_margin = params.trim_scan_edges

        return refman(
            reflections=reflections,
            experiments=experiments,
            nref_per_degree=params.reflections_per_degree,
            max_sample_size=params.maximum_sample_size,
            min_sample_size=params.minimum_sample_size,
            close_to_spindle_cutoff=params.close_to_spindle_cutoff,
            scan_margin=params.scan_margin,
            outlier_detector=outlier_detector,
            weighting_strategy_override=weighting_strategy,
        )
Esempio n. 28
0
def run(args):
    import libtbx
    from libtbx import easy_pickle
    from dials.util import log
    from dials.util.options import OptionParser

    parser = OptionParser(
        #usage=usage,
        phil=phil_scope,
        read_reflections=True,
        read_datablocks=False,
        read_experiments=True,
        check_format=False,
        #epilog=help_message
    )

    params, options, args = parser.parse_args(show_diff_phil=False,
                                              return_unhandled=True)

    # Configure the logging
    log.config(params.verbosity,
               info=params.output.log,
               debug=params.output.debug_log)

    from dials.util.version import dials_version
    logger.info(dials_version())

    # Log the diff phil
    diff_phil = parser.diff_phil.as_str()
    if diff_phil is not '':
        logger.info('The following parameters have been modified:\n')
        logger.info(diff_phil)

    if params.seed is not None:
        import random
        flex.set_random_seed(params.seed)
        random.seed(params.seed)

    if params.save_plot and not params.animate:
        import matplotlib
        # http://matplotlib.org/faq/howto_faq.html#generate-images-without-having-a-window-appear
        matplotlib.use('Agg')  # use a non-interactive backend

    datasets_input = []

    experiments = flatten_experiments(params.input.experiments)
    reflections = flatten_reflections(params.input.reflections)

    if len(experiments) or len(reflections):
        if len(reflections) == 1:
            reflections_input = reflections[0]
            reflections = []
            for i in range(len(experiments)):
                reflections.append(
                    reflections_input.select(reflections_input['id'] == i))

        if len(experiments) > len(reflections):
            flattened_reflections = []
            for refl in reflections:
                for i in range(0, flex.max(refl['id']) + 1):
                    sel = refl['id'] == i
                    flattened_reflections.append(refl.select(sel))
            reflections = flattened_reflections

        assert len(experiments) == len(reflections)

        i_refl = 0
        for i_expt in enumerate(experiments):
            refl = reflections[i_refl]

        for expt, refl in zip(experiments, reflections):
            crystal_symmetry = crystal.symmetry(
                unit_cell=expt.crystal.get_unit_cell(),
                space_group=expt.crystal.get_space_group())
            if 0 and 'intensity.prf.value' in refl:
                sel = refl.get_flags(refl.flags.integrated_prf)
                assert sel.count(True) > 0
                refl = refl.select(sel)
                data = refl['intensity.prf.value']
                variances = refl['intensity.prf.variance']
            else:
                assert 'intensity.sum.value' in refl
                sel = refl.get_flags(refl.flags.integrated_sum)
                assert sel.count(True) > 0
                refl = refl.select(sel)
                data = refl['intensity.sum.value']
                variances = refl['intensity.sum.variance']
            # FIXME probably need to do some filtering of intensities similar to that
            # done in export_mtz
            miller_indices = refl['miller_index']
            assert variances.all_gt(0)
            sigmas = flex.sqrt(variances)

            miller_set = miller.set(crystal_symmetry,
                                    miller_indices,
                                    anomalous_flag=False)
            intensities = miller.array(miller_set, data=data, sigmas=sigmas)
            intensities.set_observation_type_xray_intensity()
            intensities.set_info(
                miller.array_info(source='DIALS', source_type='pickle'))
            datasets_input.append(intensities)

    files = args

    for file_name in files:

        try:
            data = easy_pickle.load(file_name)
            intensities = data['observations'][0]
            intensities.set_info(
                miller.array_info(source=file_name, source_type='pickle'))
            intensities = intensities.customized_copy(
                anomalous_flag=False).set_info(intensities.info())
            batches = None
        except Exception:
            reader = any_reflection_file(file_name)
            assert reader.file_type() == 'ccp4_mtz'

            as_miller_arrays = reader.as_miller_arrays(merge_equivalents=False)
            intensities = [
                ma for ma in as_miller_arrays
                if ma.info().labels == ['I', 'SIGI']
            ][0]
            batches = [
                ma for ma in as_miller_arrays if ma.info().labels == ['BATCH']
            ]
            if len(batches):
                batches = batches[0]
            else:
                batches = None
            mtz_object = reader.file_content()
            intensities = intensities.customized_copy(
                anomalous_flag=False,
                indices=mtz_object.extract_original_index_miller_indices(
                )).set_info(intensities.info())

        intensities.set_observation_type_xray_intensity()
        datasets_input.append(intensities)

    if len(datasets_input) == 0:
        raise Sorry('No valid reflection files provided on command line')

    datasets = []
    for intensities in datasets_input:

        if params.batch is not None:
            assert batches is not None
            bmin, bmax = params.batch
            assert bmax >= bmin
            sel = (batches.data() >= bmin) & (batches.data() <= bmax)
            assert sel.count(True) > 0
            intensities = intensities.select(sel)

        if params.min_i_mean_over_sigma_mean is not None and (
                params.d_min is libtbx.Auto or params.d_min is not None):
            from xia2.Modules import Resolutionizer
            rparams = Resolutionizer.phil_defaults.extract().resolutionizer
            rparams.nbins = 20
            resolutionizer = Resolutionizer.resolutionizer(
                intensities, None, rparams)
            i_mean_over_sigma_mean = 4
            d_min = resolutionizer.resolution_i_mean_over_sigma_mean(
                i_mean_over_sigma_mean)
            if params.d_min is libtbx.Auto:
                intensities = intensities.resolution_filter(
                    d_min=d_min).set_info(intensities.info())
                if params.verbose:
                    logger.info('Selecting reflections with d > %.2f' % d_min)
            elif d_min > params.d_min:
                logger.info('Rejecting dataset %s as d_min too low (%.2f)' %
                            (file_name, d_min))
                continue
            else:
                logger.info('Estimated d_min for %s: %.2f' %
                            (file_name, d_min))
        elif params.d_min not in (None, libtbx.Auto):
            intensities = intensities.resolution_filter(
                d_min=params.d_min).set_info(intensities.info())

        if params.normalisation == 'kernel':
            from mmtbx.scaling import absolute_scaling
            normalisation = absolute_scaling.kernel_normalisation(
                intensities, auto_kernel=True)
            intensities = normalisation.normalised_miller.deep_copy()

        cb_op_to_primitive = intensities.change_of_basis_op_to_primitive_setting(
        )
        intensities = intensities.change_basis(cb_op_to_primitive)
        if params.mode == 'full' or params.space_group is not None:
            if params.space_group is not None:
                space_group_info = params.space_group.primitive_setting()
                if not space_group_info.group().is_compatible_unit_cell(
                        intensities.unit_cell()):
                    logger.info(
                        'Skipping data set - incompatible space group and unit cell: %s, %s'
                        % (space_group_info, intensities.unit_cell()))
                    continue
            else:
                space_group_info = sgtbx.space_group_info('P1')
            intensities = intensities.customized_copy(
                space_group_info=space_group_info)

        datasets.append(intensities)

    crystal_symmetries = [d.crystal_symmetry().niggli_cell() for d in datasets]
    lattice_ids = range(len(datasets))
    from xfel.clustering.cluster import Cluster
    from xfel.clustering.cluster_groups import unit_cell_info
    ucs = Cluster.from_crystal_symmetries(crystal_symmetries,
                                          lattice_ids=lattice_ids)
    threshold = 1000
    if params.save_plot:
        from matplotlib import pyplot as plt
        fig = plt.figure("Andrews-Bernstein distance dendogram",
                         figsize=(12, 8))
        ax = plt.gca()
    else:
        ax = None
    clusters, _ = ucs.ab_cluster(params.unit_cell_clustering.threshold,
                                 log=params.unit_cell_clustering.log,
                                 write_file_lists=False,
                                 schnell=False,
                                 doplot=params.save_plot,
                                 ax=ax)
    if params.save_plot:
        plt.tight_layout()
        plt.savefig('%scluster_unit_cell.png' % params.plot_prefix)
        plt.close(fig)
    logger.info(unit_cell_info(clusters))
    largest_cluster = None
    largest_cluster_lattice_ids = None
    for cluster in clusters:
        cluster_lattice_ids = [m.lattice_id for m in cluster.members]
        if largest_cluster_lattice_ids is None:
            largest_cluster_lattice_ids = cluster_lattice_ids
        elif len(cluster_lattice_ids) > len(largest_cluster_lattice_ids):
            largest_cluster_lattice_ids = cluster_lattice_ids

    dataset_selection = largest_cluster_lattice_ids
    if len(dataset_selection) < len(datasets):
        logger.info('Selecting subset of data for cosym analysis: %s' %
                    str(dataset_selection))
        datasets = [datasets[i] for i in dataset_selection]

    # per-dataset change of basis operator to ensure all consistent
    change_of_basis_ops = []
    for i, dataset in enumerate(datasets):
        metric_subgroups = sgtbx.lattice_symmetry.metric_subgroups(dataset,
                                                                   max_delta=5)
        subgroup = metric_subgroups.result_groups[0]
        cb_op_inp_best = subgroup['cb_op_inp_best']
        datasets[i] = dataset.change_basis(cb_op_inp_best)
        change_of_basis_ops.append(cb_op_inp_best)

    cb_op_ref_min = datasets[0].change_of_basis_op_to_niggli_cell()
    for i, dataset in enumerate(datasets):
        if params.space_group is None:
            datasets[i] = dataset.change_basis(cb_op_ref_min).customized_copy(
                space_group_info=sgtbx.space_group_info('P1'))
        else:
            datasets[i] = dataset.change_basis(cb_op_ref_min)
            datasets[i] = datasets[i].customized_copy(
                crystal_symmetry=crystal.symmetry(
                    unit_cell=datasets[i].unit_cell(),
                    space_group_info=params.space_group.primitive_setting(),
                    assert_is_compatible_unit_cell=False))
        datasets[i] = datasets[i].merge_equivalents().array()
        change_of_basis_ops[i] = cb_op_ref_min * change_of_basis_ops[i]

    result = analyse_datasets(datasets, params)

    space_groups = {}
    reindexing_ops = {}
    for dataset_id in result.reindexing_ops.iterkeys():
        if 0 in result.reindexing_ops[dataset_id]:
            cb_op = result.reindexing_ops[dataset_id][0]
            reindexing_ops.setdefault(cb_op, [])
            reindexing_ops[cb_op].append(dataset_id)
        if dataset_id in result.space_groups:
            space_groups.setdefault(result.space_groups[dataset_id], [])
            space_groups[result.space_groups[dataset_id]].append(dataset_id)

    logger.info('Space groups:')
    for sg, datasets in space_groups.iteritems():
        logger.info(str(sg.info().reference_setting()))
        logger.info(datasets)

    logger.info('Reindexing operators:')
    for cb_op, datasets in reindexing_ops.iteritems():
        logger.info(cb_op)
        logger.info(datasets)

    if (len(experiments) and len(reflections)
            and params.output.reflections is not None
            and params.output.experiments is not None):
        import copy
        from dxtbx.model import ExperimentList
        from dxtbx.serialize import dump
        reindexed_experiments = ExperimentList()
        reindexed_reflections = flex.reflection_table()
        expt_id = 0
        for cb_op, dataset_ids in reindexing_ops.iteritems():
            cb_op = sgtbx.change_of_basis_op(cb_op)
            for dataset_id in dataset_ids:
                expt = experiments[dataset_selection[dataset_id]]
                refl = reflections[dataset_selection[dataset_id]]
                reindexed_expt = copy.deepcopy(expt)
                refl_reindexed = copy.deepcopy(refl)
                cb_op_this = cb_op * change_of_basis_ops[dataset_id]
                reindexed_expt.crystal = reindexed_expt.crystal.change_basis(
                    cb_op_this)
                refl_reindexed['miller_index'] = cb_op_this.apply(
                    refl_reindexed['miller_index'])
                reindexed_experiments.append(reindexed_expt)
                refl_reindexed['id'] = flex.int(refl_reindexed.size(), expt_id)
                reindexed_reflections.extend(refl_reindexed)
                expt_id += 1

        logger.info('Saving reindexed experiments to %s' %
                    params.output.experiments)
        dump.experiment_list(reindexed_experiments, params.output.experiments)
        logger.info('Saving reindexed reflections to %s' %
                    params.output.reflections)
        reindexed_reflections.as_pickle(params.output.reflections)

    elif params.output.suffix is not None:
        for cb_op, dataset_ids in reindexing_ops.iteritems():
            cb_op = sgtbx.change_of_basis_op(cb_op)
            for dataset_id in dataset_ids:
                file_name = files[dataset_selection[dataset_id]]
                basename = os.path.basename(file_name)
                out_name = os.path.splitext(
                    basename)[0] + params.output.suffix + '_' + str(
                        dataset_selection[dataset_id]) + ".mtz"
                reader = any_reflection_file(file_name)
                assert reader.file_type() == 'ccp4_mtz'
                mtz_object = reader.file_content()
                cb_op_this = cb_op * change_of_basis_ops[dataset_id]
                if not cb_op_this.is_identity_op():
                    logger.info('reindexing %s (%s)' %
                                (file_name, cb_op_this.as_xyz()))
                    mtz_object.change_basis_in_place(cb_op_this)
                mtz_object.write(out_name)