Exemple #1
0
def model(tmpdir):
    from dials.algorithms.background.gmodel import StaticBackgroundModel
    from dials.array_family import flex

    ysize = 2527
    xsize = 2463
    data = flex.double(flex.grid(ysize, xsize), 1)
    model = StaticBackgroundModel()
    model.add(data)

    model_file = tmpdir.join("model.pickle")
    with model_file.open("wb") as fh:
        pickle.dump(model, fh, pickle.HIGHEST_PROTOCOL)
    return model_file
Exemple #2
0
    def run(self, args=None):
        """Execute the script."""
        from dials.algorithms.background.modeller import BackgroundModeller
        from dials.array_family import flex
        from dials.util.command_line import heading
        from dials.util.options import flatten_experiments

        # Parse the command line
        params, options = self.parser.parse_args(args, show_diff_phil=False)

        # Configure the logging
        dials.util.log.config(verbosity=options.verbose,
                              logfile=params.output.log)

        if params.integration.mp.nproc != 1 or params.integration.mp.njobs != 1:
            # https://github.com/dials/dials/issues/1083
            logger.warning("Multiprocessing is currently disabled. "
                           "Setting nproc = njobs = 1")
            params.integration.mp.nproc = 1
            params.integration.mp.njobs = 1

        from dials.util.version import dials_version

        logger.info(dials_version())

        # Log the diff phil
        diff_phil = self.parser.diff_phil.as_str()
        if diff_phil != "":
            logger.info("The following parameters have been modified:\n")
            logger.info(diff_phil)

        # Ensure we have a data block
        experiments = flatten_experiments(params.input.experiments)
        if len(experiments) == 0:
            self.parser.print_help()
            return

        # Only handle a single imageset at once
        imagesets = {expr.imageset for expr in experiments}
        if len(imagesets) != 1:
            sys.exit("Can only process a single imageset at a time")

        # Predict the reflections
        logger.info("")
        logger.info("=" * 80)
        logger.info("")
        logger.info(heading("Predicting reflections"))
        logger.info("")
        predicted = flex.reflection_table.from_predictions_multi(
            experiments,
            dmin=params.prediction.d_min,
            dmax=params.prediction.d_max,
            margin=params.prediction.margin,
            force_static=params.prediction.force_static,
        )

        # Create the modeller
        modeller = BackgroundModeller(experiments, predicted, params)
        model = modeller.compute()

        # Save the background model
        logger.info("Saving background model to %s" % params.output.model)
        from dials.algorithms.background.gmodel import StaticBackgroundModel

        static_model = StaticBackgroundModel()
        for m in model:
            static_model.add(m.model)
        with open(params.output.model, "wb") as outfile:
            pickle.dump(static_model,
                        outfile,
                        protocol=pickle.HIGHEST_PROTOCOL)

        # Output some diagnostic images
        image_generator = ImageGenerator(model)
        image_generator.save_mean(params.output.mean_image_prefix)
        image_generator.save_variance(params.output.variance_image_prefix)
        image_generator.save_dispersion(params.output.dispersion_image_prefix)
        image_generator.save_mask(params.output.mask_image_prefix)
        image_generator.save_min(params.output.min_image_prefix)
        image_generator.save_max(params.output.max_image_prefix)
        image_generator.save_model(params.output.model_image_prefix)
Exemple #3
0
  def run(self):
    '''Execute the script.'''
    from dials.util.command_line import heading
    from dials.array_family import flex
    from dials.util.options import flatten_experiments
    from time import time
    from dials.util import log
    from libtbx.utils import Sorry
    from dials.algorithms.background.modeller import BackgroundModeller
    start_time = time()

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=False)

    # Configure the logging
    log.config(
      params.verbosity,
      info=params.output.log,
      debug=params.output.debug_log)

    from dials.util.version import dials_version
    logger.info(dials_version())

    # Log the diff phil
    diff_phil = self.parser.diff_phil.as_str()
    if diff_phil is not '':
      logger.info('The following parameters have been modified:\n')
      logger.info(diff_phil)

    # Ensure we have a data block
    experiments = flatten_experiments(params.input.experiments)
    if len(experiments) == 0:
      self.parser.print_help()
      return

    # Only handle a single imageset at once
    imagesets = set(expr.imageset for expr in experiments)
    if len(imagesets) != 1:
      raise Sorry("Can only process a single imageset at a time")

    # Predict the reflections
    logger.info("")
    logger.info("=" * 80)
    logger.info("")
    logger.info(heading("Predicting reflections"))
    logger.info("")
    predicted = flex.reflection_table.from_predictions_multi(
      experiments,
      dmin=params.prediction.d_min,
      dmax=params.prediction.d_max,
      margin=params.prediction.margin,
      force_static=params.prediction.force_static)

    # Create the modeller
    modeller = BackgroundModeller(experiments, predicted, params)
    model = modeller.compute()

    # Save the background model
    logger.info("Saving background model to %s" % params.output.model)
    from dials.algorithms.background.gmodel import StaticBackgroundModel
    static_model = StaticBackgroundModel()
    for i in range(len(model)):
      static_model.add(model[i].model)
    with open(params.output.model, "w") as outfile:
      import cPickle as pickle
      pickle.dump(static_model, outfile, protocol=pickle.HIGHEST_PROTOCOL)

    # Output some diagnostic images
    image_generator = ImageGenerator(model)
    image_generator.save_mean(params.output.mean_image_prefix)
    image_generator.save_variance(params.output.variance_image_prefix)
    image_generator.save_dispersion(params.output.dispersion_image_prefix)
    image_generator.save_mask(params.output.mask_image_prefix)
    image_generator.save_min(params.output.min_image_prefix)
    image_generator.save_max(params.output.max_image_prefix)
    image_generator.save_model(params.output.model_image_prefix)
    #image_generator.save_polar_model(params.output.polar_model_image_prefix)

    # Print the time
    logger.info("Time Taken: %f" % (time() - start_time))
Exemple #4
0
  def run(self):
    from os.path import join, exists
    from libtbx import easy_run
    import os
    from uuid import uuid4

    os.mkdir('simple')
    os.mkdir('robust')
    os.mkdir('gmodel_simple')
    os.mkdir('gmodel_robust')

    assert exists(join(self.path, 'experiments.json'))


    from dials.array_family import flex
    from dials.algorithms.background.gmodel import StaticBackgroundModel
    ysize = 2527
    xsize = 2463
    data = flex.double(flex.grid(ysize, xsize), 1)
    model = StaticBackgroundModel()
    model.add(data)
    import cPickle as pickle
    pickle.dump(model, open("model.pickle", "w"))

    # Call dials.integrate
    easy_run.fully_buffered([
      'dials.integrate',
      join(self.path, 'experiments.json'),
      'profile.fitting=False',
      'background.algorithm=simple',
      'background.simple.outlier.algorithm=null',
      'output.reflections=./simple/reflections.pickle'
    ]).raise_if_errors()

    assert exists("simple/reflections.pickle")

    # Call dials.integrate
    easy_run.fully_buffered([
      'dials.integrate',
      join(self.path, 'experiments.json'),
      'profile.fitting=False',
      'background.algorithm=glm',
      'output.reflections=./robust/reflections.pickle'
    ]).raise_if_errors()

    assert exists("robust/reflections.pickle")

    # Call dials.integrate
    easy_run.fully_buffered([
      'dials.integrate',
      join(self.path, 'experiments.json'),
      'profile.fitting=False',
      'background.algorithm=gmodel',
      'background.gmodel.robust.algorithm=False',
      'background.gmodel.model=model.pickle',
      'output.reflections=./gmodel_simple/reflections.pickle'
    ]).raise_if_errors()

    assert exists("gmodel_simple/reflections.pickle")

    # Call dials.integrate
    easy_run.fully_buffered([
      'dials.integrate',
      join(self.path, 'experiments.json'),
      'profile.fitting=False',
      'background.algorithm=gmodel',
      'background.gmodel.robust.algorithm=True',
      'background.gmodel.model=model.pickle',
      'output.reflections=./gmodel_robust/reflections.pickle'
    ]).raise_if_errors()

    assert exists("gmodel_robust/reflections.pickle")

    from dials.array_family import flex
    reflections1 = flex.reflection_table.from_pickle("simple/reflections.pickle")
    reflections2 = flex.reflection_table.from_pickle("robust/reflections.pickle")
    reflections3 = flex.reflection_table.from_pickle("gmodel_simple/reflections.pickle")
    reflections4 = flex.reflection_table.from_pickle("gmodel_robust/reflections.pickle")

    assert len(reflections1) == len(reflections3)
    assert len(reflections2) == len(reflections4)

    flag = flex.reflection_table.flags.integrated_sum
    integrated1 = reflections1.select(reflections1.get_flags(flag, all=True))
    integrated2 = reflections2.select(reflections2.get_flags(flag, all=True))
    integrated3 = reflections3.select(reflections3.get_flags(flag, all=True))
    integrated4 = reflections4.select(reflections4.get_flags(flag, all=True))

    assert len(integrated1) > 0
    assert len(integrated2) > 0
    assert len(integrated1) == len(integrated3)
    assert len(integrated2) == len(integrated4)

    mean_bg1 = integrated1['background.mean']
    mean_bg2 = integrated2['background.mean']
    mean_bg3 = integrated3['background.mean']
    mean_bg4 = integrated4['background.mean']

    scale3 = integrated3['background.scale']
    scale4 = integrated4['background.scale']

    diff1 = flex.abs(mean_bg1 - mean_bg3)
    diff2 = flex.abs(mean_bg2 - mean_bg4)
    assert (scale3 > 0).count(False) == 0
    assert (scale4 > 0).count(False) == 0
    assert (diff1 < 1e-5).count(False) == 0

    print 'OK'
Exemple #5
0
    def run(self):
        from os.path import join, exists
        from libtbx import easy_run
        import os
        from uuid import uuid4

        os.mkdir("simple")
        os.mkdir("robust")
        os.mkdir("gmodel_simple")
        os.mkdir("gmodel_robust")

        assert exists(join(self.path, "experiments.json"))

        from dials.array_family import flex
        from dials.algorithms.background.gmodel import StaticBackgroundModel

        ysize = 2527
        xsize = 2463
        data = flex.double(flex.grid(ysize, xsize), 1)
        model = StaticBackgroundModel()
        model.add(data)
        import cPickle as pickle

        pickle.dump(model, open("model.pickle", "w"))

        # Call dials.integrate
        easy_run.fully_buffered(
            [
                "dials.integrate",
                join(self.path, "experiments.json"),
                "profile.fitting=False",
                "background.algorithm=simple",
                "background.simple.outlier.algorithm=null",
                "output.reflections=./simple/reflections.pickle",
            ]
        ).raise_if_errors()

        assert exists("simple/reflections.pickle")

        # Call dials.integrate
        easy_run.fully_buffered(
            [
                "dials.integrate",
                join(self.path, "experiments.json"),
                "profile.fitting=False",
                "background.algorithm=glm",
                "output.reflections=./robust/reflections.pickle",
            ]
        ).raise_if_errors()

        assert exists("robust/reflections.pickle")

        # Call dials.integrate
        easy_run.fully_buffered(
            [
                "dials.integrate",
                join(self.path, "experiments.json"),
                "profile.fitting=False",
                "background.algorithm=gmodel",
                "background.gmodel.robust.algorithm=False",
                "background.gmodel.model=model.pickle",
                "output.reflections=./gmodel_simple/reflections.pickle",
            ]
        ).raise_if_errors()

        assert exists("gmodel_simple/reflections.pickle")

        # Call dials.integrate
        easy_run.fully_buffered(
            [
                "dials.integrate",
                join(self.path, "experiments.json"),
                "profile.fitting=False",
                "background.algorithm=gmodel",
                "background.gmodel.robust.algorithm=True",
                "background.gmodel.model=model.pickle",
                "output.reflections=./gmodel_robust/reflections.pickle",
            ]
        ).raise_if_errors()

        assert exists("gmodel_robust/reflections.pickle")

        from dials.array_family import flex

        reflections1 = flex.reflection_table.from_pickle("simple/reflections.pickle")
        reflections2 = flex.reflection_table.from_pickle("robust/reflections.pickle")
        reflections3 = flex.reflection_table.from_pickle("gmodel_simple/reflections.pickle")
        reflections4 = flex.reflection_table.from_pickle("gmodel_robust/reflections.pickle")

        assert len(reflections1) == len(reflections3)
        assert len(reflections2) == len(reflections4)

        flag = flex.reflection_table.flags.integrated_sum
        integrated1 = reflections1.select(reflections1.get_flags(flag, all=True))
        integrated2 = reflections2.select(reflections2.get_flags(flag, all=True))
        integrated3 = reflections3.select(reflections3.get_flags(flag, all=True))
        integrated4 = reflections4.select(reflections4.get_flags(flag, all=True))

        assert len(integrated1) > 0
        assert len(integrated2) > 0
        assert len(integrated1) == len(integrated3)
        assert len(integrated2) == len(integrated4)

        mean_bg1 = integrated1["background.mean"]
        mean_bg2 = integrated2["background.mean"]
        mean_bg3 = integrated3["background.mean"]
        mean_bg4 = integrated4["background.mean"]

        scale3 = integrated3["background.scale"]
        scale4 = integrated4["background.scale"]

        diff1 = flex.abs(mean_bg1 - mean_bg3)
        diff2 = flex.abs(mean_bg2 - mean_bg4)
        assert (scale3 > 0).count(False) == 0
        assert (scale4 > 0).count(False) == 0
        assert (diff1 < 1e-5).count(False) == 0

        print "OK"
Exemple #6
0
def test_run(dials_regression, tmpdir):
    import os
    path = os.path.join(dials_regression, "centroid_test_data")

    tmpdir.chdir()

    from libtbx import easy_run

    os.mkdir('simple')
    os.mkdir('robust')
    os.mkdir('gmodel_simple')
    os.mkdir('gmodel_robust')

    assert os.path.exists(os.path.join(path, 'experiments.json'))

    from dials.array_family import flex
    from dials.algorithms.background.gmodel import StaticBackgroundModel
    ysize = 2527
    xsize = 2463
    data = flex.double(flex.grid(ysize, xsize), 1)
    model = StaticBackgroundModel()
    model.add(data)
    import six.moves.cPickle as pickle
    with open("model.pickle", "wb") as fh:
        pickle.dump(model, fh, pickle.HIGHEST_PROTOCOL)

    # Call dials.integrate
    easy_run.fully_buffered([
        'dials.integrate',
        os.path.join(path, 'experiments.json'), 'profile.fitting=False',
        'background.algorithm=simple',
        'background.simple.outlier.algorithm=null',
        'output.reflections=./simple/reflections.pickle'
    ]).raise_if_errors()

    assert os.path.exists("simple/reflections.pickle")

    # Call dials.integrate
    easy_run.fully_buffered([
        'dials.integrate',
        os.path.join(path, 'experiments.json'), 'profile.fitting=False',
        'background.algorithm=glm',
        'output.reflections=./robust/reflections.pickle'
    ]).raise_if_errors()

    assert os.path.exists("robust/reflections.pickle")

    # Call dials.integrate
    easy_run.fully_buffered([
        'dials.integrate',
        os.path.join(path, 'experiments.json'), 'profile.fitting=False',
        'background.algorithm=gmodel',
        'background.gmodel.robust.algorithm=False',
        'background.gmodel.model=model.pickle',
        'output.reflections=./gmodel_simple/reflections.pickle'
    ]).raise_if_errors()

    assert os.path.exists("gmodel_simple/reflections.pickle")

    # Call dials.integrate
    easy_run.fully_buffered([
        'dials.integrate',
        os.path.join(path, 'experiments.json'), 'profile.fitting=False',
        'background.algorithm=gmodel',
        'background.gmodel.robust.algorithm=True',
        'background.gmodel.model=model.pickle',
        'output.reflections=./gmodel_robust/reflections.pickle'
    ]).raise_if_errors()

    assert os.path.exists("gmodel_robust/reflections.pickle")

    reflections1 = flex.reflection_table.from_pickle(
        "simple/reflections.pickle")
    reflections2 = flex.reflection_table.from_pickle(
        "robust/reflections.pickle")
    reflections3 = flex.reflection_table.from_pickle(
        "gmodel_simple/reflections.pickle")
    reflections4 = flex.reflection_table.from_pickle(
        "gmodel_robust/reflections.pickle")

    assert len(reflections1) == len(reflections3)
    assert len(reflections2) == len(reflections4)

    flag = flex.reflection_table.flags.integrated_sum
    integrated1 = reflections1.select(reflections1.get_flags(flag, all=True))
    integrated2 = reflections2.select(reflections2.get_flags(flag, all=True))
    integrated3 = reflections3.select(reflections3.get_flags(flag, all=True))
    integrated4 = reflections4.select(reflections4.get_flags(flag, all=True))

    assert len(integrated1) > 0
    assert len(integrated2) > 0
    assert len(integrated1) == len(integrated3)
    assert len(integrated2) == len(integrated4)

    mean_bg1 = integrated1['background.mean']
    mean_bg2 = integrated2['background.mean']
    mean_bg3 = integrated3['background.mean']
    mean_bg4 = integrated4['background.mean']

    scale3 = integrated3['background.scale']
    scale4 = integrated4['background.scale']

    diff1 = flex.abs(mean_bg1 - mean_bg3)
    diff2 = flex.abs(mean_bg2 - mean_bg4)
    assert (scale3 > 0).count(False) == 0
    assert (scale4 > 0).count(False) == 0
    assert (diff1 < 1e-5).count(False) == 0