Example #1
0
  def process_datablock(self, tag, datablock):
    import os
    s = tag

    # before processing, set output paths according to the templates
    if self.params.output.datablock_filename is not None and "%s" in self.params.output.datablock_filename:
      self.params.output.datablock_filename = os.path.join(self.params.output.output_dir, self.params.output.datablock_filename%("idx-" + s))
    if self.params.output.strong_filename is not None and "%s" in self.params.output.strong_filename:
      self.params.output.strong_filename = os.path.join(self.params.output.output_dir, self.params.output.strong_filename%("idx-" + s))
    if self.params.output.indexed_filename is not None and "%s" in self.params.output.indexed_filename:
      self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.params.output.indexed_filename%("idx-" + s))
    if "%s" in self.params.output.refined_experiments_filename:
      self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.params.output.refined_experiments_filename%("idx-" + s))
    if "%s" in self.params.output.integrated_filename:
      self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.params.output.integrated_filename%("idx-" + s))
    self.tag = tag

    if self.params.output.datablock_filename:
      from dxtbx.datablock import DataBlockDumper
      dump = DataBlockDumper(datablock)
      dump.as_json(self.params.output.datablock_filename)

    # Do the processing
    try:
      observed = self.find_spots(datablock)
    except Exception, e:
      print "Error spotfinding", tag, str(e)
      return
def test_split_single_image_datablock(dials_regression, tmpdir):
    tmpdir.chdir()
    pytest.importorskip("h5py")
    sacla_file = os.path.join(
        dials_regression,
        "image_examples",
        "SACLA_MPCCD_Cheetah",
        "run266702-0-subset.h5",
    )
    db = DataBlockFactory.from_filenames([sacla_file])[0]
    assert db.num_images() == 4
    imageset = db.extract_imagesets()[0]
    subset = imageset[2:3]
    subblock = DataBlockFactory.from_imageset(subset)[0]
    assert subblock.num_images() == 1
    assert get_indices(subblock) == [2]

    dumped_filename = "split_datablock.json"
    dump = DataBlockDumper(subblock)
    dump.as_json(dumped_filename)

    db = DataBlockFactory.from_json_file(dumped_filename, check_format=True)[0]
    assert db.num_images() == 1
    assert get_indices(db) == [2]

    db = DataBlockFactory.from_json_file(dumped_filename,
                                         check_format=False)[0]
    assert db.num_images() == 1
    assert get_indices(db) == [2]
Example #3
0
  def process_datablock(self, tag, datablock):
    import os
    s = tag

    # before processing, set output paths according to the templates
    if self.params.output.datablock_filename is not None and "%s" in self.params.output.datablock_filename:
      self.params.output.datablock_filename = os.path.join(self.params.output.output_dir, self.params.output.datablock_filename%("idx-" + s))
    if self.params.output.strong_filename is not None and "%s" in self.params.output.strong_filename:
      self.params.output.strong_filename = os.path.join(self.params.output.output_dir, self.params.output.strong_filename%("idx-" + s))
    if self.params.output.indexed_filename is not None and "%s" in self.params.output.indexed_filename:
      self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.params.output.indexed_filename%("idx-" + s))
    if "%s" in self.params.output.refined_experiments_filename:
      self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.params.output.refined_experiments_filename%("idx-" + s))
    if "%s" in self.params.output.integrated_filename:
      self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.params.output.integrated_filename%("idx-" + s))
    self.tag = tag

    if self.params.output.datablock_filename:
      from dxtbx.datablock import DataBlockDumper
      dump = DataBlockDumper(datablock)
      dump.as_json(self.params.output.datablock_filename)

    # Do the processing
    try:
      observed = self.find_spots(datablock)
    except Exception, e:
      print "Error spotfinding", tag, str(e)
      return
Example #4
0
    def process_datablock(self, tag, datablock):
        import os

        if not self.params.output.composite_output:
            self.setup_filenames(tag)
        self.tag = tag

        if self.params.output.datablock_filename:
            from dxtbx.datablock import DataBlockDumper
            dump = DataBlockDumper(datablock)
            dump.as_json(self.params.output.datablock_filename)

        # Do the processing
        try:
            self.pre_process(datablock)
        except Exception as e:
            print("Error in pre-process", tag, str(e))
            if not self.params.dispatch.squash_errors: raise
            return
        try:
            if self.params.dispatch.find_spots:
                observed = self.find_spots(datablock)
            else:
                print("Spot Finding turned off. Exiting")
                return
        except Exception as e:
            print("Error spotfinding", tag, str(e))
            if not self.params.dispatch.squash_errors: raise
            return
        try:
            if self.params.dispatch.index:
                experiments, indexed = self.index(datablock, observed)
            else:
                print("Indexing turned off. Exiting")
                return
        except Exception as e:
            print("Couldn't index", tag, str(e))
            if not self.params.dispatch.squash_errors: raise
            return
        try:
            experiments, indexed = self.refine(experiments, indexed)
        except Exception as e:
            print("Error refining", tag, str(e))
            if not self.params.dispatch.squash_errors: raise
            return
        try:
            if self.params.dispatch.integrate:
                integrated = self.integrate(experiments, indexed)
            else:
                print("Integration turned off. Exiting")
                return
        except Exception as e:
            print("Error integrating", tag, str(e))
            if not self.params.dispatch.squash_errors: raise
            return
def run(all_paths, hits):
    print 'importing {}'.format(all_paths)
    datablocks = [do_import(path) for path in all_paths]
    split_datablocks = []
    print 'processing datablocks'
    counter = 0
    for datablock in datablocks:
        for imageset in datablock.extract_imagesets():
            paths = imageset.paths()
            for i in xrange(len(imageset)):
                print i
                subset = imageset[i:i + 1]
                split_datablocks.append(
                    DataBlockFactory.from_imageset(subset)[0])
                if i in hits:
                    counter += 1
                    print(paths[i])
                    dump = DataBlockDumper(split_datablocks[i])
                    dump.as_json('datablock_%i.json' % i)
    return counter
Example #6
0
    def run(self):
        ''' Run the script. '''
        from dials.util.options import flatten_datablocks
        from dxtbx.datablock import DataBlockDumper
        from libtbx.utils import Sorry
        import cPickle as pickle

        # Parse the command line arguments
        params, options = self.parser.parse_args(show_diff_phil=True)
        datablocks = flatten_datablocks(params.input.datablock)

        # Check number of args
        if len(datablocks) == 0:
            self.parser.print_help()
            return

        # Check the mask file is given
        if params.input.mask is None:
            self.parser.print_help()
            return

        # Check nbumber of datablocks
        if len(datablocks) != 1:
            raise Sorry('exactly 1 datablock must be specified')

        # Get the imageset
        datablock = datablocks[0]
        imagesets = datablock.extract_imagesets()
        if len(imagesets) != 1:
            raise Sorry('datablock must contain exactly 1 imageset')
        imageset = imagesets[0]

        # Set the lookup
        imageset.external_lookup.mask.filename = params.input.mask

        # Dump the datablock
        print "Writing datablock to %s" % params.output.datablock
        dump = DataBlockDumper(datablock)
        dump.as_json(filename=params.output.datablock)
Example #7
0
  def run(self):
    ''' Run the script. '''
    from dials.util.options import flatten_datablocks
    from dxtbx.datablock import DataBlockDumper
    from libtbx.utils import Sorry
    import cPickle as pickle

    # Parse the command line arguments
    params, options = self.parser.parse_args(show_diff_phil=True)
    datablocks = flatten_datablocks(params.input.datablock)

    # Check number of args
    if len(datablocks) == 0:
      self.parser.print_help()
      return

    # Check the mask file is given
    if params.input.mask is None:
      self.parser.print_help()
      return

    # Check nbumber of datablocks
    if len(datablocks) != 1:
      raise Sorry('exactly 1 datablock must be specified')

    # Get the imageset
    datablock = datablocks[0]
    imagesets = datablock.extract_imagesets()
    if len(imagesets) != 1:
      raise Sorry('datablock must contain exactly 1 imageset')
    imageset = imagesets[0]

    # Set the lookup
    imageset.external_lookup.mask.filename = params.input.mask

    # Dump the datablock
    print "Writing datablock to %s" % params.output.datablock
    dump = DataBlockDumper(datablock)
    dump.as_json(filename=params.output.datablock)
Example #8
0
    def run(self):
        '''Execute the script.'''
        from dxtbx.datablock import DataBlockTemplateImporter
        from dials.util.options import flatten_datablocks
        from dials.util import log
        from logging import info
        from time import time
        from libtbx.utils import Abort

        # Parse the command line
        params, options = self.parser.parse_args(show_diff_phil=False)
        datablocks = flatten_datablocks(params.input.datablock)

        # Check we have some filenames
        if len(datablocks) == 0:

            # Check if a template has been set and print help if not, otherwise try to
            # import the images based on the template input
            if len(params.input.template) == 0:
                self.parser.print_help()
                exit(0)
            else:
                importer = DataBlockTemplateImporter(params.input.template,
                                                     options.verbose)
                datablocks = importer.datablocks

        # Save the options
        self.options = options
        self.params = params
        self.load_reference_geometry()

        st = time()

        # Import stuff
        if len(datablocks) == 0:
            raise Abort('No datablocks specified')
        elif len(datablocks) > 1:
            raise Abort('Only 1 datablock can be processed at a time.')
        datablock = datablocks[0]

        if self.reference_detector is not None:
            for imageset in datablock.extract_imagesets():
                imageset.set_detector(self.reference_detector)

        # Configure logging
        log.config(params.verbosity,
                   info='dials.process.log',
                   debug='dials.process.debug.log')

        # Log the diff phil
        diff_phil = self.parser.diff_phil.as_str()
        if diff_phil is not '':
            info('The following parameters have been modified:\n')
            info(diff_phil)

        if self.params.output.datablock_filename:
            from dxtbx.datablock import DataBlockDumper
            dump = DataBlockDumper(datablock)
            dump.as_json(self.params.output.datablock_filename)

        # Do the processing
        observed = self.find_spots(datablock)
        experiments, indexed = self.index(datablock, observed)
        experiments, indexed = self.refine(experiments, indexed)
        integrated = self.integrate(experiments, indexed)

        # Total Time
        info("")
        info("Total Time Taken = %f seconds" % (time() - st))
Example #9
0
  def run(self):
    '''Execute the script.'''
    from dxtbx.datablock import DataBlockTemplateImporter
    from dials.util.options import flatten_datablocks
    from dials.util import log
    from logging import info
    from time import time
    from libtbx.utils import Abort

    # Parse the command line
    params, options = self.parser.parse_args(show_diff_phil=False)
    datablocks = flatten_datablocks(params.input.datablock)

    # Check we have some filenames
    if len(datablocks) == 0:

      # Check if a template has been set and print help if not, otherwise try to
      # import the images based on the template input
      if len(params.input.template) == 0:
        self.parser.print_help()
        exit(0)
      else:
        importer = DataBlockTemplateImporter(
          params.input.template,
          options.verbose)
        datablocks = importer.datablocks

    # Save the options
    self.options = options
    self.params = params
    self.load_reference_geometry()

    st = time()

    # Import stuff
    if len(datablocks) == 0:
      raise Abort('No datablocks specified')
    elif len(datablocks) > 1:
      raise Abort('Only 1 datablock can be processed at a time.')
    datablock = datablocks[0]

    if self.reference_detector is not None:
      for imageset in datablock.extract_imagesets():
        imageset.set_detector(self.reference_detector)

    # Configure logging
    log.config(
      params.verbosity,
      info='dials.process.log',
      debug='dials.process.debug.log')

    # Log the diff phil
    diff_phil = self.parser.diff_phil.as_str()
    if diff_phil is not '':
      info('The following parameters have been modified:\n')
      info(diff_phil)

    if self.params.output.datablock_filename:
      from dxtbx.datablock import DataBlockDumper
      dump = DataBlockDumper(datablock)
      dump.as_json(self.params.output.datablock_filename)

    # Do the processing
    observed = self.find_spots(datablock)
    experiments, indexed = self.index(datablock, observed)
    experiments = self.refine(experiments, indexed)
    integrated = self.integrate(experiments, indexed)

    # Total Time
    info("")
    info("Total Time Taken = %f seconds" % (time() - st))
Example #10
0
  def process_datablock(self, tag, datablock):

    if not self.params.output.composite_output:
      self.setup_filenames(tag)
    self.tag = tag

    if self.params.output.datablock_filename:
      from dxtbx.datablock import DataBlockDumper
      dump = DataBlockDumper(datablock)
      dump.as_json(self.params.output.datablock_filename)

    # Do the processing
    try:
      observed = self.find_spots(datablock)
    except Exception as e:
      print "Error spotfinding", tag, str(e)
      if not self.params.dispatch.squash_errors: raise
      return
    try:
      experiments, indexed = self.index(datablock, observed)
    except Exception as e:
      print "Couldn't index", tag, str(e)
      if not self.params.dispatch.squash_errors: raise
      return
    try:
      experiments, indexed = self.refine(experiments, indexed)
    except Exception as e:
      print "Error refining", tag, str(e)
      if not self.params.dispatch.squash_errors: raise
      return

    from diffuse_scattering.sematura import DiffuseExperiment, DiffuseImage
    def from_experiments(self,experiments):
        exp_xtal = experiments.crystals()[0]

        ### define useful attributes
        self.crystal = exp_xtal
        uc = self.crystal.get_unit_cell()
        uc_nospace = str(uc).replace(" ", "")
        uc_nospace_noparen = uc_nospace[1:-1]
        self.unit_cell = uc_nospace_noparen
        self.space_group = self.crystal.get_space_group()
        self.laue_group = self.space_group.laue_group_type()
        # self.a_matrix = crystal.get_A()
        self.experiments = experiments
    DiffuseExperiment.from_experiments = from_experiments

    if (self.params.mp.method == 'mpi'):
      from mpi4py import MPI
      comm = MPI.COMM_WORLD
      rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed
    else:
      rank = 0
    test_exp = DiffuseExperiment()
    test_exp.from_experiments(experiments)

    img_set = test_exp.experiments.imagesets()
    imgs = img_set[0]
    file_list = imgs.paths()
    img_file = file_list[0]
    test_img = DiffuseImage(img_file)
    test_img.set_general_variables()
    test_img.remove_bragg_peaks(radial=True)
    if self.ncalls == 0: 
      if rank == 0:
        self.ref_data = deepcopy(test_img.lunus_data_scitbx)
#        print "ref_data is",self.ref_data
#      else:
#        self.ref_data = None
      if (self.params.mp.method == 'mpi'):
#      from mpi4py import MPI
#      comm = MPI.COMM_WORLD
#        print "Barrier, rank = ",rank
#        print "Broadcast, rank = ",rank
        self.ref_data = comm.bcast(self.ref_data,root=0)
        comm.barrier()
#        print "Broadcast done, rank = ",rank

    if self.ref_data == None:
      print "ref_data = None for Rank = ",rank
    test_img.scale_factor_from_images(self.ref_data)
#    test_img.scale_factor()
    test_img.crystal_geometry(test_exp.crystal)

    test_img.setup_diffuse_lattice(self.params.lunus.d_min)
    test_img.integrate_diffuse()
    self.ncalls = self.ncalls + 1
Example #11
0
    def process_datablock(self, tag, datablock):

        if not self.params.output.composite_output:
            self.setup_filenames(tag)
        self.tag = tag

        if self.params.output.datablock_filename:
            from dxtbx.datablock import DataBlockDumper
            dump = DataBlockDumper(datablock)
            dump.as_json(self.params.output.datablock_filename)

        # Do the processing
        try:
            observed = self.find_spots(datablock)
        except Exception as e:
            print "Error spotfinding", tag, str(e)
            if not self.params.dispatch.squash_errors: raise
            return
        try:
            experiments, indexed = self.index(datablock, observed)
        except Exception as e:
            print "Couldn't index", tag, str(e)
            if not self.params.dispatch.squash_errors: raise
            return
        try:
            experiments, indexed = self.refine(experiments, indexed)
        except Exception as e:
            print "Error refining", tag, str(e)
            if not self.params.dispatch.squash_errors: raise
            return

        from diffuse_scattering.sematura import DiffuseExperiment, DiffuseImage

        def from_experiments(self, experiments):
            exp_xtal = experiments.crystals()[0]

            ### define useful attributes
            self.crystal = exp_xtal
            uc = self.crystal.get_unit_cell()
            uc_nospace = str(uc).replace(" ", "")
            uc_nospace_noparen = uc_nospace[1:-1]
            self.unit_cell = uc_nospace_noparen
            self.space_group = self.crystal.get_space_group()
            self.laue_group = self.space_group.laue_group_type()
            # self.a_matrix = crystal.get_A()
            self.experiments = experiments

        DiffuseExperiment.from_experiments = from_experiments

        if (self.params.mp.method == 'mpi'):
            from mpi4py import MPI
            comm = MPI.COMM_WORLD
            rank = comm.Get_rank(
            )  # each process in MPI has a unique id, 0-indexed
        else:
            rank = 0
        test_exp = DiffuseExperiment()
        test_exp.from_experiments(experiments)

        img_set = test_exp.experiments.imagesets()
        imgs = img_set[0]
        file_list = imgs.paths()
        img_file = file_list[0]
        test_img = DiffuseImage(img_file)
        test_img.set_general_variables()
        test_img.remove_bragg_peaks(radial=True)
        if self.ncalls == 0:
            if rank == 0:
                self.ref_data = deepcopy(test_img.lunus_data_scitbx)
#        print "ref_data is",self.ref_data
#      else:
#        self.ref_data = None
            if (self.params.mp.method == 'mpi'):
                #      from mpi4py import MPI
                #      comm = MPI.COMM_WORLD
                #        print "Barrier, rank = ",rank
                #        print "Broadcast, rank = ",rank
                self.ref_data = comm.bcast(self.ref_data, root=0)
                comm.barrier()
#        print "Broadcast done, rank = ",rank

        if self.ref_data == None:
            print "ref_data = None for Rank = ", rank
        test_img.scale_factor_from_images(self.ref_data)
        #    test_img.scale_factor()
        test_img.crystal_geometry(test_exp.crystal)

        test_img.setup_diffuse_lattice(self.params.lunus.d_min)
        test_img.integrate_diffuse()
        self.ncalls = self.ncalls + 1