def tst_from_datablock(self): from dxtbx.imageset import ImageSweep, NullReader, SweepFileList from dxtbx.model import Beam, Detector, Goniometer, Scan from dxtbx.datablock import DataBlockFactory from dxtbx.model.crystal import crystal_model imageset = ImageSweep(NullReader(SweepFileList("filename%01d.cbf", (0, 2)))) imageset.set_beam(Beam()) imageset.set_detector(Detector()) imageset.set_goniometer(Goniometer()) imageset.set_scan(Scan((1, 2), (0, 1))) crystal = crystal_model((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=0) datablock = DataBlockFactory.from_imageset(imageset) experiments = ExperimentListFactory.from_datablock_and_crystal( datablock, crystal) assert(len(experiments) == 1) assert(experiments[0].imageset is not None) assert(experiments[0].beam is not None) assert(experiments[0].detector is not None) assert(experiments[0].goniometer is not None) assert(experiments[0].scan is not None) assert(experiments[0].crystal is not None) print 'OK' pass
def tst_from_datablock(self): from dxtbx.imageset import ImageSweep, NullReader, SweepFileList from dxtbx.model import Beam, Detector, Goniometer, Scan from dxtbx.datablock import DataBlockFactory from dxtbx.model.crystal import crystal_model imageset = ImageSweep(NullReader(SweepFileList("filename%01d.cbf", (0, 2)))) imageset.set_beam(Beam()) imageset.set_detector(Detector()) imageset.set_goniometer(Goniometer()) imageset.set_scan(Scan((1, 2), (0, 1))) crystal = crystal_model((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=0) datablock = DataBlockFactory.from_imageset(imageset) experiments = ExperimentListFactory.from_datablock_and_crystal( datablock, crystal) assert(len(experiments) == 1) assert(experiments[0].imageset is not None) assert(experiments[0].beam is not None) assert(experiments[0].detector is not None) assert(experiments[0].goniometer is not None) assert(experiments[0].scan is not None) assert(experiments[0].crystal is not None) print 'OK'
def tst_from_null_sweep(self): from dxtbx.datablock import DataBlockFactory from dxtbx.imageset import NullReader, ImageSweep, SweepFileList from dxtbx.model import Beam, Detector, Goniometer, Scan sweep = ImageSweep( NullReader(SweepFileList("template_%2d.cbf", (0, 10)))) sweep.set_beam(Beam((0, 0, 1))) sweep.set_detector(Detector()) sweep.set_goniometer(Goniometer((1, 0, 0))) sweep.set_scan(Scan((1, 10), (0, 0.1))) # Create the datablock datablock = DataBlockFactory.from_imageset(sweep) assert (len(datablock) == 1) datablock = datablock[0] sweeps = datablock.extract_sweeps() assert (len(sweeps) == 1) assert (sweeps[0].get_beam() == sweep.get_beam()) assert (sweeps[0].get_detector() == sweep.get_detector()) assert (sweeps[0].get_goniometer() == sweep.get_goniometer()) assert (sweeps[0].get_scan() == sweep.get_scan()) print 'OK'
def run(self): params, options = self.parser.parse_args(show_diff_phil=True) assert params.input.single_img is not None filebase = os.path.splitext(params.input.single_img)[0] for item in dir(params.output): value = getattr(params.output, item) try: if "%s" in value: setattr(params.output, item, value % filebase) except Exception: pass self.params = params self.options = options # load the image img = dxtbx.load(params.input.single_img) imgset = MemImageSet([img]) datablock = DataBlockFactory.from_imageset(imgset)[0] # Cannot export MemImageSets # if self.params.output.datablock_filename: # from dxtbx.datablock import DataBlockDumper # dump = DataBlockDumper(datablock) # dump.as_json(self.params.output.datablock_filename) observed = self.find_spots(datablock) experiments, indexed = self.index(datablock, observed) experiments = self.refine(experiments, indexed) integrated = self.integrate(experiments, indexed)
def tst_from_null_sweep(self): from dxtbx.format.Format import Format from dxtbx.imageset import ImageSweep from dxtbx.model import Beam, Detector, Goniometer, Scan filenames = ["template_%2d.cbf" % (i+1) for i in range(0, 10)] sweep = Format.get_imageset( filenames, beam = Beam((0, 0, 1)), detector = Detector(), goniometer = Goniometer((1, 0, 0)), scan = Scan((1, 10), (0, 0.1))) # Create the datablock datablock = DataBlockFactory.from_imageset(sweep) assert(len(datablock) == 1) datablock = datablock[0] sweeps = datablock.extract_sweeps() assert(len(sweeps) == 1) assert(sweeps[0].get_beam() == sweep.get_beam()) assert(sweeps[0].get_detector() == sweep.get_detector()) assert(sweeps[0].get_goniometer() == sweep.get_goniometer()) assert(sweeps[0].get_scan() == sweep.get_scan()) print 'OK'
def test_experimentlist_factory_from_datablock(): filenames = ["filename_%01d.cbf" % (i + 1) for i in range(0, 2)] imageset = Format.get_imageset( filenames, beam=Beam(), detector=Detector(), goniometer=Goniometer(), scan=Scan((1, 2), (0, 1)), as_sequence=True, ) crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol="P1") datablock = DataBlockFactory.from_imageset(imageset) assert datablock[0].format_class() experiments = ExperimentListFactory.from_datablock_and_crystal( datablock, crystal) assert len(experiments) == 1 assert experiments[0].imageset assert experiments[0].beam assert experiments[0].detector is not None assert experiments[0].goniometer assert experiments[0].scan assert experiments[0].crystal
def test_experimentlist_factory_from_datablock(): from dxtbx.model import Beam, Detector, Goniometer, Scan from dxtbx.datablock import DataBlockFactory from dxtbx.model import Crystal from dxtbx.format.Format import Format filenames = ["filename_%01d.cbf" % (i+1) for i in range(0, 2)] imageset = Format.get_imageset( filenames, beam = Beam(), detector = Detector(), goniometer = Goniometer(), scan = Scan((1,2), (0,1)), as_sweep=True) crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol="P1") datablock = DataBlockFactory.from_imageset(imageset) experiments = ExperimentListFactory.from_datablock_and_crystal( datablock, crystal) assert len(experiments) == 1 assert experiments[0].imageset is not None assert experiments[0].beam is not None assert experiments[0].detector is not None assert experiments[0].goniometer is not None assert experiments[0].scan is not None assert experiments[0].crystal is not None
def datablock_from_numpyarrays(image, detector, beam, mask=None): """ So that one can do e.g. >> dblock = datablock_from_numpyarrays( image, detector, beam) >> refl = flex.reflection_table.from_observations(dblock, spot_finder_params) without having to utilize the harddisk :param image: numpy array image, or list of numpy arrays :param mask: numpy mask, should be same shape format as numpy array :param detector: dxtbx detector model :param beam: dxtbx beam model :return: datablock for the image """ if isinstance(image, list): image = np.array(image) if mask is not None: if isinstance(mask, list): mask = np.array(mask).astype(bool) I = FormatInMemory(image=image, mask=mask) reader = MemReader([I]) #masker = MemMasker([I]) iset_Data = ImageSetData(reader, None) #, masker) iset = ImageSet(iset_Data) iset.set_beam(beam) iset.set_detector(detector) dblock = DataBlockFactory.from_imageset([iset])[0] return dblock
def do_import(filename): logger.info("Loading %s" % os.path.basename(filename)) datablocks = DataBlockFactory.from_filenames([filename]) if len(datablocks) == 0: try: datablocks = DataBlockFactory.from_json_file(filename) except ValueError: raise Abort("Could not load %s" % filename) if len(datablocks) == 0: raise Abort("Could not load %s" % filename) if len(datablocks) > 1: raise Abort("Got multiple datablocks from file %s" % filename) # Ensure the indexer and downstream applications treat this as set of stills reset_sets = [] from dxtbx.imageset import ImageSetFactory for imageset in datablocks[0].extract_imagesets(): imageset = ImageSetFactory.imageset_from_anyset(imageset) imageset.set_scan(None) imageset.set_goniometer(None) reset_sets.append(imageset) return DataBlockFactory.from_imageset(reset_sets)[0]
def test_split_single_image_datablock(dials_regression, tmpdir): tmpdir.chdir() pytest.importorskip("h5py") sacla_file = os.path.join( dials_regression, "image_examples", "SACLA_MPCCD_Cheetah", "run266702-0-subset.h5", ) db = DataBlockFactory.from_filenames([sacla_file])[0] assert db.num_images() == 4 imageset = db.extract_imagesets()[0] subset = imageset[2:3] subblock = DataBlockFactory.from_imageset(subset)[0] assert subblock.num_images() == 1 assert get_indices(subblock) == [2] dumped_filename = "split_datablock.json" dump = DataBlockDumper(subblock) dump.as_json(dumped_filename) db = DataBlockFactory.from_json_file(dumped_filename, check_format=True)[0] assert db.num_images() == 1 assert get_indices(db) == [2] db = DataBlockFactory.from_json_file(dumped_filename, check_format=False)[0] assert db.num_images() == 1 assert get_indices(db) == [2]
def test_split_single_image_datablock(dials_data, tmpdir): tmpdir.chdir() pytest.importorskip("h5py") sacla_file = os.path.join( dials_data("image_examples"), "SACLA-MPCCD-run266702-0-subset.h5", ) db = DataBlockFactory.from_filenames([sacla_file])[0] assert db.num_images() == 4 imageset = db.extract_imagesets()[0] subset = imageset[2:3] subblock = DataBlockFactory.from_imageset(subset)[0] assert subblock.num_images() == 1 assert get_indices(subblock) == [2]
def test_cspad_cbf_in_memory(dials_regression, tmpdir): tmpdir.chdir() # Check the data files for this test exist image_path = os.path.join(dials_regression, "image_examples", "LCLS_cspad_nexus", 'idx-20130301060858801.cbf') assert os.path.isfile(image_path) with open("process_lcls.phil", 'w') as f: f.write(""" dispatch.squash_errors = False spotfinder { filter.min_spot_size=2 threshold.dispersion.gain=25 threshold.dispersion.global_threshold=100 } indexing { known_symmetry { space_group = P6122 unit_cell = 92.9 92.9 130.4 90 90 120 } refinement_protocol.d_min_start=1.7 stills.refine_candidates_with_known_symmetry=True } """) params = phil_scope.fetch(parse(file_name="process_lcls.phil")).extract() params.output.datablock_filename = None processor = Processor(params) mem_img = dxtbx.load(image_path) raw_data = mem_img.get_raw_data( ) # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet(ImageSetData(MemReader([mem_img]), MemMasker([mem_img]))) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) datablock = DataBlockFactory.from_imageset(imgset)[0] processor.process_datablock("20130301060858801", datablock) # index/integrate the image result = "idx-20130301060858801_integrated.pickle" n_refls = range( 140, 152) # large ranges to handle platform-specific differences with open(result, 'rb') as f: table = pickle.load(f) assert len(table) in n_refls, len(table) assert 'id' in table assert (table['id'] == 0).count(False) == 0
def test_from_null_sweep(): filenames = ["template_%2d.cbf" % (i + 1) for i in range(0, 10)] sweep = Format.get_imageset(filenames, beam=Beam((0, 0, 1)), detector=Detector(), goniometer=Goniometer((1, 0, 0)), scan=Scan((1, 10), (0, 0.1))) # Create the datablock datablock = DataBlockFactory.from_imageset(sweep) assert len(datablock) == 1 datablock = datablock[0] sweeps = datablock.extract_sweeps() assert len(sweeps) == 1 assert sweeps[0].get_beam() == sweep.get_beam() assert sweeps[0].get_detector() == sweep.get_detector() assert sweeps[0].get_goniometer() == sweep.get_goniometer() assert sweeps[0].get_scan() == sweep.get_scan()
def run(all_paths, hits): print 'importing {}'.format(all_paths) datablocks = [do_import(path) for path in all_paths] split_datablocks = [] print 'processing datablocks' counter = 0 for datablock in datablocks: for imageset in datablock.extract_imagesets(): paths = imageset.paths() for i in xrange(len(imageset)): print i subset = imageset[i:i + 1] split_datablocks.append( DataBlockFactory.from_imageset(subset)[0]) if i in hits: counter += 1 print(paths[i]) dump = DataBlockDumper(split_datablocks[i]) dump.as_json('datablock_%i.json' % i) return counter
def datablock_from_numpyarrays(image, detector, beam, mask=None): """ So that one can do e.g. >> dblock = datablock_from_numpyarrays( image, detector, beam) >> refl = flex.reflection_table.from_observations(dblock, spot_finder_params) without having to utilize the harddisk :param image: numpy array image :param mask: numpy mask :param detector: dxtbx detector model :param beam: dxtbx beam model :return: datablock for the image """ I = FormatInMemory(image=image, mask=mask) reader = MemReader([I]) masker = MemMasker([I]) iset_Data = ImageSetData(reader, masker) iset = ImageSet(iset_Data) iset.set_beam(beam) iset.set_detector(detector) dblock = DataBlockFactory.from_imageset([iset])[0] return dblock
def test_from_null_sequence(): filenames = ["template_%2d.cbf" % (i + 1) for i in range(0, 10)] sequence = Format.get_imageset( filenames, beam=Beam((0, 0, 1)), detector=Detector(), goniometer=Goniometer((1, 0, 0)), scan=Scan((1, 10), (0, 0.1)), ) # Create the datablock datablock = DataBlockFactory.from_imageset(sequence) assert len(datablock) == 1 datablock = datablock[0] assert datablock.format_class() sequences = datablock.extract_sequences() assert len(sequences) == 1 assert sequences[0].get_beam() == sequence.get_beam() assert sequences[0].get_detector() == sequence.get_detector() assert sequences[0].get_goniometer() == sequence.get_goniometer() assert sequences[0].get_scan() == sequence.get_scan()
def datablock_from_numpyarrays(image, detector, beam, mask=None): """ put the numpy array image(s) into a dials datablock :param image: numpy array of images or an image :param detector: dxtbx det :param beam: dxtbx beam :param mask: mask same shape as image , 1 is not masked, boolean :return: """ if isinstance( image, list): image = np.array( image) if mask is not None: if isinstance( mask, list): mask = np.array(mask).astype(bool) I = FormatInMemory(image=image, mask=mask) reader = MemReader([I]) masker = MemMasker([I]) iset_Data = ImageSetData(reader, masker) iset = ImageSet(iset_Data) iset.set_beam(beam) iset.set_detector(detector) dblock = DataBlockFactory.from_imageset([iset])[0] return dblock
def do_import(filename): logger.info("Loading %s"%os.path.basename(filename)) try: datablocks = DataBlockFactory.from_json_file(filename) except ValueError: datablocks = DataBlockFactory.from_filenames([filename]) if len(datablocks) == 0: raise Abort("Could not load %s"%filename) if len(datablocks) > 1: raise Abort("Got multiple datablocks from file %s"%filename) # Ensure the indexer and downstream applications treat this as set of stills from dxtbx.imageset import ImageSet reset_sets = [] for imageset in datablocks[0].extract_imagesets(): imageset = ImageSet(imageset.reader(), imageset.indices()) imageset._models = imageset._models imageset.set_scan(None) imageset.set_goniometer(None) reset_sets.append(imageset) return DataBlockFactory.from_imageset(reset_sets)[0]
def tst_from_null_sweep(self): from dxtbx.datablock import DataBlockFactory from dxtbx.imageset import NullReader, ImageSweep, SweepFileList from dxtbx.model import Beam, Detector, Goniometer, Scan sweep = ImageSweep(NullReader(SweepFileList("template_%2d.cbf", (0, 10)))) sweep.set_beam(Beam((0, 0, 1))) sweep.set_detector(Detector()) sweep.set_goniometer(Goniometer((1, 0, 0))) sweep.set_scan(Scan((1, 10), (0, 0.1))) # Create the datablock datablock = DataBlockFactory.from_imageset(sweep) assert(len(datablock) == 1) datablock = datablock[0] sweeps = datablock.extract_sweeps() assert(len(sweeps) == 1) assert(sweeps[0].get_beam() == sweep.get_beam()) assert(sweeps[0].get_detector() == sweep.get_detector()) assert(sweeps[0].get_goniometer() == sweep.get_goniometer()) assert(sweeps[0].get_scan() == sweep.get_scan()) print 'OK'
def test_elliptical_distortion(tmpdir): """Create distortion maps for elliptical distortion using a dummy datablock with a small detector, for speed. Check those maps seem sensible""" tmpdir.chdir() # Make a detector model d = make_detector() # The beam is also essential for a datablock to be serialisable b = Beam((0, 0, 1), 1.0) # Create and write out a datablock imageset = ImageSet( ImageSetData(Reader(["non-existent.cbf"]), Masker(["non-existent.cbf"]))) imageset.set_detector(d) imageset.set_beam(b) datablocks = DataBlockFactory.from_imageset(imageset) dump = DataBlockDumper(datablocks) dump.as_file("dummy_datablock.json") # Centre of distortion will be the far corner from the origin of the first # panel centre_xy = d[0].get_image_size_mm() # Generate distortion maps cmd = ("dials.generate_distortion_maps dummy_datablock.json " "mode=ellipse centre_xy={},{} " "phi=0 l1=1.0 l2=0.95").format(*centre_xy) result = easy_run.fully_buffered(command=cmd).raise_if_errors() # Load the maps with open("dx.pickle", "rb") as f: dx = pickle.load(f) with open("dy.pickle", "rb") as f: dy = pickle.load(f) # Check there are 4 maps each assert len(dx) == len(dy) == 4 # Ellipse has phi=0, so all correction is in the dy map for arr in dx: assert min(arr) == max(arr) == 0.0 # The ellipse correction is centred at the middle of the detector and all in # the Y direction. Therefore we expect a few things from the dy maps: # # (1) Within each panel the columns of the array are identical. # (2) The two upper panels should be the same # (3) The two lower panels should be the same. # (4) One column from an upper panel is a negated, reversed column from a # lower panel. # # All together expect the 4 dy maps to look something like this: # # /-----------\ /-----------\ # |-3 -3 -3 -3| |-3 -3 -3 -3| # |-2 -2 -2 -2| |-2 -2 -2 -2| # |-1 -1 -1 -1| |-1 -1 -1 -1| # | 0 0 0 0| | 0 0 0 0| # \-----------/ \-----------/ # /-----------\ /-----------\ # | 0 0 0 0| | 0 0 0 0| # | 1 1 1 1| | 1 1 1 1| # | 2 2 2 2| | 2 2 2 2| # | 3 3 3 3| | 3 3 3 3| # \-----------/ \-----------/ # So the fundamental data is all in the first column of first panel's map col0 = dy[0].matrix_copy_column(0) # The correction should be 5% of the distance from the ellipse centre to a # corrected pixel (l2 = 0.95 above) along the slow axis. Check that is the # case (for the first pixel at least) vec_centre_to_first_px = (matrix.col(d[0].get_pixel_lab_coord( (0.5, 0.5))) - matrix.col(d[0].get_lab_coord(centre_xy))) dist_centre_to_first_px = vec_centre_to_first_px.dot( matrix.col(d[0].get_slow_axis())) corr_mm = dist_centre_to_first_px * 0.05 corr_px = corr_mm / d[0].get_pixel_size()[1] assert col0[0] == pytest.approx(corr_px) # Test (1) from above list for panel 0 for i in range(1, 50): assert (col0 == dy[0].matrix_copy_column(i)).all_eq(True) # Test (2) assert (dy[0] == dy[1]).all_eq(True) # Test (3) assert (dy[2] == dy[3]).all_eq(True) # Test (4) assert col0 == pytest.approx(-1.0 * dy[2].matrix_copy_column(0).reversed()) # Test (1) for panel 2 as well, which then covers everything needed col0 = dy[2].matrix_copy_column(0) for i in range(1, 50): assert (col0 == dy[2].matrix_copy_column(i)).all_eq(True)
def run(self): '''Execute the script.''' from dials.util import log from time import time from libtbx import easy_mp import copy # Parse the command line params, options, all_paths = self.parser.parse_args( show_diff_phil=False, return_unhandled=True, quick_parse=True) # Check we have some filenames if not all_paths: self.parser.print_help() return # Mask validation for mask_path in params.spotfinder.lookup.mask, params.integration.lookup.mask: if mask_path is not None and not os.path.isfile(mask_path): raise Sorry("Mask %s not found" % mask_path) # Save the options self.options = options self.params = params st = time() # Configure logging log.config(params.verbosity, info='dials.process.log', debug='dials.process.debug.log') # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) for abs_params in self.params.integration.absorption_correction: if abs_params.apply: if not (self.params.integration.debug.output and not self.params.integration.debug.separate_files): raise Sorry('Shoeboxes must be saved to integration intermediates to apply an absorption correction. '\ +'Set integration.debug.output=True, integration.debug.separate_files=False and '\ +'integration.debug.delete_shoeboxes=True to temporarily store shoeboxes.') self.load_reference_geometry() from dials.command_line.dials_import import ManualGeometryUpdater update_geometry = ManualGeometryUpdater(params) # Import stuff logger.info("Loading files...") pre_import = params.dispatch.pre_import or len(all_paths) == 1 if pre_import: # Handle still imagesets by breaking them apart into multiple datablocks # Further handle single file still imagesets (like HDF5) by tagging each # frame using its index datablocks = [do_import(path) for path in all_paths] indices = [] basenames = [] split_datablocks = [] for datablock in datablocks: for imageset in datablock.extract_imagesets(): paths = imageset.paths() for i in xrange(len(imageset)): subset = imageset[i:i + 1] split_datablocks.append( DataBlockFactory.from_imageset(subset)[0]) indices.append(i) basenames.append( os.path.splitext(os.path.basename(paths[i]))[0]) tags = [] for i, basename in zip(indices, basenames): if basenames.count(basename) > 1: tags.append("%s_%05d" % (basename, i)) else: tags.append(basename) # Wrapper function def do_work(i, item_list): processor = Processor(copy.deepcopy(params), composite_tag="%04d" % i) for item in item_list: try: for imageset in item[1].extract_imagesets(): update_geometry(imageset) except RuntimeError as e: logger.warning( "Error updating geometry on item %s, %s" % (str(item[0]), str(e))) continue if self.reference_detector is not None: from dxtbx.model import Detector for i in range(len(imageset)): imageset.set_detector(Detector.from_dict( self.reference_detector.to_dict()), index=i) processor.process_datablock(item[0], item[1]) processor.finalize() iterable = zip(tags, split_datablocks) else: basenames = [ os.path.splitext(os.path.basename(filename))[0] for filename in all_paths ] tags = [] for i, basename in enumerate(basenames): if basenames.count(basename) > 1: tags.append("%s_%05d" % (basename, i)) else: tags.append(basename) # Wrapper function def do_work(i, item_list): processor = Processor(copy.deepcopy(params), composite_tag="%04d" % i) for item in item_list: tag, filename = item datablock = do_import(filename) imagesets = datablock.extract_imagesets() if len(imagesets) == 0 or len(imagesets[0]) == 0: logger.info("Zero length imageset in file: %s" % filename) return if len(imagesets) > 1: raise Abort( "Found more than one imageset in file: %s" % filename) if len(imagesets[0]) > 1: raise Abort( "Found a multi-image file. Run again with pre_import=True" ) try: update_geometry(imagesets[0]) except RuntimeError as e: logger.warning( "Error updating geometry on item %s, %s" % (tag, str(e))) continue if self.reference_detector is not None: from dxtbx.model import Detector imagesets[0].set_detector( Detector.from_dict( self.reference_detector.to_dict())) processor.process_datablock(tag, datablock) processor.finalize() iterable = zip(tags, all_paths) # Process the data if params.mp.method == 'mpi': from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank( ) # each process in MPI has a unique id, 0-indexed size = comm.Get_size( ) # size: number of processes running in this job subset = [ item for i, item in enumerate(iterable) if (i + rank) % size == 0 ] do_work(rank, subset) else: from dxtbx.command_line.image_average import splitit if params.mp.nproc == 1: do_work(0, iterable) else: result = list( easy_mp.multi_core_run( myfunction=do_work, argstuples=list( enumerate(splitit(iterable, params.mp.nproc))), nproc=params.mp.nproc)) error_list = [r[2] for r in result] if error_list.count(None) != len(error_list): print( "Some processes failed excecution. Not all images may have processed. Error messages:" ) for error in error_list: if error is None: continue print(error) # Total Time logger.info("") logger.info("Total Time Taken = %f seconds" % (time() - st))
def run(self): '''Execute the script.''' from dials.util import log from time import time from libtbx import easy_mp import copy # Parse the command line params, options, all_paths = self.parser.parse_args(show_diff_phil=False, return_unhandled=True) # Check we have some filenames if not all_paths: self.parser.print_help() return # Save the options self.options = options self.params = params st = time() # Configure logging log.config( params.verbosity, info='dials.process.log', debug='dials.process.debug.log') # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) for abs_params in self.params.integration.absorption_correction: if abs_params.apply: if not (self.params.integration.debug.output and not self.params.integration.debug.separate_files): raise Sorry('Shoeboxes must be saved to integration intermediates to apply an absorption correction. '\ +'Set integration.debug.output=True and integration.debug.separate_files=False to save shoeboxes.') self.load_reference_geometry() from dials.command_line.dials_import import ManualGeometryUpdater update_geometry = ManualGeometryUpdater(params) # Import stuff logger.info("Loading files...") pre_import = params.dispatch.pre_import or len(all_paths) == 1 if pre_import: # Handle still imagesets by breaking them apart into multiple datablocks # Further handle single file still imagesets (like HDF5) by tagging each # frame using its index datablocks = [do_import(path) for path in all_paths] if self.reference_detector is not None: from dxtbx.model import Detector for datablock in datablocks: for imageset in datablock.extract_imagesets(): for i in range(len(imageset)): imageset.set_detector( Detector.from_dict(self.reference_detector.to_dict()), index=i) for datablock in datablocks: for imageset in datablock.extract_imagesets(): update_geometry(imageset) indices = [] basenames = [] split_datablocks = [] for datablock in datablocks: for imageset in datablock.extract_imagesets(): paths = imageset.paths() for i in xrange(len(imageset)): subset = imageset[i:i+1] split_datablocks.append(DataBlockFactory.from_imageset(subset)[0]) indices.append(i) basenames.append(os.path.splitext(os.path.basename(paths[i]))[0]) tags = [] for i, basename in zip(indices, basenames): if basenames.count(basename) > 1: tags.append("%s_%05d"%(basename, i)) else: tags.append(basename) # Wrapper function def do_work(item): Processor(copy.deepcopy(params)).process_datablock(item[0], item[1]) iterable = zip(tags, split_datablocks) else: basenames = [os.path.splitext(os.path.basename(filename))[0] for filename in all_paths] tags = [] for i, basename in enumerate(basenames): if basenames.count(basename) > 1: tags.append("%s_%05d"%(basename, i)) else: tags.append(basename) # Wrapper function def do_work(item): tag, filename = item datablock = do_import(filename) imagesets = datablock.extract_imagesets() if len(imagesets) == 0 or len(imagesets[0]) == 0: logger.info("Zero length imageset in file: %s"%filename) return if len(imagesets) > 1: raise Abort("Found more than one imageset in file: %s"%filename) if len(imagesets[0]) > 1: raise Abort("Found a multi-image file. Run again with pre_import=True") if self.reference_detector is not None: from dxtbx.model import Detector imagesets[0].set_detector(Detector.from_dict(self.reference_detector.to_dict())) update_geometry(imagesets[0]) Processor(copy.deepcopy(params)).process_datablock(tag, datablock) iterable = zip(tags, all_paths) # Process the data if params.mp.method == 'mpi': from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed size = comm.Get_size() # size: number of processes running in this job for i, item in enumerate(iterable): if (i+rank)%size == 0: do_work(item) else: easy_mp.parallel_map( func=do_work, iterable=iterable, processes=params.mp.nproc, method=params.mp.method, preserve_order=True, preserve_exception_message=True) # Total Time logger.info("") logger.info("Total Time Taken = %f seconds" % (time() - st))
def run(self): '''Execute the script.''' from dials.util import log from time import time from libtbx import easy_mp import copy # Parse the command line params, options, all_paths = self.parser.parse_args(show_diff_phil=False, return_unhandled=True) # Check we have some filenames if not all_paths: self.parser.print_help() return # Save the options self.options = options self.params = params st = time() # Configure logging log.config( params.verbosity, info='dials.process.log', debug='dials.process.debug.log') # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': logger.info('The following parameters have been modified:\n') logger.info(diff_phil) self.load_reference_geometry() from dials.command_line.dials_import import ManualGeometryUpdater update_geometry = ManualGeometryUpdater(params) # Import stuff logger.info("Loading files...") pre_import = params.dispatch.pre_import or len(all_paths) == 1 if pre_import: # Handle still imagesets by breaking them apart into multiple datablocks # Further handle single file still imagesets (like HDF5) by tagging each # frame using its index datablocks = [do_import(path) for path in all_paths] if self.reference_detector is not None: from dxtbx.model import Detector for datablock in datablocks: for imageset in datablock.extract_imagesets(): for i in range(len(imageset)): imageset.set_detector( Detector.from_dict(self.reference_detector.to_dict()), index=i) for datablock in datablocks: for imageset in datablock.extract_imagesets(): update_geometry(imageset) indices = [] basenames = [] split_datablocks = [] for datablock in datablocks: for imageset in datablock.extract_imagesets(): paths = imageset.paths() for i in xrange(len(imageset)): subset = imageset[i:i+1] split_datablocks.append(DataBlockFactory.from_imageset(subset)[0]) indices.append(i) basenames.append(os.path.splitext(os.path.basename(paths[i]))[0]) tags = [] for i, basename in zip(indices, basenames): if basenames.count(basename) > 1: tags.append("%s_%05d"%(basename, i)) else: tags.append(basename) # Wrapper function def do_work(item): Processor(copy.deepcopy(params)).process_datablock(item[0], item[1]) iterable = zip(tags, split_datablocks) else: basenames = [os.path.splitext(os.path.basename(filename))[0] for filename in all_paths] tags = [] for i, basename in enumerate(basenames): if basenames.count(basename) > 1: tags.append("%s_%05d"%(basename, i)) else: tags.append(basename) # Wrapper function def do_work(item): tag, filename = item datablock = do_import(filename) imagesets = datablock.extract_imagesets() if len(imagesets) == 0 or len(imagesets[0]) == 0: logger.info("Zero length imageset in file: %s"%filename) return if len(imagesets) > 1: raise Abort("Found more than one imageset in file: %s"%filename) if len(imagesets[0]) > 1: raise Abort("Found a multi-image file. Run again with pre_import=True") if self.reference_detector is not None: from dxtbx.model import Detector imagesets[0].set_detector(Detector.from_dict(self.reference_detector.to_dict())) update_geometry(imagesets[0]) Processor(copy.deepcopy(params)).process_datablock(tag, datablock) iterable = zip(tags, all_paths) # Process the data if params.mp.method == 'mpi': from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() # each process in MPI has a unique id, 0-indexed size = comm.Get_size() # size: number of processes running in this job for i, item in enumerate(iterable): if (i+rank)%size == 0: do_work(item) else: easy_mp.parallel_map( func=do_work, iterable=iterable, processes=params.mp.nproc, method=params.mp.method, preserve_order=True, preserve_exception_message=True) # Total Time logger.info("") logger.info("Total Time Taken = %f seconds" % (time() - st))
def process_event(self, run, timestamp): """ Process a single event from a run @param run psana run object @param timestamp psana timestamp object """ ts = cspad_tbx.evt_timestamp((timestamp.seconds(),timestamp.nanoseconds()/1e6)) if ts is None: print "No timestamp, skipping shot" return if len(self.params_cache.debug.event_timestamp) > 0 and ts not in self.params_cache.debug.event_timestamp: return if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events: if ts in self.known_events: if self.known_events[ts] not in ["stop", "done", "fail"]: if self.params_cache.debug.skip_bad_events: print "Skipping event %s: possibly caused an unknown exception previously"%ts return elif self.params_cache.debug.skip_processed_events: print "Skipping event %s: processed successfully previously"%ts return else: if self.params_cache.debug.skip_unprocessed_events: print "Skipping event %s: not processed previously"%ts return self.debug_start(ts) evt = run.event(timestamp) if evt.get("skip_event") or "skip_event" in [key.key() for key in evt.keys()]: print "Skipping event",ts self.debug_write("psana_skip", "skip") return print "Accepted", ts self.params = copy.deepcopy(self.params_cache) # the data needs to have already been processed and put into the event by psana if self.params.format.file_format == 'cbf': # get numpy array, 32x185x388 data = cspad_cbf_tbx.get_psana_corrected_data(self.psana_det, evt, use_default=False, dark=True, common_mode=self.common_mode, apply_gain_mask=self.params.format.cbf.gain_mask_value is not None, gain_mask_value=self.params.format.cbf.gain_mask_value, per_pixel_gain=False) if data is None: print "No data" self.debug_write("no_data", "skip") return if self.params.format.cbf.override_distance is None: distance = cspad_tbx.env_distance(self.params.input.address, run.env(), self.params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" self.debug_write("no_distance", "skip") return else: distance = self.params.format.cbf.override_distance if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" self.debug_write("no_wavelength", "skip") return else: wavelength = 12398.4187/self.params.format.cbf.override_energy if self.params.format.file_format == 'pickle': image_dict = evt.get(self.params.format.pickle.out_key) data = image_dict['DATA'] timestamp = t = ts s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23] print "Processing shot", s if self.params.format.file_format == 'cbf': # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data(self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address) if self.params.input.reference_geometry is not None: from dxtbx.model import Detector # copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead. cspad_img._detector_instance = Detector.from_dict(self.reference_detector.to_dict()) cspad_img.sync_detector_to_cbf() elif self.params.format.file_format == 'pickle': from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory cspad_img = FormatPYunspecifiedStillInMemory(image_dict) cspad_img.timestamp = s if self.params.dispatch.dump_all: self.save_image(cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s)) self.cache_ranges(cspad_img, self.params) imgset = MemImageSet([cspad_img]) if self.params.dispatch.estimate_gain_only: from dials.command_line.estimate_gain import estimate_gain estimate_gain(imgset) return if not self.params.dispatch.find_spots: self.debug_write("data_loaded", "done") return datablock = DataBlockFactory.from_imageset(imgset)[0] # before calling DIALS for processing, set output paths according to the templates if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template: self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.indexed_filename_template%("idx-" + s)) if "%s" in self.refined_experiments_filename_template: self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.refined_experiments_filename_template%("idx-" + s)) if "%s" in self.integrated_filename_template: self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.integrated_filename_template%("idx-" + s)) if "%s" in self.reindexedstrong_filename_template: self.params.output.reindexedstrong_filename = os.path.join(self.params.output.output_dir, self.reindexedstrong_filename_template%("idx-" + s)) # Load a dials mask from the trusted range and psana mask from dials.util.masking import MaskGenerator generator = MaskGenerator(self.params.border_mask) mask = generator.generate(imgset) if self.params.format.file_format == "cbf": mask = tuple([a&b for a, b in zip(mask,self.dials_mask)]) if self.spotfinder_mask is None: self.params.spotfinder.lookup.mask = mask else: self.params.spotfinder.lookup.mask = tuple([a&b for a, b in zip(mask,self.spotfinder_mask)]) if self.integration_mask is None: self.params.integration.lookup.mask = mask else: self.params.integration.lookup.mask = tuple([a&b for a, b in zip(mask,self.integration_mask)]) self.debug_write("spotfind_start") try: observed = self.find_spots(datablock) except Exception, e: import traceback; traceback.print_exc() print str(e), "event", timestamp self.debug_write("spotfinding_exception", "fail") return
def test_cspad_cbf_in_memory(self): from os.path import join, exists import os, dxtbx from uuid import uuid4 from dials.command_line.stills_process import phil_scope, Processor from libtbx.phil import parse from dxtbx.imageset import ImageSet, ImageSetData, MemReader, MemMasker from dxtbx.datablock import DataBlockFactory from dxtbx.format.FormatCBFCspad import FormatCBFCspadInMemory import cPickle as pickle dirname = 'tmp_%s' % uuid4().hex os.mkdir(dirname) os.chdir(dirname) assert exists(join(self.lcls_path, 'idx-20130301060858801.cbf')) f = open("process_lcls.phil", 'w') f.write(""" dispatch.squash_errors = False spotfinder { filter.min_spot_size=2 threshold.dispersion.gain=25 threshold.dispersion.global_threshold=100 } indexing { known_symmetry { space_group = P6122 unit_cell = 92.9 92.9 130.4 90 90 120 } refinement_protocol.d_min_start=1.7 stills.refine_candidates_with_known_symmetry=True } """) f.close() params = phil_scope.fetch( parse(file_name="process_lcls.phil")).extract() params.output.datablock_filename = None processor = Processor(params) mem_img = dxtbx.load(join(self.lcls_path, 'idx-20130301060858801.cbf')) raw_data = mem_img.get_raw_data( ) # cache the raw data to prevent swig errors mem_img = FormatCBFCspadInMemory(mem_img._cbf_handle) mem_img._raw_data = raw_data mem_img._cbf_handle = None # drop the file handle to prevent swig errors imgset = ImageSet( ImageSetData(MemReader([mem_img]), MemMasker([mem_img]))) imgset.set_beam(mem_img.get_beam()) imgset.set_detector(mem_img.get_detector()) datablock = DataBlockFactory.from_imageset(imgset)[0] processor.process_datablock("20130301060858801", datablock) # index/integrate the image result = "idx-20130301060858801_integrated.pickle" #n_refls = range(140,152) # large ranges to handle platform-specific differences # 09/20/17 Changes to still indexer: refine candidate basis vectors in target symmetry if supplied #n_refls = range(128,140) # large ranges to handle platform-specific differences # 09/27/17 Bugfix for refine_candidates_with_known_symmetry n_refls = range( 140, 152) # large ranges to handle platform-specific differences table = pickle.load(open(result, 'rb')) assert len(table) in n_refls, len(table) assert 'id' in table assert (table['id'] == 0).count(False) == 0 print 'OK'
if not fnames: exit() print fnames idx = 2 imgs = [] refls = [] for image_fname in fnames: loader = dxtbx.load(image_fname) img = loader.get_raw_data(idx).as_numpy_array() iset = loader.get_imageset(loader.get_image_file()) dblock = DataBlockFactory.from_imageset(iset[idx:idx + 1])[0] refl = flex.reflection_table.from_observations(dblock, find_spot_params) imgs.append(img) refls.append(refl) info_fname = image_fname.replace(".h5", ".pkl") sim_data = utils.open_flex(info_fname) orient = indexer_two_color( reflections=count_spots.as_single_shot_reflections(refl, inplace=False), imagesets=[iset], params=indexing_params) try:
def run(args, verbose=False): from libtbx.utils import Sorry try: from dials.array_family import flex except ImportError: return str(Sorry("DIALS is not configured")) from iotbx.phil import parse import os from spotfinder.servers import LoggingFramework from dials.array_family import flex from dxtbx.datablock import DataBlockFactory phil_scope = parse(""" file_name = None .type = str frame_number = None .type = int stats = True .type = bool include scope dials.algorithms.spot_finding.factory.phil_scope """, process_includes=True) #For the Apache server version, do not allow site, user, or dataset preferences #all parameters are to be passed in through the http: query line logfile = LoggingFramework() phil_objects = [] for key in args.keys(): arg = "%s=%s" % (key, args.get(key, "")) try: phil_objects.append(parse(arg)) except Exception: return str(Sorry("Unknown file or keyword: %s" % arg)) working_params = phil_scope.fetch(sources=phil_objects) params = working_params.extract() #working_params.show() if not os.path.isfile(params.file_name): return str(Sorry("%s is not a readable file" % params.file_name)) print "Image: %s\n" % params.file_name try: datablock = DataBlockFactory.from_filenames([params.file_name])[0] imageset = datablock.extract_imagesets()[0] if datablock.num_images() > 0 and params.frame_number is not None: print "Frame number", params.frame_number imageset = imageset[params.frame_number:params.frame_number + 1] datablock = DataBlockFactory.from_imageset(imageset)[0] reflections = flex.reflection_table.from_observations( datablock, params) if params.stats: from dials.algorithms.spot_finding.per_image_analysis import stats_single_image print stats_single_image(imageset, reflections, i=None, resolution_analysis=True, plot=False) except Exception: import traceback logger = StringIO.StringIO() logger.write("Sorry, can't process %s. Please contact authors.\n" % params.file_name) traceback.print_exc(file=logger) return str(Sorry(logger.getvalue())) + logfile.getvalue() print "Found %d strong reflections" % len(reflections) return logfile.getvalue()
imgAB = loader.get_raw_data(2).as_numpy_array() ##################################### # format of this special imageset # its 4 images, # 0th is simulated colorA, # 1st is simulated colorB # 2nd is two color, # 3rd is the data image that was indexed # we will grab just the first 3 simulated # images and find spots.. xdata, ydata, _ = map(np.array, spot_utils.xyz_from_refl(data['refl'])) iset = loader.get_imageset(loader.get_image_file()) dblockA = DataBlockFactory.from_imageset(iset[0:1])[0] dblockB = DataBlockFactory.from_imageset(iset[1:2])[0] dblockAB = DataBlockFactory.from_imageset(iset[2:3])[0] reflA = flex.reflection_table.from_observations(dblockA, find_spot_params) reflB = flex.reflection_table.from_observations(dblockB, find_spot_params) reflAB = flex.reflection_table.from_observations(dblockAB, find_spot_params) refl_dat = data[ 'refl'] # experimental image observations are stored here.. # adhoc thresholds: threshA = 0 #imgA[ imgA > 0].mean() * 0.05 threshB = 0 #imgB[ imgB > 0].mean() * 0.05 threshAB = 0 #imgAB[ imgAB > 0].mean() * 0.05
def run(): parser = OptionParser(phil=phil_scope) params, options = parser.parse_args(show_diff_phil=True) assert params.input.single_img is not None assert params.output_dir is not None # load the image img = dxtbx.load(params.input.single_img) imgset = MemImageSet([img]) datablock = DataBlockFactory.from_imageset(imgset)[0] spotfinder = SpotFinderFactory.from_parameters(params) reflections = spotfinder(datablock) base_name = os.path.splitext(params.input.single_img)[0] reflections.as_pickle( os.path.join(params.output_dir, base_name + "_strong.pickle")) # DGW commented out as reflections.minimum_number_of_reflections no longer exists # if len(reflections) < params.refinement.reflections.minimum_number_of_reflections: # print "Not enough spots to index" # return # create the spot finder print("Spotfinder spots found:", len(reflections)) if params.indexing.method == "fft3d": from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer elif params.indexing.method == "fft1d": from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer elif params.method == "real_space_grid_search": from dials.algorithms.indexing.real_space_grid_search import ( indexer_real_space_grid_search as indexer, ) try: idxr = indexer(reflections, [imgset], params=params.indexing) except (RuntimeError, Sorry) as e: print(str(e)) return indexed = idxr.refined_reflections experiments = idxr.refined_experiments # from dxtbx.model.experiment.experiment_list import ExperimentListDumper # dump = ExperimentListDumper(experiments) # dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json")) indexed.as_pickle( os.path.join(params.output_dir, base_name + "_indexed.pickle")) refiner = RefinerFactory.from_parameters_data_experiments( params, indexed, experiments) refiner.run() refined_experiments = refiner.get_experiments() # dump = ExperimentListDumper(refined_experiments) # dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json")) # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator reference = indexed reference = process_reference(reference) profile_model = ProfileModelFactory.create(params, refined_experiments, reference) predicted = flex.reflection_table.from_predictions_multi( refined_experiments, dmin=params.prediction.dmin, dmax=params.prediction.dmax, margin=params.prediction.margin, force_static=params.prediction.force_static, ) predicted.match_with_reference(reference) integrator = IntegratorFactory.create(params, experiments, profile_model, predicted) # Integrate the reflections integrated = integrator.integrate() integrated.as_pickle( os.path.join(params.output_dir, base_name + "_integrated.pickle"))
if idx in weak_shots and skip_weak: print("Skipping weak shots %d" % idx) continue if idx in failed_shots and skip_failed: print("Skipping failed idx shots %d" % idx) continue if idx in indexed_shots and skip_indexed: print("Skipping already idx shots %d" % idx) continue iset = IMGSET[idx:idx + 1] iset.set_detector(DET) iset.set_beam(BEAM) #detector = iset.get_detector(0) dblock = DataBlockFactory.from_imageset(iset)[0] refls_strong = flex.reflection_table.from_observations(dblock, spot_par) if len(refls_strong) < 10: print("Not enough spots shot %d, continuing!" % idx) weak_shots.append(idx) try: np.savetxt(weak_shots_f, weak_shots, fmt="%d") except: pass continue waveA = parameters.ENERGY_CONV / ENERGIES[0] waveB = parameters.ENERGY_CONV / ENERGIES[1] beamA = deepcopy(iset.get_beam())
def run(): parser = OptionParser( phil = phil_scope) params, options = parser.parse_args(show_diff_phil=True) assert params.input.single_img is not None assert params.output_dir is not None # load the image img = dxtbx.load(params.input.single_img) imgset = MemImageSet([img]) datablock = DataBlockFactory.from_imageset(imgset)[0] spotfinder = SpotFinderFactory.from_parameters(params) reflections = spotfinder(datablock) base_name = os.path.splitext(params.input.single_img)[0] reflections.as_pickle(os.path.join(params.output_dir, base_name + "_strong.pickle")) # DGW commented out as reflections.minimum_number_of_reflections no longer exists #if len(reflections) < params.refinement.reflections.minimum_number_of_reflections: # print "Not enough spots to index" # return # create the spot finder print "Spotfinder spots found:", len(reflections) if params.indexing.method == "fft3d": from dials.algorithms.indexing.fft3d import indexer_fft3d as indexer elif params.indexing.method == "fft1d": from dials.algorithms.indexing.fft1d import indexer_fft1d as indexer elif params.method == "real_space_grid_search": from dials.algorithms.indexing.real_space_grid_search \ import indexer_real_space_grid_search as indexer try: idxr = indexer(reflections, [imgset], params=params.indexing) except (RuntimeError, Sorry) as e: print str(e) return indexed = idxr.refined_reflections experiments = idxr.refined_experiments #from dxtbx.model.experiment.experiment_list import ExperimentListDumper #dump = ExperimentListDumper(experiments) #dump.as_json(os.path.join(params.output_dir, base_name + "_experiments.json")) indexed.as_pickle(os.path.join(params.output_dir, base_name + "_indexed.pickle")) refiner = RefinerFactory.from_parameters_data_experiments( params, indexed, experiments) refiner.run() refined_experiments = refiner.get_experiments() #dump = ExperimentListDumper(refined_experiments) #dump.as_json(os.path.join(params.output_dir, base_name + "_refined.json")) # Compute the profile model # Predict the reflections # Match the predictions with the reference # Create the integrator reference = indexed reference = process_reference(reference) profile_model = ProfileModelFactory.create(params, refined_experiments, reference) predicted = flex.reflection_table.from_predictions_multi( refined_experiments, dmin=params.prediction.dmin, dmax=params.prediction.dmax, margin=params.prediction.margin, force_static=params.prediction.force_static) predicted.match_with_reference(reference) integrator = IntegratorFactory.create(params, experiments, profile_model, predicted) # Integrate the reflections integrated = integrator.integrate() integrated.as_pickle(os.path.join(params.output_dir, base_name + "_integrated.pickle"))
from dxtbx.datablock import DataBlockFactory, DataBlockDumper from dxtbx.imageset import ImageSetFactory, ImageSweep, ImageSetData from dxtbx.model.goniometer import GoniometerFactory from dxtbx.model.scan import ScanFactory import glob, os, sys """ Modification of AB's prepare_sweep.py. Usage: libtbx.python prepare_sweep.py img_dir start_img, end_img savefile """ root = sys.argv[1] start, end = int(sys.argv[2]), int(sys.argv[3]) g = GoniometerFactory.single_axis() s = ScanFactory.make_scan((start, end), 0, (0, 1), [0] * (end - start + 1)) sw = ImageSetFactory.from_template(template=os.path.join( root, "fft_frame_I_mf_####.cbf"), scan=s, goniometer=g, image_range=(start, end)) dump = DataBlockDumper(DataBlockFactory.from_imageset(sw)) dump.as_file(sys.argv[4])
def process_event(self, run, timestamp): """ Process a single event from a run @param run psana run object @param timestamp psana timestamp object """ ts = cspad_tbx.evt_timestamp((timestamp.seconds(),timestamp.nanoseconds()/1e6)) if ts is None: print "No timestamp, skipping shot" return if len(self.params_cache.debug.event_timestamp) > 0 and ts not in self.params_cache.debug.event_timestamp: return if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events: if ts in self.known_events: if self.known_events[ts] == "unknown": if self.params_cache.debug.skip_bad_events and self.known_events[ts] == "unknown": print "Skipping event %s: possibly caused an unknown exception previously"%ts return elif self.params_cache.debug.skip_processed_events: print "Skipping event %s: processed successfully previously"%ts return else: if self.params_cache.debug.skip_unprocessed_events: print "Skipping event %s: not processed previously"%ts return print "Accepted", ts self.debug_file_handle.write("%s,%s"%(socket.gethostname(), ts)) self.params = copy.deepcopy(self.params_cache) evt = run.event(timestamp) id = evt.get(psana.EventId) if evt.get("skip_event"): print "Skipping event",id self.debug_file_handle.write(",psana_skip\n") return # the data needs to have already been processed and put into the event by psana if self.params.format.file_format == 'cbf': # get numpy array, 32x185x388 data = self.psana_det.calib(evt) # applies psana's complex run-dependent calibrations if data is None: print "No data" self.debug_file_handle.write(",no_data\n") return if self.params.format.cbf.gain_mask_value is not None: # apply gain mask data *= self.gain_mask distance = cspad_tbx.env_distance(self.params.input.address, run.env(), self.params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" self.debug_file_handle.write(",no_distance\n") return if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" self.debug_file_handle.write(",no_wavelength\n") return else: wavelength = 12398.4187/self.params.format.cbf.override_energy if self.params.format.file_format == 'pickle': image_dict = evt.get(self.params.format.pickle.out_key) data = image_dict['DATA'] timestamp = t = ts s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[20:23] print "Processing shot", s if self.params.format.file_format == 'cbf': # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data(self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address) elif self.params.format.file_format == 'pickle': from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory cspad_img = FormatPYunspecifiedStillInMemory(image_dict) cspad_img.timestamp = s if self.params.dispatch.dump_all: self.save_image(cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s)) self.cache_ranges(cspad_img, self.params) imgset = MemImageSet([cspad_img]) datablock = DataBlockFactory.from_imageset(imgset)[0] # before calling DIALS for processing, set output paths according to the templates if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template: self.params.output.indexed_filename = os.path.join(self.params.output.output_dir, self.indexed_filename_template%("idx-" + s)) if "%s" in self.refined_experiments_filename_template: self.params.output.refined_experiments_filename = os.path.join(self.params.output.output_dir, self.refined_experiments_filename_template%("idx-" + s)) if "%s" in self.integrated_filename_template: self.params.output.integrated_filename = os.path.join(self.params.output.output_dir, self.integrated_filename_template%("idx-" + s)) # if border is requested, generate a border only mask if self.params.border_mask.border > 0: from dials.command_line.generate_mask import MaskGenerator generator = MaskGenerator(self.params.border_mask) mask = generator.generate(imgset) self.params.spotfinder.lookup.mask = mask try: observed = self.find_spots(datablock) except Exception, e: import traceback; traceback.print_exc() print str(e), "event", timestamp self.debug_file_handle.write(",spotfinding_exception\n") return
def run(self): '''Execute the script.''' from dials.util import log from logging import info from time import time from libtbx.utils import Abort from libtbx import easy_mp import os, copy from dxtbx.datablock import DataBlockFactory # Parse the command line params, options, all_paths = self.parser.parse_args(show_diff_phil=False, return_unhandled=True) # Check we have some filenames if len(all_paths) == 0: self.parser.print_help() return # Save the options self.options = options self.params = params st = time() # Configure logging log.config( params.verbosity, info='dials.process.log', debug='dials.process.debug.log') # Log the diff phil diff_phil = self.parser.diff_phil.as_str() if diff_phil is not '': info('The following parameters have been modified:\n') info(diff_phil) # Import stuff info("Loading files...") if len(all_paths) == 1: datablocks = DataBlockFactory.from_filenames(all_paths) else: def do_import(filename): info("Loading %s"%os.path.basename(filename)) datablocks = DataBlockFactory.from_filenames([filename]) if len(datablocks) == 0: raise Abort("Could not load %s"%filename) if len(datablocks) > 1: raise Abort("Got multiple datablocks from file %s"%filename) return datablocks[0] datablocks = easy_mp.parallel_map( func=do_import, iterable=all_paths, processes=params.mp.nproc, method=params.mp.method, preserve_order=True, preserve_exception_message=True) if len(datablocks) == 0: raise Abort('No datablocks specified') # Handle still imagesets by breaking them apart into multiple datablocks # Further handle single file still imagesets (like HDF5) by tagging each # frame using its index indices = [] basenames = [] split_datablocks = [] for datablock in datablocks: for imageset in datablock.extract_imagesets(): for i in xrange(len(imageset)): subset = imageset[i:i+1] split_datablocks.append(DataBlockFactory.from_imageset(subset)[0]) indices.append(i) basenames.append(os.path.splitext(os.path.basename(subset.paths()[0]))[0]) tags = [] for i, basename in zip(indices, basenames): if basenames.count(basename) > 1: tags.append("%s_%d"%(basename, i)) else: tags.append(basename) # Wrapper function def do_work(item): Processor(copy.deepcopy(params)).process_datablock(item[0], item[1]) # Process the data easy_mp.parallel_map( func=do_work, iterable=zip(tags, split_datablocks), processes=params.mp.nproc, method=params.mp.method, preserve_order=True, preserve_exception_message=True) # Total Time info("") info("Total Time Taken = %f seconds" % (time() - st))
def process_event(self, run, timestamp): """ Process a single event from a run @param run psana run object @param timestamp psana timestamp object """ ts = cspad_tbx.evt_timestamp( (timestamp.seconds(), timestamp.nanoseconds() / 1e6)) if ts is None: print "No timestamp, skipping shot" return if len(self.params_cache.debug.event_timestamp ) > 0 and ts not in self.params_cache.debug.event_timestamp: return if self.params_cache.debug.skip_processed_events or self.params_cache.debug.skip_unprocessed_events or self.params_cache.debug.skip_bad_events: if ts in self.known_events: if self.known_events[ts] not in ["stop", "done", "fail"]: if self.params_cache.debug.skip_bad_events: print "Skipping event %s: possibly caused an unknown exception previously" % ts return elif self.params_cache.debug.skip_processed_events: print "Skipping event %s: processed successfully previously" % ts return else: if self.params_cache.debug.skip_unprocessed_events: print "Skipping event %s: not processed previously" % ts return self.debug_start(ts) evt = run.event(timestamp) if evt.get("skip_event") or "skip_event" in [ key.key() for key in evt.keys() ]: print "Skipping event", ts self.debug_write("psana_skip", "skip") return print "Accepted", ts self.params = copy.deepcopy(self.params_cache) # the data needs to have already been processed and put into the event by psana if self.params.format.file_format == 'cbf': # get numpy array, 32x185x388 data = cspad_cbf_tbx.get_psana_corrected_data( self.psana_det, evt, use_default=False, dark=True, common_mode=self.common_mode, apply_gain_mask=self.params.format.cbf.gain_mask_value is not None, gain_mask_value=self.params.format.cbf.gain_mask_value, per_pixel_gain=False) if data is None: print "No data" self.debug_write("no_data", "skip") return if self.params.format.cbf.override_distance is None: distance = cspad_tbx.env_distance( self.params.input.address, run.env(), self.params.format.cbf.detz_offset) if distance is None: print "No distance, skipping shot" self.debug_write("no_distance", "skip") return else: distance = self.params.format.cbf.override_distance if self.params.format.cbf.override_energy is None: wavelength = cspad_tbx.evt_wavelength(evt) if wavelength is None: print "No wavelength, skipping shot" self.debug_write("no_wavelength", "skip") return else: wavelength = 12398.4187 / self.params.format.cbf.override_energy if self.params.format.file_format == 'pickle': image_dict = evt.get(self.params.format.pickle.out_key) data = image_dict['DATA'] timestamp = t = ts s = t[0:4] + t[5:7] + t[8:10] + t[11:13] + t[14:16] + t[17:19] + t[ 20:23] print "Processing shot", s if self.params.format.file_format == 'cbf': # stitch together the header, data and metadata into the final dxtbx format object cspad_img = cspad_cbf_tbx.format_object_from_data( self.base_dxtbx, data, distance, wavelength, timestamp, self.params.input.address) if self.params.input.reference_geometry is not None: from dxtbx.model import Detector # copy.deep_copy(self.reference_detctor) seems unsafe based on tests. Use from_dict(to_dict()) instead. cspad_img._detector_instance = Detector.from_dict( self.reference_detector.to_dict()) cspad_img.sync_detector_to_cbf() elif self.params.format.file_format == 'pickle': from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStillInMemory cspad_img = FormatPYunspecifiedStillInMemory(image_dict) cspad_img.timestamp = s if self.params.dispatch.dump_all: self.save_image( cspad_img, self.params, os.path.join(self.params.output.output_dir, "shot-" + s)) self.cache_ranges(cspad_img, self.params) imgset = MemImageSet([cspad_img]) if self.params.dispatch.estimate_gain_only: from dials.command_line.estimate_gain import estimate_gain estimate_gain(imgset) return if not self.params.dispatch.find_spots: self.debug_write("data_loaded", "done") return datablock = DataBlockFactory.from_imageset(imgset)[0] # before calling DIALS for processing, set output paths according to the templates if self.indexed_filename_template is not None and "%s" in self.indexed_filename_template: self.params.output.indexed_filename = os.path.join( self.params.output.output_dir, self.indexed_filename_template % ("idx-" + s)) if "%s" in self.refined_experiments_filename_template: self.params.output.refined_experiments_filename = os.path.join( self.params.output.output_dir, self.refined_experiments_filename_template % ("idx-" + s)) if "%s" in self.integrated_filename_template: self.params.output.integrated_filename = os.path.join( self.params.output.output_dir, self.integrated_filename_template % ("idx-" + s)) if "%s" in self.reindexedstrong_filename_template: self.params.output.reindexedstrong_filename = os.path.join( self.params.output.output_dir, self.reindexedstrong_filename_template % ("idx-" + s)) # Load a dials mask from the trusted range and psana mask from dials.util.masking import MaskGenerator generator = MaskGenerator(self.params.border_mask) mask = generator.generate(imgset) if self.params.format.file_format == "cbf": mask = tuple([a & b for a, b in zip(mask, self.dials_mask)]) if self.spotfinder_mask is None: self.params.spotfinder.lookup.mask = mask else: self.params.spotfinder.lookup.mask = tuple( [a & b for a, b in zip(mask, self.spotfinder_mask)]) if self.integration_mask is None: self.params.integration.lookup.mask = mask else: self.params.integration.lookup.mask = tuple( [a & b for a, b in zip(mask, self.integration_mask)]) self.debug_write("spotfind_start") try: observed = self.find_spots(datablock) except Exception, e: import traceback traceback.print_exc() print str(e), "event", timestamp self.debug_write("spotfinding_exception", "fail") return