def do_import(filename): logger.info("Loading %s" % os.path.basename(filename)) datablocks = DataBlockFactory.from_filenames([filename]) if len(datablocks) == 0: try: datablocks = DataBlockFactory.from_json_file(filename) except ValueError: raise Abort("Could not load %s" % filename) if len(datablocks) == 0: raise Abort("Could not load %s" % filename) if len(datablocks) > 1: raise Abort("Got multiple datablocks from file %s" % filename) # Ensure the indexer and downstream applications treat this as set of stills reset_sets = [] from dxtbx.imageset import ImageSetFactory for imageset in datablocks[0].extract_imagesets(): imageset = ImageSetFactory.imageset_from_anyset(imageset) imageset.set_scan(None) imageset.set_goniometer(None) reset_sets.append(imageset) return DataBlockFactory.from_imageset(reset_sets)[0]
def test_cbf_writer(image_file, dials_regression, run_in_tmpdir): filename = os.path.join(dials_regression, image_file) datablock = DataBlockFactory.from_filenames([filename])[0] imageset = datablock.extract_imagesets()[0] FormatCBFMini.as_file( imageset.get_detector(), imageset.get_beam(), imageset.get_goniometer(), imageset.get_scan(), imageset.get_raw_data(0)[0], "image_0001.cbf", ) assert datablock.format_class() datablock2 = DataBlockFactory.from_filenames(["image_0001.cbf"])[0] imageset2 = datablock2.extract_imagesets()[0] tolerance = tolerance_phil_scope.extract().tolerance diff = SweepDiff(tolerance) print("\n".join(diff(imageset, imageset2))) assert BeamComparison()(imageset.get_beam(), imageset2.get_beam()) assert DetectorComparison(origin_tolerance=tolerance.detector.origin)( imageset.get_detector(), imageset2.get_detector()) assert GoniometerComparison()(imageset.get_goniometer(), imageset2.get_goniometer()) s1 = imageset.get_scan() s2 = imageset.get_scan() assert s1.get_exposure_times() == s2.get_exposure_times() assert s1.get_oscillation() == s2.get_oscillation() assert s1.get_image_range() == s2.get_image_range() assert imageset.get_raw_data(0) == imageset2.get_raw_data(0)
def tst_json(self): filenames = self.multiple_block_filenames() blocks1 = DataBlockFactory.from_filenames(filenames) blocks2 = self.encode_json_then_decode(blocks1) assert(len(blocks2) == len(blocks1)) for b1, b2 in zip(blocks1, blocks2): assert(b1.format_class() == b2.format_class()) assert(b1 == b2) assert(blocks1 == blocks2) filenames = self.multiple_block_filenames() blocks1 = DataBlockFactory.from_filenames(filenames) blocks2 = self.encode_json_then_decode(blocks1, check_format=False) assert(len(blocks2) == len(blocks1)) for b1, b2 in zip(blocks1, blocks2): for im1, im2 in zip(b1.extract_imagesets(), b2.extract_imagesets()): assert(len(im1) == len(im2)) if isinstance(im1, ImageSweep): assert(isinstance(im2, ImageSweep)) assert(im1.get_beam() == im2.get_beam()) assert(im1.get_detector() == im2.get_detector()) assert(im1.get_goniometer() == im2.get_goniometer()) assert(im1.get_scan() == im2.get_scan()) else: assert(not isinstance(im2, ImageSweep)) for i in xrange(len(im1)): assert(im1.get_beam(i) == im2.get_beam(i)) assert(im1.get_detector(i) == im2.get_detector(i)) print 'OK'
def test_translate(dials_regression, tmpdir): """Test as written in https://github.com/dials/dials/issues/471. This is pretty slow!""" tmpdir.chdir() dials_regression = "~/sw/cctbx/modules/dials_regression" # use the i04_weak_data for this test data_dir = os.path.join(dials_regression, "image_examples", "DLS_I04") image_path = os.path.join(data_dir, "grid_full_cbf_0005.cbf") # Generate distortion maps cmd = ("dials.generate_distortion_maps {0} " "dx=1 dy=2").format(image_path) result = easy_run.fully_buffered(command=cmd).raise_if_errors() # Import without correction cmd = ("dials.import {0}").format(image_path) result = easy_run.fully_buffered(command=cmd).raise_if_errors() db1 = DataBlockFactory.from_serialized_format('datablock.json')[0] det1 = db1.unique_detectors()[0] # Import with correction cmd = ("dials.import {0} dx=dx.pickle dy=dy.pickle " "output.datablock=corrected_datablock.json").format(image_path) result = easy_run.fully_buffered(command=cmd).raise_if_errors() db2 = DataBlockFactory.from_serialized_format( 'corrected_datablock.json')[0] det2 = db2.unique_detectors()[0] # FIXME, why doesn't db2 have dx, dy set? assert db2.extract_imagesets()[0].external_lookup.dx.filename
def tst_json(self): from dxtbx.datablock import DataBlockFactory from dxtbx.imageset import ImageSweep filenames = self.multiple_block_filenames() blocks1 = DataBlockFactory.from_filenames(filenames) blocks2 = self.encode_json_then_decode(blocks1) assert(len(blocks2) == len(blocks1)) for b1, b2 in zip(blocks1, blocks2): assert(b1.format_class() == b2.format_class()) assert(b1 == b2) assert(blocks1 == blocks2) filenames = self.multiple_block_filenames() blocks1 = DataBlockFactory.from_filenames(filenames) blocks2 = self.encode_json_then_decode(blocks1, check_format=False) assert(len(blocks2) == len(blocks1)) for b1, b2 in zip(blocks1, blocks2): for im1, im2 in zip(b1.extract_imagesets(), b2.extract_imagesets()): assert(len(im1) == len(im2)) if isinstance(im1, ImageSweep): assert(isinstance(im2, ImageSweep)) assert(im1.get_beam() == im2.get_beam()) assert(im1.get_detector() == im2.get_detector()) assert(im1.get_goniometer() == im2.get_goniometer()) assert(im1.get_scan() == im2.get_scan()) else: assert(not isinstance(im2, ImageSweep)) for i in xrange(len(im1)): assert(im1.get_beam(i) == im2.get_beam(i)) assert(im1.get_detector(i) == im2.get_detector(i)) print 'OK'
def test_split_single_image_datablock(dials_regression, tmpdir): tmpdir.chdir() pytest.importorskip("h5py") sacla_file = os.path.join( dials_regression, "image_examples", "SACLA_MPCCD_Cheetah", "run266702-0-subset.h5", ) db = DataBlockFactory.from_filenames([sacla_file])[0] assert db.num_images() == 4 imageset = db.extract_imagesets()[0] subset = imageset[2:3] subblock = DataBlockFactory.from_imageset(subset)[0] assert subblock.num_images() == 1 assert get_indices(subblock) == [2] dumped_filename = "split_datablock.json" dump = DataBlockDumper(subblock) dump.as_json(dumped_filename) db = DataBlockFactory.from_json_file(dumped_filename, check_format=True)[0] assert db.num_images() == 1 assert get_indices(db) == [2] db = DataBlockFactory.from_json_file(dumped_filename, check_format=False)[0] assert db.num_images() == 1 assert get_indices(db) == [2]
def loader(x): try: obj = DataBlockFactory.from_filenames([x])[0].extract_imagesets()[0] except IndexError: import dxtbx.datablock try: obj = DataBlockFactory.from_json_file(x)[0].extract_imagesets()[0] except dxtbx.datablock.InvalidDataBlockError: obj = ExperimentListFactory.from_json_file(x)[0].imageset return obj
def do_import(filename): logger.info("Loading %s" % os.path.basename(filename)) try: datablocks = DataBlockFactory.from_json_file(filename) except ValueError: datablocks = DataBlockFactory.from_filenames([filename]) if len(datablocks) == 0: raise Abort("Could not load %s" % filename) if len(datablocks) > 1: raise Abort("Got multiple datablocks from file %s" % filename) return datablocks[0]
def test_split_single_image_datablock(dials_data, tmpdir): tmpdir.chdir() pytest.importorskip("h5py") sacla_file = os.path.join( dials_data("image_examples"), "SACLA-MPCCD-run266702-0-subset.h5", ) db = DataBlockFactory.from_filenames([sacla_file])[0] assert db.num_images() == 4 imageset = db.extract_imagesets()[0] subset = imageset[2:3] subblock = DataBlockFactory.from_imageset(subset)[0] assert subblock.num_images() == 1 assert get_indices(subblock) == [2]
def test_combination_of_multiple_datablocks_and_strong_spots_files( dials_regression, tmpdir): tmpdir.chdir() path = os.path.join(dials_regression, "centroid_test_data/centroid_####.cbf") # example combined two different spot-finding settings for the same dataset # e.d. for comparison with the reciprocal lattice viewer. cmd = "dials.import template={0}".format(path) result = easy_run.fully_buffered(cmd).raise_if_errors() cmd = "dials.find_spots datablock.json output.reflections=strong1.pickle" result = easy_run.fully_buffered(cmd).raise_if_errors() cmd = ("dials.find_spots datablock.json sigma_strong=5 " "output.reflections=strong2.pickle") result = easy_run.fully_buffered(cmd).raise_if_errors() cmd = ("dev.dials.combine_datablocks datablock.json datablock.json " "strong1.pickle strong2.pickle") result = easy_run.fully_buffered(cmd).raise_if_errors() # load results comb_db = DataBlockFactory.from_json_file('combined_datablocks.json')[0] comb_strong = flex.reflection_table.from_pickle("combined_strong.pickle") # load reference models and reflections db = DataBlockFactory.from_json_file('datablock.json')[0] ref_detector = db.unique_detectors()[0] ref_beam = db.unique_beams()[0] ref_scan = db.unique_scans()[0] ref_goniometer = db.unique_goniometers()[0] strong1 = flex.reflection_table.from_pickle("strong1.pickle") strong2 = flex.reflection_table.from_pickle("strong2.pickle") # check the models have not been modified for imset in comb_db.extract_imagesets(): assert imset.get_detector() == ref_detector assert imset.get_beam() == ref_beam assert imset.get_scan() == ref_scan assert imset.get_goniometer() == ref_goniometer # check the reflections are unaffected, except for the change in id s1 = comb_strong.select(comb_strong['id'] == 0) s2 = comb_strong.select(comb_strong['id'] == 1) s2['id'] = flex.size_t(len(s2), 0) for r1, r2 in zip(s1, strong1): assert r1 == r2 for r1, r2 in zip(s2, strong2): assert r1 == r2
def to_datablocks(self): ''' Return the experiment list as a datablock list. This assumes that the experiment contains 1 datablock.''' from dxtbx.datablock import DataBlockFactory # Convert the experiment list to dict obj = self.to_dict() # Convert the dictionary to a datablock dictionary obj['__id__'] = 'DataBlock' for e in obj['experiment']: iid = e['imageset'] imageset = obj['imageset'][iid] if 'beam' in e: imageset['beam'] = e['beam'] if 'detector' in e: imageset['detector'] = e['detector'] if 'goniometer' in e: imageset['goniometer'] = e['goniometer'] if 'scan' in e: imageset['scan'] = e['scan'] # Remove the experiments del obj['experiment'] # Create the datablock return DataBlockFactory.from_dict([obj])
def exercise_one_image(path, count_only_shadow, count_mask_shadow, count_mask_no_shadow): from dxtbx.datablock import DataBlockFactory assert os.path.exists(path), path for shadowing in (libtbx.Auto, True, False): format_kwargs = {'dynamic_shadowing': shadowing} datablock = DataBlockFactory.from_filenames( [path], format_kwargs=format_kwargs)[0] imageset = datablock.extract_imagesets()[0] detector = imageset.get_detector() scan = imageset.get_scan() filename = imageset.get_path(0) masker = imageset.masker().format_class( filename, **format_kwargs).get_goniometer_shadow_masker() assert masker is not None mask = masker.get_mask(detector, scan_angle=scan.get_oscillation()[0]) assert len(mask) == len(detector) # only shadowed pixels masked assert mask[0].count(False) == count_only_shadow, ( mask[0].count(False), count_only_shadow) mask = imageset.get_mask(0) # dead pixels, pixels in gaps, etc also masked if shadowing is libtbx.Auto or shadowing is True: assert mask[0].count(False) == count_mask_shadow, ( mask[0].count(False), count_mask_shadow) else: assert mask[0].count(False) == count_mask_no_shadow, ( mask[0].count(False), count_mask_no_shadow)
def run(self): params, options = self.parser.parse_args(show_diff_phil=True) assert params.input.single_img is not None filebase = os.path.splitext(params.input.single_img)[0] for item in dir(params.output): value = getattr(params.output, item) try: if "%s" in value: setattr(params.output, item, value % filebase) except Exception: pass self.params = params self.options = options # load the image img = dxtbx.load(params.input.single_img) imgset = MemImageSet([img]) datablock = DataBlockFactory.from_imageset(imgset)[0] # Cannot export MemImageSets # if self.params.output.datablock_filename: # from dxtbx.datablock import DataBlockDumper # dump = DataBlockDumper(datablock) # dump.as_json(self.params.output.datablock_filename) observed = self.find_spots(datablock) experiments, indexed = self.index(datablock, observed) experiments = self.refine(experiments, indexed) integrated = self.integrate(experiments, indexed)
def run(self): from os.path import join, exists from libtbx import easy_run import os input_filename = join(self.path, "datablock.json") output_filename = "output_datablock.json" mask_filename = join(self.path, "lookup_mask.pickle") easy_run.fully_buffered( ['dials.apply_mask', 'input.datablock=%s' % input_filename, 'input.mask=%s' % mask_filename, 'output.datablock=%s' % output_filename]).raise_if_errors() from dxtbx.datablock import DataBlockFactory datablocks = DataBlockFactory.from_json_file(output_filename) assert len(datablocks) == 1 imagesets = datablocks[0].extract_imagesets() assert len(imagesets) == 1 imageset = imagesets[0] assert imageset.external_lookup.mask.filename == mask_filename print 'OK'
def from_filenames( filenames, verbose=False, unhandled=None, compare_beam=None, compare_detector=None, compare_goniometer=None, scan_tolerance=None, format_kwargs=None, load_models=True, ): """Create a list of data blocks from a list of directory or file names.""" experiments = ExperimentList() for db in DataBlockFactory.from_filenames( filenames, verbose=verbose, unhandled=unhandled, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs, ): experiments.extend( ExperimentListFactory.from_datablock_and_crystal(db, None, load_models) ) return experiments
def test_experimentlist_factory_from_datablock(): filenames = ["filename_%01d.cbf" % (i + 1) for i in range(0, 2)] imageset = Format.get_imageset( filenames, beam=Beam(), detector=Detector(), goniometer=Goniometer(), scan=Scan((1, 2), (0, 1)), as_sequence=True, ) crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol="P1") datablock = DataBlockFactory.from_imageset(imageset) assert datablock[0].format_class() experiments = ExperimentListFactory.from_datablock_and_crystal( datablock, crystal) assert len(experiments) == 1 assert experiments[0].imageset assert experiments[0].beam assert experiments[0].detector is not None assert experiments[0].goniometer assert experiments[0].scan assert experiments[0].crystal
def run(args=None): args = args or sys.argv[1:] user_phil = [] files = [] for arg in args: if os.path.isfile(arg): files.append(arg) else: try: user_phil.append(parse(arg)) except Exception: raise Sorry("Unrecognized argument %s" % arg) params = phil_scope.fetch(sources=user_phil).extract() fig = plt.figure() colormap = plt.cm.gist_ncar colors = [colormap(i) for i in np.linspace(0, 0.9, len(files))] for file_name, color in zip(files, colors): # read the data and get the detector models try: datablocks = DataBlockFactory.from_json_file(file_name, check_format=False) detectors = sum((db.unique_detectors() for db in datablocks), []) except Exception: try: experiments = ExperimentListFactory.from_json_file( file_name, check_format=False) except ValueError: experiments = ExperimentListFactory.from_filenames([file_name]) detectors = experiments.detectors() if not params.plot_all_detectors: detectors = detectors[0:1] for detector in detectors: # plot the hierarchy if params.orthographic: ax = fig.gca() else: ax = fig.gca(projection="3d") plot_group( detector.hierarchy(), color, ax, orthographic=params.orthographic, show_origin_vectors=params.show_origin_vectors, panel_numbers=params.panel_numbers, ) plt.xlabel("x") plt.ylabel("y") if params.orthographic: plt.axes().set_aspect("equal", "datalim") if params.pdf_file: pp = PdfPages(params.pdf_file) for i in plt.get_fignums(): pp.savefig(plt.figure(i)) pp.close() else: plt.show()
def run(self, idx, img): if os.path.isfile(self.term_file): raise IOTATermination('IOTA_TRACKER: Termination signal received!') else: datablock = DataBlockFactory.from_filenames([img])[0] observed = self.processor.find_spots(datablock=datablock) return [idx, int(len(observed)), img, None, None]
def tst_with_external_lookup(self): from dxtbx.datablock import DataBlockFactory from dxtbx.imageset import ImageSweep from os.path import join filename = join(self.dials_regression, "centroid_test_data", "datablock_with_lookup.json") blocks = DataBlockFactory.from_json_file(filename) assert(len(blocks) == 1) imageset = blocks[0].extract_imagesets()[0] assert imageset.external_lookup.mask.data is not None assert imageset.external_lookup.gain.data is not None assert imageset.external_lookup.pedestal.data is not None assert imageset.external_lookup.mask.filename is not None assert imageset.external_lookup.gain.filename is not None assert imageset.external_lookup.pedestal.filename is not None assert imageset.external_lookup.mask.data.all_eq(True) assert imageset.external_lookup.gain.data.all_eq(1) assert imageset.external_lookup.pedestal.data.all_eq(0) blocks = self.encode_json_then_decode(blocks) assert(len(blocks) == 1) imageset = blocks[0].extract_imagesets()[0] assert imageset.external_lookup.mask.data is not None assert imageset.external_lookup.gain.data is not None assert imageset.external_lookup.pedestal.data is not None assert imageset.external_lookup.mask.filename is not None assert imageset.external_lookup.gain.filename is not None assert imageset.external_lookup.pedestal.filename is not None assert imageset.external_lookup.mask.data.all_eq(True) assert imageset.external_lookup.gain.data.all_eq(1) assert imageset.external_lookup.pedestal.data.all_eq(0) print 'OK'
def test_experimentlist_factory_from_datablock(): from dxtbx.model import Beam, Detector, Goniometer, Scan from dxtbx.datablock import DataBlockFactory from dxtbx.model import Crystal from dxtbx.format.Format import Format filenames = ["filename_%01d.cbf" % (i+1) for i in range(0, 2)] imageset = Format.get_imageset( filenames, beam = Beam(), detector = Detector(), goniometer = Goniometer(), scan = Scan((1,2), (0,1)), as_sweep=True) crystal = Crystal((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol="P1") datablock = DataBlockFactory.from_imageset(imageset) experiments = ExperimentListFactory.from_datablock_and_crystal( datablock, crystal) assert len(experiments) == 1 assert experiments[0].imageset is not None assert experiments[0].beam is not None assert experiments[0].detector is not None assert experiments[0].goniometer is not None assert experiments[0].scan is not None assert experiments[0].crystal is not None
def load_reference_geometry(self): if self.params.input.reference_geometry is None: return try: ref_datablocks = DataBlockFactory.from_json_file(self.params.input.reference_geometry, check_format=False) except Exception: ref_datablocks = None if ref_datablocks is None: from dxtbx.model.experiment_list import ExperimentListFactory try: ref_experiments = ExperimentListFactory.from_json_file(self.params.input.reference_geometry, check_format=False) except Exception: try: import dxtbx img = dxtbx.load(self.params.input.reference_geometry) except Exception: raise Sorry("Couldn't load geometry file %s"%self.params.input.reference_geometry) else: self.reference_detector = img.get_detector() else: assert len(ref_experiments.detectors()) == 1 self.reference_detector = ref_experiments.detectors()[0] else: assert len(ref_datablocks) == 1 and len(ref_datablocks[0].unique_detectors()) == 1 self.reference_detector = ref_datablocks[0].unique_detectors()[0]
def tst_from_datablock(self): from dxtbx.imageset import ImageSweep, NullReader, SweepFileList from dxtbx.model import Beam, Detector, Goniometer, Scan from dxtbx.datablock import DataBlockFactory from dxtbx.model.crystal import crystal_model imageset = ImageSweep(NullReader(SweepFileList("filename%01d.cbf", (0, 2)))) imageset.set_beam(Beam()) imageset.set_detector(Detector()) imageset.set_goniometer(Goniometer()) imageset.set_scan(Scan((1, 2), (0, 1))) crystal = crystal_model((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=0) datablock = DataBlockFactory.from_imageset(imageset) experiments = ExperimentListFactory.from_datablock_and_crystal( datablock, crystal) assert(len(experiments) == 1) assert(experiments[0].imageset is not None) assert(experiments[0].beam is not None) assert(experiments[0].detector is not None) assert(experiments[0].goniometer is not None) assert(experiments[0].scan is not None) assert(experiments[0].crystal is not None) print 'OK' pass
def run(self): from os.path import join from libtbx import easy_run input_filename = join(self.path, "datablock.json") output_filename = "output_datablock.json" mask_filename = join(self.path, "lookup_mask.pickle") easy_run.fully_buffered([ 'dials.apply_mask', 'input.datablock=%s' % input_filename, 'input.mask=%s' % mask_filename, 'output.datablock=%s' % output_filename ]).raise_if_errors() from dxtbx.datablock import DataBlockFactory datablocks = DataBlockFactory.from_json_file(output_filename) assert len(datablocks) == 1 imagesets = datablocks[0].extract_imagesets() assert len(imagesets) == 1 imageset = imagesets[0] assert imageset.external_lookup.mask.filename == mask_filename print 'OK'
def __init__(self): super(BigWidget, self).__init__() my_box = QVBoxLayout() top_box = QHBoxLayout() left_top_box = QVBoxLayout() right_top_box = QVBoxLayout() self.my_painter = ImgPainter() #json_file_path = "/home/luiso/dui/dui_test/only_9_img/dui_idials_tst_04/dials-1/1_import/datablock.json" json_file_path = "/home/luiso/dui/dui_test/X4_wide/test_02/dials-1/1_import/datablock.json" #json_file_path = "/home/lui/dui/dui_test/X4_wide/tst01/datablock.json" datablocks = DataBlockFactory.from_json_file(json_file_path) db = datablocks[0] self.my_sweep = db.extract_sweeps()[0] print "self.my_sweep.get_array_range() =", self.my_sweep.get_array_range() print "self.my_sweep.get_image_size() =", self.my_sweep.get_image_size() n_of_imgs = self.my_sweep.get_array_range()[1] print "n_of_imgs =", n_of_imgs self.palette_lst = ["hot ascend", "hot descend", "black2white", "white2black"] self.palette = self.palette_lst[0] self.img_num = 0 self.set_img() img_select = QComboBox() for num in xrange(n_of_imgs): labl = "image number:" + str(num) img_select.addItem(labl) img_select.setCurrentIndex(0) img_select.currentIndexChanged.connect(self.img_changed_by_user) palette_select = QComboBox() for plt in self.palette_lst: palette_select.addItem(plt) palette_select.currentIndexChanged.connect(self.palette_changed_by_user) left_top_box.addWidget(palette_select) top_box.addLayout(left_top_box) right_top_box.addWidget(img_select) top_box.addLayout(right_top_box) my_box.addLayout(top_box) my_scrollable =QScrollArea() my_scrollable.setWidget(self.my_painter) my_box.addWidget(my_scrollable) self.setLayout(my_box) self.show()
def test_VMXi_rotation_scan(): master_h5 = "/dls/mx/data/mx21314/mx21314-27/VMXi-AB0816/well_7/images/image_14364_master.h5" assert FormatNexus.understand(master_h5) datablocks = DataBlockFactory.from_filenames([master_h5]) imageset = datablocks[0].extract_imagesets()[0] assert imageset.get_format_class() == FormatNexus detector = imageset.get_detector() gonio = imageset.get_goniometer() scan = imageset.get_scan() beam = imageset.get_beam() panel = detector[0] assert panel.get_pixel_size() == (0.075, 0.075) assert panel.get_image_size() == (2068, 2162) assert panel.get_trusted_range() == (-1, 4096) assert panel.get_fast_axis() == (1, 0, 0) assert panel.get_slow_axis() == (0, -1, 0) assert panel.get_origin() == pytest.approx( (-78.05999999999999, 87.03, -194.5039999999999) ) assert panel.get_distance() == pytest.approx(194.504) assert isinstance(gonio, Goniometer) assert gonio.get_rotation_axis() == (0, 1, 0) assert gonio.get_fixed_rotation() == (1, 0, 0, 0, 1, 0, 0, 0, 1) assert gonio.get_setting_rotation() == (1, 0, 0, 0, 1, 0, 0, 0, 1) assert scan.get_oscillation() == pytest.approx((-30, 0.1)) assert scan.get_image_range() == (1, 600) assert beam.get_wavelength() == pytest.approx(0.979492) assert beam.get_s0() == pytest.approx((0, 0, -1 / beam.get_wavelength()))
def test_rotation_scan(master_h5): assert FormatNexusEigerDLS16MI04.understand(master_h5) datablocks = DataBlockFactory.from_filenames( [master_h5], format_kwargs={"dynamic_shadowing": True}) imageset = datablocks[0].extract_imagesets()[0] assert imageset.get_format_class() == FormatNexusEigerDLS16MI04 detector = imageset.get_detector() gonio = imageset.get_goniometer() scan = imageset.get_scan() beam = imageset.get_beam() panel = detector[0] assert panel.get_pixel_size() == (0.075, 0.075) assert panel.get_image_size() == (4148, 4362) assert panel.get_trusted_range() == (-1, 65535) assert panel.get_fast_axis() == (1, 0, 0) assert panel.get_slow_axis() == (0, -1, 0) assert panel.get_origin() == pytest.approx( (-166.07661632390744, 172.5371934106162, -200.0)) assert panel.get_distance() == 200 assert len(gonio.get_axes()) == 3 expected_axes = ((1, 0, 0), (0, 0, -1), (1, 0, 0)) for a1, a2 in zip(gonio.get_axes(), expected_axes): assert a1 == pytest.approx(a2, abs=5e-2) assert gonio.get_scan_axis() == 2 assert scan.get_oscillation() == (0, 0.2) assert scan.get_image_range() == (1, 900) assert beam.get_wavelength() == pytest.approx(0.979499) assert beam.get_s0() == pytest.approx((0, 0, -1 / beam.get_wavelength()))
def test_grid_scan(): master_h5 = "/dls/i04/data/2019/cm23004-1/20190109/Eiger/grid/Thaum/Thau_5/Thau_5_1_master.h5" assert FormatNexusEigerDLS16MI04.understand(master_h5) datablocks = DataBlockFactory.from_filenames([master_h5]) imageset = datablocks[0].extract_imagesets()[0] assert imageset.get_format_class() == FormatNexusEigerDLS16MI04 detector = imageset.get_detector() gonio = imageset.get_goniometer() scan = imageset.get_scan() beam = imageset.get_beam() panel = detector[0] assert panel.get_pixel_size() == (0.075, 0.075) assert panel.get_image_size() == (4148, 4362) assert panel.get_trusted_range() == (-1, 65535) assert panel.get_fast_axis() == (1, 0, 0) assert panel.get_slow_axis() == (0, -1, 0) assert panel.get_origin() == pytest.approx( (-167.44717577120824, 172.46833023184868, -350.0)) assert panel.get_distance() == 350 assert len(gonio.get_axes()) == 3 expected_axes = ((1, 0, 0), (0, 0, -1), (1, 0, 0)) for a1, a2 in zip(gonio.get_axes(), expected_axes): assert a1 == pytest.approx(a2, abs=5e-2) # assert gonio.get_scan_axis() == 2 assert scan is None assert beam.get_wavelength() == pytest.approx(0.979499) assert beam.get_s0() == pytest.approx((0, 0, -1 / beam.get_wavelength()))
def to_datablocks(self): """Return the experiment list as a datablock list. This assumes that the experiment contains 1 datablock.""" from dxtbx.datablock import DataBlockFactory # Convert the experiment list to dict obj = self.to_dict() # Convert the dictionary to a datablock dictionary obj["__id__"] = "DataBlock" for e in obj["experiment"]: iid = e["imageset"] imageset = obj["imageset"][iid] if "beam" in e: imageset["beam"] = e["beam"] if "detector" in e: imageset["detector"] = e["detector"] if "goniometer" in e: imageset["goniometer"] = e["goniometer"] if "scan" in e: imageset["scan"] = e["scan"] # Remove the experiments del obj["experiment"] # Create the datablock return DataBlockFactory.from_dict([obj])
def ini_datablock(self, json_file_path): if(json_file_path != None): try: datablocks = DataBlockFactory.from_json_file(json_file_path) ##TODO check length of datablock for safety datablock = datablocks[0] self.my_sweep = datablock.extract_sweeps()[0] self.img_select.clear() except: print "Failed to load images from datablock.json" try: print "self.my_sweep.get_array_range() =", self.my_sweep.get_array_range() n_of_imgs = len(self.my_sweep.indices()) print "n_of_imgs =", n_of_imgs self.img_select.setMaximum(n_of_imgs) self.img_select.setMinimum(1) self.img_step.setMaximum(n_of_imgs / 2) self.img_step.setMinimum(1) self.num_of_imgs_to_add.setMaximum(n_of_imgs) self.num_of_imgs_to_add.setMinimum(1) except: print "Failed to set up IMG control dialog" self.btn_first_clicked() self.ini_contrast() self.set_img() QTimer.singleShot(1000, self.scale2border)
def tst_from_datablock(self): from dxtbx.imageset import ImageSweep, NullReader, SweepFileList from dxtbx.model import Beam, Detector, Goniometer, Scan from dxtbx.datablock import DataBlockFactory from dxtbx.model.crystal import crystal_model imageset = ImageSweep(NullReader(SweepFileList("filename%01d.cbf", (0, 2)))) imageset.set_beam(Beam()) imageset.set_detector(Detector()) imageset.set_goniometer(Goniometer()) imageset.set_scan(Scan((1, 2), (0, 1))) crystal = crystal_model((1, 0, 0), (0, 1, 0), (0, 0, 1), space_group_symbol=0) datablock = DataBlockFactory.from_imageset(imageset) experiments = ExperimentListFactory.from_datablock_and_crystal( datablock, crystal) assert(len(experiments) == 1) assert(experiments[0].imageset is not None) assert(experiments[0].beam is not None) assert(experiments[0].detector is not None) assert(experiments[0].goniometer is not None) assert(experiments[0].scan is not None) assert(experiments[0].crystal is not None) print 'OK'
def tst_with_external_lookup(self): filename = join(self.dials_regression, "centroid_test_data", "datablock_with_lookup.json") blocks = DataBlockFactory.from_json_file(filename) assert(len(blocks) == 1) imageset = blocks[0].extract_imagesets()[0] assert imageset.external_lookup.mask.data is not None assert imageset.external_lookup.gain.data is not None assert imageset.external_lookup.pedestal.data is not None assert imageset.external_lookup.mask.filename is not None assert imageset.external_lookup.gain.filename is not None assert imageset.external_lookup.pedestal.filename is not None assert imageset.external_lookup.mask.data.all_eq(True) assert imageset.external_lookup.gain.data.all_eq(1) assert imageset.external_lookup.pedestal.data.all_eq(0) blocks = self.encode_json_then_decode(blocks) assert(len(blocks) == 1) imageset = blocks[0].extract_imagesets()[0] assert imageset.external_lookup.mask.data is not None assert imageset.external_lookup.gain.data is not None assert imageset.external_lookup.pedestal.data is not None assert imageset.external_lookup.mask.filename is not None assert imageset.external_lookup.gain.filename is not None assert imageset.external_lookup.pedestal.filename is not None assert imageset.external_lookup.mask.data.all_eq(True) assert imageset.external_lookup.gain.data.all_eq(1) assert imageset.external_lookup.pedestal.data.all_eq(0) print 'OK'
def test_spring8_ccp4_2018_zenodo_1443110_data03(): # https://zenodo.org/record/1443110#.XD8bD5ynzmE master_h5 = "/dls/mx-scratch/rjgildea/zenodo/spring8-ccp4-2018/1443110/ccp4school2018_bl41xu/05/data03/data03_master.h5" assert FormatHDF5EigerNearlyNexusSPring8.understand(master_h5) datablocks = DataBlockFactory.from_filenames([master_h5]) imageset = datablocks[0].extract_imagesets()[0] assert imageset.get_format_class() == FormatHDF5EigerNearlyNexusSPring8 detector = imageset.get_detector() gonio = imageset.get_goniometer() scan = imageset.get_scan() beam = imageset.get_beam() panel = detector[0] assert panel.get_pixel_size() == pytest.approx((0.075, 0.075)) assert panel.get_image_size() == (4150, 4371) assert panel.get_trusted_range() == (-1, 2.094707e06) assert panel.get_fast_axis() == (1, 0, 0) assert panel.get_slow_axis() == (0, -1, 0) assert panel.get_origin() == pytest.approx((-151.939, 169.629, -180), abs=1e-3) assert panel.get_distance() == pytest.approx(180) assert isinstance(gonio, Goniometer) assert gonio.get_rotation_axis() == (-1, 0, 0) assert gonio.get_fixed_rotation() == (1, 0, 0, 0, 1, 0, 0, 0, 1) assert gonio.get_setting_rotation() == (1, 0, 0, 0, 1, 0, 0, 0, 1) assert scan.get_oscillation() == pytest.approx((-10, 1)) assert scan.get_image_range() == (1, 180) assert beam.get_wavelength() == pytest.approx(1.28241, abs=1e-5) assert beam.get_s0() == pytest.approx((0, 0, -1 / beam.get_wavelength()))
def test_with_external_lookup(centroid_test_data): filename = os.path.join(centroid_test_data, "datablock_with_lookup.json") blocks = DataBlockFactory.from_json_file(filename) assert len(blocks) == 1 imageset = blocks[0].extract_imagesets()[0] assert not imageset.external_lookup.mask.data.empty() assert not imageset.external_lookup.gain.data.empty() assert not imageset.external_lookup.pedestal.data.empty() assert imageset.external_lookup.mask.filename is not None assert imageset.external_lookup.gain.filename is not None assert imageset.external_lookup.pedestal.filename is not None assert imageset.external_lookup.mask.data.tile(0).data().all_eq(True) assert imageset.external_lookup.gain.data.tile(0).data().all_eq(1) assert imageset.external_lookup.pedestal.data.tile(0).data().all_eq(0) blocks = encode_json_then_decode(blocks) assert len(blocks) == 1 imageset = blocks[0].extract_imagesets()[0] assert not imageset.external_lookup.mask.data.empty() assert not imageset.external_lookup.gain.data.empty() assert not imageset.external_lookup.pedestal.data.empty() assert imageset.external_lookup.mask.filename is not None assert imageset.external_lookup.gain.filename is not None assert imageset.external_lookup.pedestal.filename is not None assert imageset.external_lookup.mask.data.tile(0).data().all_eq(True) assert imageset.external_lookup.gain.data.tile(0).data().all_eq(1) assert imageset.external_lookup.pedestal.data.tile(0).data().all_eq(0)
def test_units(): master_h5 = "/dls/i04/data/2019/cm23004-1/20190114/Eiger/grid/Se_Thaum/Se_Thaum_12/Se_Thaum_12_2_master.h5" assert FormatNexusEigerDLS16MI04.understand(master_h5) datablocks = DataBlockFactory.from_filenames([master_h5]) imageset = datablocks[0].extract_imagesets()[0] assert imageset.get_format_class() == FormatNexusEigerDLS16MI04 detector = imageset.get_detector() gonio = imageset.get_goniometer() scan = imageset.get_scan() beam = imageset.get_beam() panel = detector[0] assert panel.get_pixel_size() == (0.075, 0.075) assert panel.get_fast_axis() == (1, 0, 0) assert panel.get_slow_axis() == (0, -1, 0) # XXX Need to check and update expected values here, however the ones # dxtbx is currently generating from the file are very wrong assert panel.get_origin() == pytest.approx( (-167.35570274412459, 172.4729262553403, -339.9887931971389)) assert panel.get_distance() == pytest.approx(339.9887931971389) assert scan is None assert beam.get_wavelength() == pytest.approx(0.979499) assert beam.get_s0() == pytest.approx((0, 0, -1 / beam.get_wavelength())) assert panel.get_beam_centre_px(beam.get_s0()) == pytest.approx( (2231.41, 2299.64))
def datablock_from_numpyarrays(image, detector, beam, mask=None): """ So that one can do e.g. >> dblock = datablock_from_numpyarrays( image, detector, beam) >> refl = flex.reflection_table.from_observations(dblock, spot_finder_params) without having to utilize the harddisk :param image: numpy array image, or list of numpy arrays :param mask: numpy mask, should be same shape format as numpy array :param detector: dxtbx detector model :param beam: dxtbx beam model :return: datablock for the image """ if isinstance(image, list): image = np.array(image) if mask is not None: if isinstance(mask, list): mask = np.array(mask).astype(bool) I = FormatInMemory(image=image, mask=mask) reader = MemReader([I]) #masker = MemMasker([I]) iset_Data = ImageSetData(reader, None) #, masker) iset = ImageSet(iset_Data) iset.set_beam(beam) iset.set_detector(detector) dblock = DataBlockFactory.from_imageset([iset])[0] return dblock
def do_import(filename): info("Loading %s"%os.path.basename(filename)) datablocks = DataBlockFactory.from_filenames([filename]) if len(datablocks) == 0: raise Abort("Could not load %s"%filename) if len(datablocks) > 1: raise Abort("Got multiple datablocks from file %s"%filename) return datablocks[0]
def __init__(self, source_image=None, object_folder=None, gain = 0.32, params=None): '''Initialise the script.''' from dials.util.options import OptionParser from dxtbx.datablock import DataBlockFactory from dials.array_family import flex from iotbx.phil import parse from xfel.command_line.xfel_process import phil_scope phil_scope = parse(''' include scope xfel.command_line.xtc_process.phil_scope ''', process_includes=True) sub_phil_scope = parse(''' output { cxi_merge_picklefile = None .type = str .help = Output integration results for each color data to separate cctbx.xfel-style pickle files } indexing { stills { ewald_proximity_resolution_cutoff = 2.0 .type = float .help = For calculating the area under the green curve, or the acceptable .help = volume of reciprocal space for spot prediction, use this high-resolution cutoff } } cxi_merge { include scope xfel.command_line.cxi_merge.master_phil } ''', process_includes=True) phil_scope.adopt_scope(sub_phil_scope) # Create the parser self.parser = OptionParser( phil=phil_scope, read_datablocks=True, read_datablocks_from_images=True) self.params = params self.img = [source_image] self.obj_base = object_folder self.phil = phil_scope.extract() with misc.Capturing() as junk_output: self.datablock = DataBlockFactory.from_filenames(self.img)[0] self.obj_filename = "int_{}".format(os.path.basename(self.img[0])) self.phil.output.cxi_merge_picklefile = os.path.join(self.obj_base, self.img[0])
def from_string(self, s): from dxtbx.datablock import DataBlockFactory from os.path import exists from libtbx.utils import Sorry if s is None: return None if s not in self.cache: if not exists(s): raise Sorry('File %s does not exist' % s) self.cache[s] = FilenameDataWrapper(s, DataBlockFactory.from_json_file(s, check_format=self._check_format)) return self.cache[s]
def datablock(filename, check_format=True): ''' Load a given JSON or pickle file. Params: filename The input filename Returns: The datablock ''' from dxtbx.datablock import DataBlockFactory return DataBlockFactory.from_serialized_format( filename, check_format=check_format)
def work(filename, cl=[]): from dials.command_line.find_spots import phil_scope as params from dxtbx.datablock import DataBlockFactory from dials.array_family import flex interp = params.command_line_argument_interpreter() for cla in cl: params = params.fetch(interp.process(cla)) datablock = DataBlockFactory.from_filenames([filename])[0] reflections = flex.reflection_table.from_observations( datablock, params.extract()) detector = datablock.unique_detectors()[0] beam = datablock.unique_beams()[0] return analyse(reflections, detector, beam)
def do_import(filename): logger.info("Loading %s"%os.path.basename(filename)) try: datablocks = DataBlockFactory.from_json_file(filename) except ValueError: datablocks = DataBlockFactory.from_filenames([filename]) if len(datablocks) == 0: raise Abort("Could not load %s"%filename) if len(datablocks) > 1: raise Abort("Got multiple datablocks from file %s"%filename) # Ensure the indexer and downstream applications treat this as set of stills from dxtbx.imageset import ImageSet reset_sets = [] for imageset in datablocks[0].extract_imagesets(): imageset = ImageSet(imageset.reader(), imageset.indices()) imageset._models = imageset._models imageset.set_scan(None) imageset.set_goniometer(None) reset_sets.append(imageset) return DataBlockFactory.from_imageset(reset_sets)[0]
def tst_pickling(self): from dxtbx.datablock import DataBlockFactory filenames = self.multiple_block_filenames() blocks1 = DataBlockFactory.from_filenames(filenames) blocks2 = self.pickle_then_unpickle(blocks1) assert(len(blocks2) == len(blocks1)) for b1, b2 in zip(blocks1, blocks2): assert(b1.format_class() == b2.format_class()) assert(b1 == b2) assert(blocks1 == blocks2) print 'OK'
def tst_create_multiple_sweeps(self): from dxtbx.datablock import DataBlockFactory filenames = self.multiple_sweep_filenames() blocks = DataBlockFactory.from_filenames(filenames) assert(len(blocks) == 1) assert(blocks[0].num_images() == 6) imageset = blocks[0].extract_imagesets() assert(len(imageset) == 2) sweeps = blocks[0].extract_sweeps() assert(len(sweeps) == 2) assert(len(sweeps[0]) == 3) assert(len(sweeps[1]) == 3) print 'OK'
def OnChooseDirectory (self, event) : dir_name = self.dir_ctrl.GetPhilValue() if (dir_name is not None) : from dxtbx.datablock import DataBlockFactory datablocks = DataBlockFactory.from_filenames([dir_name]) imagesets = datablocks[0].extract_imagesets() self._imagesets = imagesets #from iotbx.detectors import identify_dataset #self._datasets = identify_dataset(dir_name) #choices = [ d.format() for d in self._datasets ] choices = [imgset.get_template() for imgset in self._imagesets] self.stack_ctrl.SetItems(choices) for i in range(len(choices)): self.stack_ctrl.SetSelection(i)
def __init__(self, source_image, object_folder, final_folder, final_filename, final, logfile, gain = 0.32, params=None): '''Initialise the script.''' from dxtbx.datablock import DataBlockFactory self.params = params # Read settings from the DIALS target (.phil) file # If none is provided, use default settings (and may God have mercy) if self.params.dials.target != None: with open(self.params.dials.target, 'r') as settings_file: settings_file_contents = settings_file.read() settings = parse(settings_file_contents) current_phil = phil_scope.fetch(sources=[settings]) self.phil = current_phil.extract() else: self.phil = phil_scope.extract() # Set general file-handling settings file_basename = os.path.basename(source_image).split('.')[0] self.phil.output.datablock_filename = "{}/{}.json".format(object_folder, file_basename) self.phil.output.indexed_filename = "{}/{}_indexed.pickle".format(object_folder, file_basename) self.phil.output.strong_filename = "{}/{}_strong.pickle".format(object_folder, file_basename) self.phil.output.refined_experiments_filename = "{}/{}_refined_experiments.json".format(object_folder, file_basename) self.phil.output.integrated_filename = "{}/{}_integrated.pickle".format(object_folder, file_basename) self.phil.output.profile_filename = "{}/{}_profile.phil".format(object_folder, file_basename) self.phil.output.integration_pickle = final_filename self.int_log = logfile #"{}/int_{}.log".format(final_folder, file_basename) self.img = [source_image] self.obj_base = object_folder self.gain = gain self.fail = None self.frame = None self.final = final self.final['final'] = final_filename with misc.Capturing() as junk_output: self.datablock = DataBlockFactory.from_filenames(self.img)[0] self.obj_filename = "int_{}".format(os.path.basename(self.img[0]))
def ini_datablock(self, json_file_path): if json_file_path is not None: try: datablocks = DataBlockFactory.from_json_file(json_file_path) # TODO check length of datablock for safety datablock = datablocks[0] self.my_sweep = datablock.extract_sweeps()[0] self.img_select.clear() except BaseException as e: # We don't want to catch bare exceptions but don't know # what this was supposed to catch. Log it. logger.error( "Caught unknown exception type %s: %s", type(e).__name__, e ) logger.debug("Failed to load images from datablock.json") try: logger.debug( "self.my_sweep.get_array_range() = %s", self.my_sweep.get_array_range(), ) n_of_imgs = len(self.my_sweep.indices()) logger.debug("n_of_imgs = %s", n_of_imgs) self.img_select.setMaximum(n_of_imgs) self.img_select.setMinimum(1) self.img_step.setMaximum(n_of_imgs / 2) self.img_step.setMinimum(1) self.num_of_imgs_to_add.setMaximum(n_of_imgs) self.num_of_imgs_to_add.setMinimum(1) except BaseException as e: # We don't want to catch bare exceptions but don't know # what this was supposed to catch. Log it. logger.error( "Caught unknown exception type %s: %s", type(e).__name__, e ) logger.debug("Failed to set up IMG control dialog") self.btn_first_clicked() self.ini_contrast() self.set_img() QTimer.singleShot(1000, self.scale2border)
def try_read_datablocks_from_images(self, args, verbose, compare_beam, compare_detector, compare_goniometer, scan_tolerance, format_kwargs): ''' Try to import images. :param args: The input arguments :param verbose: Print verbose output :return: Unhandled arguments ''' from dxtbx.datablock import DataBlockFactory from dials.phil import FilenameDataWrapper, DataBlockConverters from glob import glob # If filenames contain wildcards, expand args_new = [] for arg in args: if "*" in arg: args_new.extend(glob(arg)) else: args_new.append(arg) args = args_new unhandled = [] datablocks = DataBlockFactory.from_filenames( args, verbose=verbose, unhandled=unhandled, compare_beam=compare_beam, compare_detector=compare_detector, compare_goniometer=compare_goniometer, scan_tolerance=scan_tolerance, format_kwargs=format_kwargs) if len(datablocks) > 0: filename = "<image files>" obj = FilenameDataWrapper(filename, datablocks) DataBlockConverters.cache[filename] = obj self.datablocks.append(obj) return unhandled
def tst_create_multiple_blocks(self): from dxtbx.datablock import DataBlockFactory filenames = self.multiple_block_filenames() blocks = DataBlockFactory.from_filenames(filenames, verbose=False) assert(len(blocks) == 22) # Block 1 assert(blocks[0].num_images() == 9) imageset = blocks[0].extract_imagesets() assert(len(imageset) == 1) assert(len(imageset[0]) == 9) sweeps = blocks[0].extract_sweeps() assert(len(sweeps) == 1) assert(len(sweeps[0]) == 9) print 'OK'
def load_reference_geometry(self): if self.params.input.reference_geometry is None: return try: ref_datablocks = DataBlockFactory.from_json_file(self.params.input.reference_geometry, check_format=False) except Exception: ref_datablocks = None if ref_datablocks is None: from dxtbx.model.experiment.experiment_list import ExperimentListFactory try: ref_experiments = ExperimentListFactory.from_json_file(self.params.input.reference_geometry, check_format=False) except Exception: raise Sorry("Couldn't load geometry file %s"%self.params.input.reference_geometry) assert len(ref_experiments.detectors()) == 1 self.reference_detector = ref_experiments.detectors()[0] else: assert len(ref_datablocks) == 1 and len(ref_datablocks[0].unique_detectors()) == 1 self.reference_detector = ref_datablocks[0].unique_detectors()[0]
def __call__(self): ''' Import the datablocks ''' from dxtbx.datablock import DataBlockTemplateImporter from dxtbx.datablock import DataBlockFactory from dials.util.options import flatten_datablocks from libtbx.utils import Sorry # Get the datablocks datablocks = flatten_datablocks(self.params.input.datablock) # Check we have some filenames if len(datablocks) == 0: format_kwargs = { 'dynamic_shadowing' : self.params.format.dynamic_shadowing } # Check if a template has been set and print help if not, otherwise try to # import the images based on the template input if len(self.params.input.template) > 0: importer = DataBlockTemplateImporter( self.params.input.template, max(self.params.verbosity-1, 0), format_kwargs=format_kwargs) datablocks = importer.datablocks if len(datablocks) == 0: raise Sorry('No datablocks found matching template %s' % self.params.input.template) elif len(self.params.input.directory) > 0: datablocks = DataBlockFactory.from_filenames( self.params.input.directory, max(self.params.verbosity-1, 0), format_kwargs=format_kwargs) if len(datablocks) == 0: raise Sorry('No datablocks found in directories %s' % self.params.input.directory) else: raise Sorry('No datablocks found') if len(datablocks) > 1: raise Sorry("More than 1 datablock found") # Return the datablocks return datablocks[0]
def __init__(self, img, gain, params): """ Initialization and data read-in """ from dxtbx.datablock import DataBlockFactory self.gain = gain self.params = params # Read settings from the DIALS target (.phil) file # If none is provided, use default settings (and may God have mercy) if self.params.dials.target != None: with open(self.params.dials.target, 'r') as settings_file: settings_file_contents = settings_file.read() settings = parse(settings_file_contents) current_phil = phil_scope.fetch(sources=[settings]) self.phil = current_phil.extract() else: self.phil = phil_scope.extract() # Convert raw image into single-image datablock with misc.Capturing() as junk_output: self.datablock = DataBlockFactory.from_filenames([img])[0]
def tst_from_null_sweep(self): from dxtbx.datablock import DataBlockFactory from dxtbx.imageset import NullReader, ImageSweep, SweepFileList from dxtbx.model import Beam, Detector, Goniometer, Scan sweep = ImageSweep(NullReader(SweepFileList("template_%2d.cbf", (0, 10)))) sweep.set_beam(Beam((0, 0, 1))) sweep.set_detector(Detector()) sweep.set_goniometer(Goniometer((1, 0, 0))) sweep.set_scan(Scan((1, 10), (0, 0.1))) # Create the datablock datablock = DataBlockFactory.from_imageset(sweep) assert(len(datablock) == 1) datablock = datablock[0] sweeps = datablock.extract_sweeps() assert(len(sweeps) == 1) assert(sweeps[0].get_beam() == sweep.get_beam()) assert(sweeps[0].get_detector() == sweep.get_detector()) assert(sweeps[0].get_goniometer() == sweep.get_goniometer()) assert(sweeps[0].get_scan() == sweep.get_scan()) print 'OK'
filenames = [] for arg in args: if "indexing.data" in arg: path = arg.split('=')[1] if os.path.isdir(path): for subfile in os.listdir(path): subpath = os.path.join(path, subfile) if os.path.isfile(subpath): filenames.append(subpath) else: filenames.append(path) print filenames datablock = DataBlockFactory.from_filenames(filenames)[0] observed = flex.reflection_table.from_observations(datablock, params) observed.as_pickle("strong.pickle") print "Number of observed reflections:", len(observed) working_params = copy.deepcopy(params) imagesets = datablock.extract_imagesets() # old labelit # from spotfinder.applications.xfel import cxi_phil # horizons_phil = cxi_phil.cxi_versioned_extract(args) print "indexing..." t0 = clock()
def exercise_spotfinder(): if not libtbx.env.has_module("dials_regression"): print "Skipping exercise_spotfinder: dials_regression not present" return data_dir = libtbx.env.find_in_repositories(relative_path="dials_regression/centroid_test_data", test=os.path.isdir) template = glob(os.path.join(data_dir, "centroid*.cbf")) args = ["dials.find_spots", " ".join(template), "output.reflections=spotfinder.pickle", "output.shoeboxes=True"] result = easy_run.fully_buffered(command=" ".join(args)).raise_if_errors() assert os.path.exists("spotfinder.pickle") with open("spotfinder.pickle", "rb") as f: reflections = pickle.load(f) assert len(reflections) == 653, len(reflections) refl = reflections[0] assert approx_equal(refl["intensity.sum.value"], 42) assert approx_equal(refl["bbox"], (1398, 1400, 513, 515, 0, 1)) assert approx_equal(refl["xyzobs.px.value"], (1399.1190476190477, 514.2142857142857, 0.5)) assert "shoebox" in reflections print "OK" # now with a resolution filter args = [ "dials.find_spots", "filter.d_min=2", "filter.d_max=15", " ".join(template), "output.reflections=spotfinder.pickle", "output.shoeboxes=False", ] result = easy_run.fully_buffered(command=" ".join(args)).raise_if_errors() assert os.path.exists("spotfinder.pickle") with open("spotfinder.pickle", "rb") as f: reflections = pickle.load(f) assert len(reflections) == 467, len(reflections) assert "shoebox" not in reflections print "OK" # now with more generous parameters args = [ "dials.find_spots", "min_spot_size=3", "max_separation=3", " ".join(template), "output.reflections=spotfinder.pickle", ] result = easy_run.fully_buffered(command=" ".join(args)).raise_if_errors() assert os.path.exists("spotfinder.pickle") with open("spotfinder.pickle", "rb") as f: reflections = pickle.load(f) assert len(reflections) == 678, len(reflections) print "OK" # Now with a user defined mask template = glob(os.path.join(data_dir, "centroid*.cbf")) args = [ "dials.find_spots", " ".join(template), "output.reflections=spotfinder.pickle", "output.shoeboxes=True", "lookup.mask=%s" % os.path.join(data_dir, "mask.pickle"), ] result = easy_run.fully_buffered(command=" ".join(args)).raise_if_errors() assert os.path.exists("spotfinder.pickle") with open("spotfinder.pickle", "rb") as f: reflections = pickle.load(f) from dxtbx.datablock import DataBlockFactory datablocks = DataBlockFactory.from_json_file(os.path.join(data_dir, "datablock.json")) assert len(datablocks) == 1 imageset = datablocks[0].extract_imagesets()[0] detector = imageset.get_detector() beam = imageset.get_beam() for x, y, z in reflections["xyzobs.px.value"]: d = detector[0].get_resolution_at_pixel(beam.get_s0(), (x, y)) assert d >= 3 # Now with a user defined mask template = glob(os.path.join(data_dir, "centroid*.cbf")) args = [ "dials.find_spots", " ".join(template), "output.reflections=spotfinder.pickle", "output.shoeboxes=True", "region_of_interest=800,1200,800,1200", ] result = easy_run.fully_buffered(command=" ".join(args)).raise_if_errors() assert os.path.exists("spotfinder.pickle") with open("spotfinder.pickle", "rb") as f: reflections = pickle.load(f) x, y, z = reflections["xyzobs.px.value"].parts() assert x.all_ge(800) assert y.all_ge(800) assert x.all_lt(1200) assert y.all_lt(1200) print "OK" # now with XFEL stills data_dir = libtbx.env.find_in_repositories( relative_path="dials_regression/spotfinding_test_data", test=os.path.isdir ) template = os.path.join(data_dir, "idx-s00-20131106040302615.cbf") args = ["dials.find_spots", template, "output.reflections=spotfinder.pickle"] result = easy_run.fully_buffered(command=" ".join(args)).raise_if_errors() assert os.path.exists("spotfinder.pickle") with open("spotfinder.pickle", "rb") as f: reflections = pickle.load(f) assert len(reflections) == 2643, len(reflections) print "OK"
def index_from_files(f_one,f_two,f_three): import dxtbx from iotbx.phil import parse from dxtbx.datablock import DataBlockFactory from dials.array_family import flex from dials.algorithms.indexing.indexer import indexer_base from dials.util.options import OptionParser import copy phil_scope_str=''' output {{ shoeboxes = True .type = bool .help = Save the raw pixel values inside the reflection shoeboxes. }} include scope dials.algorithms.spot_finding.factory.phil_scope include scope dials.algorithms.indexing.indexer.index_only_phil_scope include scope dials.algorithms.refinement.refiner.phil_scope indexing.known_symmetry.unit_cell={0} .type = unit_cell indexing.known_symmetry.space_group={1} .type = space_group ''' phil_scope = parse(phil_scope_str.format(target_cell,target_sg), process_includes=True) # from dials.util.options import OptionParser parser = OptionParser(phil=phil_scope) params, options = parser.parse_args(args=[], show_diff_phil=True) params.refinement.parameterisation.scan_varying = False params.indexing.method='real_space_grid_search' # params.indexing.method='fft3d' # params.indexing.max_cell=800 # params.spotfinder.filter.min_spot_size=3 filenames = [f_one,f_two,f_three] datablock = DataBlockFactory.from_filenames(filenames)[0] observed = flex.reflection_table.from_observations(datablock, params) observed.as_pickle("strong.pickle") print "Number of observed reflections:", len(observed) working_params = copy.deepcopy(params) imagesets = datablock.extract_imagesets() print "indexing..." t0 = time() # new dials, fix by Aaron idxr = indexer_base.from_parameters(observed, imagesets, params=params) idxr.index() tel = time()-t0 print "done indexing (",tel," sec)" # new dials indexed = idxr.refined_reflections experiments = idxr.refined_experiments print experiments.crystals()[0] crystal_params = experiments.crystals()[0] with open('crystal.pkl', 'wb') as output: pickle.dump(crystal_params, output, pickle.HIGHEST_PROTOCOL) return
print mycrystal # get a CS-PAD detector for testing import os import libtbx.load_env dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) data_dir = os.path.join(dials_regression, "refinement_test_data", "hierarchy_test") datablock_path = os.path.join(data_dir, "datablock.json") assert os.path.exists(datablock_path) # load models from dxtbx.datablock import DataBlockFactory datablock = DataBlockFactory.from_serialized_format(datablock_path, check_format=False) im_set = datablock[0].extract_imagesets()[0] from copy import deepcopy cspad = deepcopy(im_set.get_detector()) # get also a hierarchical type P6M detector data_dir = os.path.join(dials_regression, "refinement_test_data", "metrology", "i03-2.5A-thaumatin-20140514-split") datablock_path = os.path.join(data_dir, "datablock.json") datablock = DataBlockFactory.from_serialized_format(datablock_path, check_format=False) im_set = datablock[0].extract_imagesets()[0] from copy import deepcopy p6m = deepcopy(im_set.get_detector()) print p6m[0] # get a P12M (cropped to middle 18 modules)
def test_refinement(): '''Test a refinement run''' dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) # Get a beam and detector from a datablock. This one has a CS-PAD, but that # is irrelevant data_dir = os.path.join(dials_regression, "refinement_test_data", "hierarchy_test") datablock_path = os.path.join(data_dir, "datablock.json") assert os.path.exists(datablock_path) # load models from dxtbx.datablock import DataBlockFactory datablock = DataBlockFactory.from_serialized_format(datablock_path, check_format=False) im_set = datablock[0].extract_imagesets()[0] from copy import deepcopy detector = deepcopy(im_set.get_detector()) beam = im_set.get_beam() # Invent a crystal, goniometer and scan for this test from dxtbx.model.crystal import crystal_model crystal = crystal_model((40.,0.,0.) ,(0.,40.,0.), (0.,0.,40.), space_group_symbol = "P1") orig_xl = deepcopy(crystal) from dxtbx.model.experiment import goniometer_factory goniometer = goniometer_factory.known_axis((1., 0., 0.)) # Build a mock scan for a 180 degree sweep from dxtbx.model.scan import scan_factory sf = scan_factory() scan = sf.make_scan(image_range = (1,1800), exposure_times = 0.1, oscillation = (0, 0.1), epochs = range(1800), deg = True) sweep_range = scan.get_oscillation_range(deg=False) im_width = scan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) from dxtbx.model.experiment.experiment_list import ExperimentList, Experiment # Build an experiment list experiments = ExperimentList() experiments.append(Experiment( beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None)) # simulate some reflections refs, _ = generate_reflections(experiments) # change unit cell a bit (=0.1 Angstrom length upsets, 0.1 degree of # alpha and beta angles) from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalUnitCellParameterisation xluc_param = CrystalUnitCellParameterisation(crystal) xluc_p_vals = xluc_param.get_param_vals() cell_params = crystal.get_unit_cell().parameters() cell_params = [a + b for a, b in zip(cell_params, [0.1, -0.1, 0.1, 0.1, -0.1, 0.0])] from cctbx.uctbx import unit_cell from rstbx.symmetry.constraints.parameter_reduction import \ symmetrize_reduce_enlarge from scitbx import matrix new_uc = unit_cell(cell_params) newB = matrix.sqr(new_uc.fractionalization_matrix()).transpose() S = symmetrize_reduce_enlarge(crystal.get_space_group()) S.set_orientation(orientation=newB) X = tuple([e * 1.e5 for e in S.forward_independent_parameters()]) xluc_param.set_param_vals(X) # reparameterise the crystal at the perturbed geometry xluc_param = CrystalUnitCellParameterisation(crystal) # Dummy parameterisations for other models beam_param = None xlo_param = None det_param = None # parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.parameter_report import \ ParameterReporter pred_param = TwoThetaPredictionParameterisation(experiments, det_param, beam_param, xlo_param, [xluc_param]) param_reporter = ParameterReporter(det_param, beam_param, xlo_param, [xluc_param]) # reflection manager refman = TwoThetaReflectionManager(refs, experiments, nref_per_degree=20, verbosity=2) # reflection predictor ref_predictor = TwoThetaExperimentsPredictor(experiments) # target function target = TwoThetaTarget(experiments, ref_predictor, refman, pred_param) # minimisation engine from dials.algorithms.refinement.engine \ import LevenbergMarquardtIterations as Refinery refinery = Refinery(target = target, prediction_parameterisation = pred_param, log = None, verbosity = 0, track_step = False, track_gradient = False, track_parameter_correlation = False, max_iterations = 20) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner(reflections=refs, experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, verbosity=1) history = refiner.run() # compare crystal with original crystal refined_xl = refiner.get_experiments()[0].crystal #print refined_xl assert refined_xl.is_similar_to(orig_xl, uc_rel_length_tolerance=0.001, uc_abs_angle_tolerance=0.01) #print "Unit cell esds:" #print refined_xl.get_cell_parameter_sd() return
def test1(): dials_regression = libtbx.env.find_in_repositories( relative_path="dials_regression", test=os.path.isdir) # use a datablock that contains a CS-PAD detector description data_dir = os.path.join(dials_regression, "refinement_test_data", "hierarchy_test") datablock_path = os.path.join(data_dir, "datablock.json") assert os.path.exists(datablock_path) # load models from dxtbx.datablock import DataBlockFactory datablock = DataBlockFactory.from_serialized_format(datablock_path, check_format=False) im_set = datablock[0].extract_imagesets()[0] from copy import deepcopy detector = deepcopy(im_set.get_detector()) beam = im_set.get_beam() # we'll invent a crystal, goniometer and scan for this test from dxtbx.model.crystal import crystal_model crystal = crystal_model((40.,0.,0.) ,(0.,40.,0.), (0.,0.,40.), space_group_symbol = "P1") from dxtbx.model.experiment import goniometer_factory goniometer = goniometer_factory.known_axis((1., 0., 0.)) # Build a mock scan for a 180 degree sweep from dxtbx.model.scan import scan_factory sf = scan_factory() scan = sf.make_scan(image_range = (1,1800), exposure_times = 0.1, oscillation = (0, 0.1), epochs = range(1800), deg = True) sweep_range = scan.get_oscillation_range(deg=False) im_width = scan.get_oscillation(deg=False)[1] assert sweep_range == (0., pi) assert approx_equal(im_width, 0.1 * pi / 180.) from dxtbx.model.experiment.experiment_list import ExperimentList, Experiment # Build an experiment list experiments = ExperimentList() experiments.append(Experiment( beam=beam, detector=detector, goniometer=goniometer, scan=scan, crystal=crystal, imageset=None)) # simulate some reflections refs, ref_predictor = generate_reflections(experiments) # move the detector quadrants apart by 2mm both horizontally and vertically from dials.algorithms.refinement.parameterisation \ import DetectorParameterisationHierarchical det_param = DetectorParameterisationHierarchical(detector, level=1) det_p_vals = det_param.get_param_vals() p_vals = list(det_p_vals) p_vals[1] += 2 p_vals[2] -= 2 p_vals[7] += 2 p_vals[8] += 2 p_vals[13] -= 2 p_vals[14] += 2 p_vals[19] -= 2 p_vals[20] -= 2 det_param.set_param_vals(p_vals) # reparameterise the detector at the new perturbed geometry det_param = DetectorParameterisationHierarchical(detector, level=1) # parameterise other models from dials.algorithms.refinement.parameterisation.beam_parameters import \ BeamParameterisation from dials.algorithms.refinement.parameterisation.crystal_parameters import \ CrystalOrientationParameterisation, CrystalUnitCellParameterisation beam_param = BeamParameterisation(beam, goniometer) xlo_param = CrystalOrientationParameterisation(crystal) xluc_param = CrystalUnitCellParameterisation(crystal) # fix beam beam_param.set_fixed([True]*3) # fix crystal xluc_param.set_fixed([True]*6) xlo_param.set_fixed([True]*3) # parameterisation of the prediction equation from dials.algorithms.refinement.parameterisation.prediction_parameters import \ XYPhiPredictionParameterisation from dials.algorithms.refinement.parameterisation.parameter_report import \ ParameterReporter pred_param = XYPhiPredictionParameterisation(experiments, [det_param], [beam_param], [xlo_param], [xluc_param]) param_reporter = ParameterReporter([det_param], [beam_param], [xlo_param], [xluc_param]) # reflection manager and target function from dials.algorithms.refinement.target import \ LeastSquaresPositionalResidualWithRmsdCutoff from dials.algorithms.refinement.reflection_manager import ReflectionManager refman = ReflectionManager(refs, experiments, nref_per_degree=20) # set a very tight rmsd target of 1/10000 of a pixel target = LeastSquaresPositionalResidualWithRmsdCutoff(experiments, ref_predictor, refman, pred_param, restraints_parameterisation=None, frac_binsize_cutoff=0.0001) # minimisation engine from dials.algorithms.refinement.engine \ import LevenbergMarquardtIterations as Refinery refinery = Refinery(target = target, prediction_parameterisation = pred_param, log = None, verbosity = 0, track_step = False, track_gradient = False, track_parameter_correlation = False, max_iterations = 20) # Refiner from dials.algorithms.refinement.refiner import Refiner refiner = Refiner(reflections=refs, experiments=experiments, pred_param=pred_param, param_reporter=param_reporter, refman=refman, target=target, refinery=refinery, verbosity=0) history = refiner.run() assert history.reason_for_termination == "RMSD target achieved" #compare detector with original detector orig_det = im_set.get_detector() refined_det = refiner.get_experiments()[0].detector from scitbx import matrix import math for op, rp in zip(orig_det, refined_det): # compare the origin vectors by... o1 = matrix.col(op.get_origin()) o2 = matrix.col(rp.get_origin()) # ...their relative lengths assert approx_equal( math.fabs(o1.length() - o2.length()) / o1.length(), 0, eps=1e-5) # ...the angle between them assert approx_equal(o1.accute_angle(o2), 0, eps=1e-5) print "OK" return
from __future__ import division if __name__ == '__main__': import sys from dxtbx.datablock import DataBlockFactory from dxtbx.model import ParallaxCorrectedPxMmStrategy datablocks = DataBlockFactory.from_args(sys.argv[1:]) assert(len(datablocks) == 1) detectors = datablocks[0].unique_detectors() assert(len(detectors) == 1) detector = detectors[0] assert(len(detector) == 1) px_mm = detector[0].get_px_mm_strategy() assert(isinstance(px_mm, ParallaxCorrectedPxMmStrategy)) print "Mu: %f mm^-1 " % px_mm.mu() print "t0: %f mm" % px_mm.t0() from matplotlib import pylab from scitbx.array_family import flex image_size = detector[0].get_image_size()[::-1] xcorr = flex.double(flex.grid(image_size)) ycorr = flex.double(flex.grid(image_size)) pixel_size = detector[0].get_pixel_size() for j in range(xcorr.all()[0]): for i in range(xcorr.all()[1]): x1, y1 = detector[0].pixel_to_millimeter((i,j)) x0, y0 = i * pixel_size[0], j * pixel_size[1] xcorr[j,i] = x1 - x0 ycorr[j,i] = y1 - y0 vmin = min([flex.min(xcorr), flex.min(ycorr)])