def test_resolve_path(): os.environ["HELLO_WORLD"] = "EXPANDED" new_path = os.path.join("~", "$HELLO_WORLD", "path") path = resolve_path(new_path) assert path == os.path.join(os.path.expanduser("~"), "EXPANDED", "path") new_path = os.path.join("$HELLO_WORLD", "path") path = resolve_path(new_path) assert path == os.path.abspath(os.path.join("EXPANDED", "path"))
def test_resolve_path(monkeypatch): monkeypatch.setenv("HELLO_WORLD", "EXPANDED") new_path = os.path.join("~", "$HELLO_WORLD", "path") path = resolve_path(new_path) assert path == os.path.join(os.path.expanduser("~"), "EXPANDED", "path") new_path = os.path.join("$HELLO_WORLD", "path") path = resolve_path(new_path) assert path == os.path.abspath(os.path.join("EXPANDED", "path"))
def _load_pickle_path(self, imageset_data, param): # type: (Dict, str) -> Tuple[Optional[str], Any] """ Read a filename from an imageset dict and load if required. Args: imageset_data: The dictionary holding imageset information param: The key name to lookup in the imageset dictionary Returns: A tuple of (filename, data) where data has been loaded from the pickle file. If there is no key entry then (None, None) is returned. If the configuration parameter check_format is False then (filename, None) will be returned. """ if param not in imageset_data: return "", None filename = resolve_path(imageset_data[param], directory=self._directory) if self._check_format and filename: with open(filename, "rb") as fh: if six.PY3: return filename, pickle.load(fh, encoding="bytes") else: return filename, pickle.load(fh) return filename or "", None
def _experimentlist_from_file(filename, directory=None): """Load a model dictionary from a file.""" filename = resolve_path(filename, directory=directory) try: with open(filename, "r") as infile: return json.load(infile, object_hook=_decode_dict) except IOError: raise IOError("unable to read file, %s" % filename)
def basic_imageset_from_dict(d, directory=None): """ Construct an ImageSet class from the dictionary.""" # Get the filename list and create the imageset filenames = [ resolve_path(str(p), directory=directory) for p in d["filenames"] ] imageset = ImageSetFactory.new(filenames)[0] # Set some external lookups if "mask" in d and d["mask"] is not None and d["mask"] != "": path = resolve_path(d["mask"], directory=directory) with open(path) as infile: imageset.external_lookup.mask.filename = path imageset.external_lookup.mask.data = ImageBool(pickle.load(infile)) if "gain" in d and d["gain"] is not None and d["gain"] != "": path = resolve_path(d["gain"], directory=directory) with open(path) as infile: imageset.external_lookup.gain.filename = path imageset.external_lookup.gain.data = ImageDouble( pickle.load(infile)) if "pedestal" in d and d["pedestal"] is not None and d["pedestal"] != "": path = resolve_path(d["pedestal"], directory=directory) with open(path) as infile: imageset.external_lookup.pedestal.filename = path imageset.external_lookup.pedestal.data = ImageDouble( pickle.load(infile)) # Get the existing models as dictionaries beam_dict = imageset.get_beam(0).to_dict() detector_dict = imageset.get_detector(0).to_dict() # Set models imageset.set_beam(BeamFactory.from_dict(d.get("beam"), beam_dict)) imageset.set_detector( DetectorFactory.from_dict(d.get("detector"), detector_dict)) # Return the imageset return imageset
def _make_stills(self, imageset, format_kwargs=None): """Make a still imageset.""" filenames = [ resolve_path(p, directory=self._directory) if not urlparse(p).scheme else p for p in imageset["images"] ] indices = None if "single_file_indices" in imageset: indices = imageset["single_file_indices"] assert len(indices) == len(filenames) return ImageSetFactory.make_imageset( filenames, None, check_format=self._check_format, single_file_indices=indices, format_kwargs=format_kwargs, )
def _make_sequence( self, imageset, beam=None, detector=None, goniometer=None, scan=None, format_kwargs=None, ): """Make an image sequence.""" # Get the template format template = resolve_path(imageset["template"], directory=self._directory) # Get the number of images (if no scan is given we'll try # to find all the images matching the template if scan is None: i0, i1 = template_image_range(template) else: i0, i1 = scan.get_image_range() format_class = None if self._check_format is False: if "single_file_indices" in imageset: format_class = FormatMultiImage # Make a sequence from the input data return ImageSetFactory.make_sequence( template, list(range(i0, i1 + 1)), format_class=format_class, check_format=self._check_format, beam=beam, detector=detector, goniometer=goniometer, scan=scan, format_kwargs=format_kwargs, )
def imagesequence_from_dict(d, check_format=True, directory=None): """Construct and image sequence from the dictionary.""" # Get the template (required) template = resolve_path(str(d["template"]), directory=directory) # If the scan isn't set, find all available files scan_dict = d.get("scan") if scan_dict is None: image_range = None else: image_range = scan_dict.get("image_range") # Set the models with the exisiting models as templates beam = BeamFactory.from_dict(d.get("beam")) goniometer = GoniometerFactory.from_dict(d.get("goniometer")) detector = DetectorFactory.from_dict(d.get("detector")) scan = ScanFactory.from_dict(d.get("scan")) # Construct the sequence try: sequence = ImageSetFactory.from_template( template, image_range, beam=beam, detector=detector, goniometer=goniometer, scan=scan, check_format=check_format, )[0] except Exception: indices = list(range(image_range[0], image_range[1] + 1)) sequence = ImageSetFactory.make_sequence( template, indices, beam=beam, detector=detector, goniometer=goniometer, scan=scan, check_format=check_format, ) # Set some external lookups if "mask" in d and d["mask"] is not None and d["mask"] != "": path = resolve_path(d["mask"], directory=directory) with open(path) as infile: sequence.external_lookup.mask.filename = path sequence.external_lookup.mask.data = ImageBool(pickle.load(infile)) if "gain" in d and d["gain"] is not None and d["gain"] != "": path = resolve_path(d["gain"], directory=directory) with open(path) as infile: sequence.external_lookup.gain.filename = path sequence.external_lookup.gain.data = ImageDouble( pickle.load(infile)) if "pedestal" in d and d["pedestal"] is not None and d["pedestal"] != "": path = resolve_path(d["pedestal"], directory=directory) with open(path) as infile: sequence.external_lookup.pedestal.filename = path sequence.external_lookup.pedestal.data = ImageDouble( pickle.load(infile)) return sequence
def test_load_path_deprecated(): os.environ["HELLO_WORLD"] = "EXPANDED" new_path = os.path.join("~", "$HELLO_WORLD", "path") with pytest.deprecated_call(): path = load_path(new_path) assert path == resolve_path(new_path)
def test_load_path_deprecated(monkeypatch): monkeypatch.setenv("HELLO_WORLD", "EXPANDED") new_path = os.path.join("~", "$HELLO_WORLD", "path") with pytest.deprecated_call(): path = load_path(new_path) assert path == resolve_path(new_path)
def datablocks_from_dict(obj, check_format=True, directory=None): """Get the datablocks from the dictionary.""" # If we have a list, extract for each dictionary in the list if isinstance(obj, list): return [ datablocks_from_dict(dd, check_format, directory) for dd in obj ] elif not isinstance(obj, dict): raise InvalidDataBlockError( "Unexpected datablock type {} instead of dict".format(type(obj))) # Make sure the id signature is correct if not obj.get("__id__") == "DataBlock": raise InvalidDataBlockError( "Expected __id__ 'DataBlock', but found {}".format( repr(obj.get("__id__")))) # Get the list of models blist = obj.get("beam", []) dlist = obj.get("detector", []) glist = obj.get("goniometer", []) slist = obj.get("scan", []) def load_models(obj): try: beam = dxtbx.model.BeamFactory.from_dict(blist[obj["beam"]]) except Exception: beam = None try: dobj = dlist[obj["detector"]] detector = dxtbx.model.DetectorFactory.from_dict(dobj) except Exception: detector = None try: gonio = dxtbx.model.GoniometerFactory.from_dict( glist[obj["goniometer"]]) except Exception: gonio = None try: scan = dxtbx.model.ScanFactory.from_dict(slist[obj["scan"]]) except Exception: scan = None return beam, detector, gonio, scan # Loop through all the imagesets imagesets = [] for imageset in obj["imageset"]: ident = imageset["__id__"] if "params" in imageset: format_kwargs = imageset["params"] else: format_kwargs = {} if ident == "ImageSequence" or ident == "ImageSweep": beam, detector, gonio, scan = load_models(imageset) if "template" in imageset: template = resolve_path(imageset["template"], directory=directory) i0, i1 = scan.get_image_range() iset = dxtbx.imageset.ImageSetFactory.make_sequence( template, list(range(i0, i1 + 1)), None, beam, detector, gonio, scan, check_format, format_kwargs=format_kwargs, ) if "mask" in imageset and imageset["mask"] is not None: imageset["mask"] = resolve_path(imageset["mask"], directory=directory) iset.external_lookup.mask.filename = imageset["mask"] if check_format: with open(imageset["mask"], "rb") as infile: iset.external_lookup.mask.data = ImageBool( pickle.load(infile, encoding="bytes")) if "gain" in imageset and imageset["gain"] is not None: imageset["gain"] = resolve_path(imageset["gain"], directory=directory) iset.external_lookup.gain.filename = imageset["gain"] if check_format: with open(imageset["gain"], "rb") as infile: iset.external_lookup.gain.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "pedestal" in imageset and imageset["pedestal"] is not None: imageset["pedestal"] = resolve_path(imageset["pedestal"], directory=directory) iset.external_lookup.pedestal.filename = imageset[ "pedestal"] if check_format: with open(imageset["pedestal"], "rb") as infile: iset.external_lookup.pedestal.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dx" in imageset and imageset["dx"] is not None: imageset["dx"] = resolve_path(imageset["dx"], directory=directory) iset.external_lookup.dx.filename = imageset["dx"] with open(imageset["dx"], "rb") as infile: iset.external_lookup.dx.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dy" in imageset and imageset["dy"] is not None: imageset["dy"] = resolve_path(imageset["dy"], directory=directory) iset.external_lookup.dy.filename = imageset["dy"] with open(imageset["dy"], "rb") as infile: iset.external_lookup.dy.data = ImageDouble( pickle.load(infile, encoding="bytes")) iset.update_detector_px_mm_data() elif "master" in imageset: template = resolve_path(imageset["master"], directory=directory) i0, i1 = scan.get_image_range() if not check_format: format_class = FormatMultiImage else: format_class = None iset = dxtbx.imageset.ImageSetFactory.make_sequence( template, list(range(i0, i1 + 1)), format_class=format_class, beam=beam, detector=detector, goniometer=gonio, scan=scan, check_format=check_format, format_kwargs=format_kwargs, ) if "mask" in imageset and imageset["mask"] is not None: imageset["mask"] = resolve_path(imageset["mask"], directory) iset.external_lookup.mask.filename = imageset["mask"] if check_format: with open(imageset["mask"], "rb") as infile: iset.external_lookup.mask.data = ImageBool( pickle.load(infile, encoding="bytes")) if "gain" in imageset and imageset["gain"] is not None: imageset["gain"] = resolve_path(imageset["gain"], directory) iset.external_lookup.gain.filename = imageset["gain"] if check_format: with open(imageset["gain"], "rb") as infile: iset.external_lookup.gain.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "pedestal" in imageset and imageset["pedestal"] is not None: imageset["pedestal"] = resolve_path( imageset["pedestal"], directory) iset.external_lookup.pedestal.filename = imageset[ "pedestal"] if check_format: with open(imageset["pedestal"], "rb") as infile: iset.external_lookup.pedestal.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dx" in imageset and imageset["dx"] is not None: imageset["dx"] = resolve_path(imageset["dx"], directory) iset.external_lookup.dx.filename = imageset["dx"] with open(imageset["dx"], "rb") as infile: iset.external_lookup.dx.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dy" in imageset and imageset["dy"] is not None: imageset["dy"] = resolve_path(imageset["dy"], directory) iset.external_lookup.dy.filename = imageset["dy"] with open(imageset["dy"], "rb") as infile: iset.external_lookup.dy.data = ImageDouble( pickle.load(infile, encoding="bytes")) iset.update_detector_px_mm_data() imagesets.append(iset) elif ident == "ImageSet" or ident == "ImageGrid": filenames = [image["filename"] for image in imageset["images"]] indices = [ image["image"] for image in imageset["images"] if "image" in image ] assert len(indices) == 0 or len(indices) == len(filenames) iset = dxtbx.imageset.ImageSetFactory.make_imageset( filenames, None, check_format, indices, format_kwargs=format_kwargs) if ident == "ImageGrid": grid_size = imageset["grid_size"] iset = dxtbx.imageset.ImageGrid.from_imageset(iset, grid_size) for i, image in enumerate(imageset["images"]): beam, detector, gonio, scan = load_models(image) iset.set_beam(beam, i) iset.set_detector(detector, i) iset.set_goniometer(gonio, i) iset.set_scan(scan, i) if "mask" in imageset and imageset["mask"] is not None: imageset["mask"] = resolve_path(imageset["mask"], directory) iset.external_lookup.mask.filename = imageset["mask"] if check_format: with open(imageset["mask"], "rb") as infile: iset.external_lookup.mask.data = ImageBool( pickle.load(infile, encoding="bytes")) if "gain" in imageset and imageset["gain"] is not None: imageset["gain"] = resolve_path(imageset["gain"], directory) iset.external_lookup.gain.filename = imageset["gain"] if check_format: with open(imageset["gain"], "rb") as infile: iset.external_lookup.gain.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "pedestal" in imageset and imageset["pedestal"] is not None: imageset["pedestal"] = resolve_path(imageset["pedestal"], directory) iset.external_lookup.pedestal.filename = imageset["pedestal"] if check_format: with open(imageset["pedestal"], "rb") as infile: iset.external_lookup.pedestal.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dx" in imageset and imageset["dx"] is not None: imageset["dx"] = resolve_path(imageset["dx"], directory) iset.external_lookup.dx.filename = imageset["dx"] with open(imageset["dx"], "rb") as infile: iset.external_lookup.dx.data = ImageDouble( pickle.load(infile, encoding="bytes")) if "dy" in imageset and imageset["dy"] is not None: imageset["dy"] = resolve_path(imageset["dy"], directory) iset.external_lookup.dy.filename = imageset["dy"] with open(imageset["dy"], "rb") as infile: iset.external_lookup.dy.data = ImageDouble( pickle.load(infile, encoding="bytes")) iset.update_detector_px_mm_data() imagesets.append(iset) else: raise RuntimeError("expected ImageSet/ImageSequence, got %s" % ident) return DataBlock(imagesets)