def test_resolve_path():
os.environ["HELLO_WORLD"] = "EXPANDED"
new_path = os.path.join("~", "$HELLO_WORLD", "path")
path = resolve_path(new_path)
assert path == os.path.join(os.path.expanduser("~"), "EXPANDED", "path")
new_path = os.path.join("$HELLO_WORLD", "path")
path = resolve_path(new_path)
assert path == os.path.abspath(os.path.join("EXPANDED", "path"))
def test_resolve_path(monkeypatch):
monkeypatch.setenv("HELLO_WORLD", "EXPANDED")
new_path = os.path.join("~", "$HELLO_WORLD", "path")
path = resolve_path(new_path)
assert path == os.path.join(os.path.expanduser("~"), "EXPANDED", "path")
new_path = os.path.join("$HELLO_WORLD", "path")
path = resolve_path(new_path)
assert path == os.path.abspath(os.path.join("EXPANDED", "path"))
def _load_pickle_path(self, imageset_data, param):
# type: (Dict, str) -> Tuple[Optional[str], Any]
"""
Read a filename from an imageset dict and load if required.
Args:
imageset_data: The dictionary holding imageset information
param: The key name to lookup in the imageset dictionary
Returns:
A tuple of (filename, data) where data has been loaded from
the pickle file. If there is no key entry then (None, None)
is returned. If the configuration parameter check_format is
False then (filename, None) will be returned.
"""
if param not in imageset_data:
return "", None
filename = resolve_path(imageset_data[param], directory=self._directory)
if self._check_format and filename:
with open(filename, "rb") as fh:
if six.PY3:
return filename, pickle.load(fh, encoding="bytes")
else:
return filename, pickle.load(fh)
return filename or "", None
def _experimentlist_from_file(filename, directory=None):
"""Load a model dictionary from a file."""
filename = resolve_path(filename, directory=directory)
try:
with open(filename, "r") as infile:
return json.load(infile, object_hook=_decode_dict)
except IOError:
raise IOError("unable to read file, %s" % filename)
def basic_imageset_from_dict(d, directory=None):
""" Construct an ImageSet class from the dictionary."""
# Get the filename list and create the imageset
filenames = [
resolve_path(str(p), directory=directory) for p in d["filenames"]
]
imageset = ImageSetFactory.new(filenames)[0]
# Set some external lookups
if "mask" in d and d["mask"] is not None and d["mask"] != "":
path = resolve_path(d["mask"], directory=directory)
with open(path) as infile:
imageset.external_lookup.mask.filename = path
imageset.external_lookup.mask.data = ImageBool(pickle.load(infile))
if "gain" in d and d["gain"] is not None and d["gain"] != "":
path = resolve_path(d["gain"], directory=directory)
with open(path) as infile:
imageset.external_lookup.gain.filename = path
imageset.external_lookup.gain.data = ImageDouble(
pickle.load(infile))
if "pedestal" in d and d["pedestal"] is not None and d["pedestal"] != "":
path = resolve_path(d["pedestal"], directory=directory)
with open(path) as infile:
imageset.external_lookup.pedestal.filename = path
imageset.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile))
# Get the existing models as dictionaries
beam_dict = imageset.get_beam(0).to_dict()
detector_dict = imageset.get_detector(0).to_dict()
# Set models
imageset.set_beam(BeamFactory.from_dict(d.get("beam"), beam_dict))
imageset.set_detector(
DetectorFactory.from_dict(d.get("detector"), detector_dict))
# Return the imageset
return imageset
def _make_stills(self, imageset, format_kwargs=None):
"""Make a still imageset."""
filenames = [
resolve_path(p, directory=self._directory) if not urlparse(p).scheme else p
for p in imageset["images"]
]
indices = None
if "single_file_indices" in imageset:
indices = imageset["single_file_indices"]
assert len(indices) == len(filenames)
return ImageSetFactory.make_imageset(
filenames,
None,
check_format=self._check_format,
single_file_indices=indices,
format_kwargs=format_kwargs,
)
def _make_sequence(
self,
imageset,
beam=None,
detector=None,
goniometer=None,
scan=None,
format_kwargs=None,
):
"""Make an image sequence."""
# Get the template format
template = resolve_path(imageset["template"],
directory=self._directory)
# Get the number of images (if no scan is given we'll try
# to find all the images matching the template
if scan is None:
i0, i1 = template_image_range(template)
else:
i0, i1 = scan.get_image_range()
format_class = None
if self._check_format is False:
if "single_file_indices" in imageset:
format_class = FormatMultiImage
# Make a sequence from the input data
return ImageSetFactory.make_sequence(
template,
list(range(i0, i1 + 1)),
format_class=format_class,
check_format=self._check_format,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
format_kwargs=format_kwargs,
)
def imagesequence_from_dict(d, check_format=True, directory=None):
"""Construct and image sequence from the dictionary."""
# Get the template (required)
template = resolve_path(str(d["template"]), directory=directory)
# If the scan isn't set, find all available files
scan_dict = d.get("scan")
if scan_dict is None:
image_range = None
else:
image_range = scan_dict.get("image_range")
# Set the models with the exisiting models as templates
beam = BeamFactory.from_dict(d.get("beam"))
goniometer = GoniometerFactory.from_dict(d.get("goniometer"))
detector = DetectorFactory.from_dict(d.get("detector"))
scan = ScanFactory.from_dict(d.get("scan"))
# Construct the sequence
try:
sequence = ImageSetFactory.from_template(
template,
image_range,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format,
)[0]
except Exception:
indices = list(range(image_range[0], image_range[1] + 1))
sequence = ImageSetFactory.make_sequence(
template,
indices,
beam=beam,
detector=detector,
goniometer=goniometer,
scan=scan,
check_format=check_format,
)
# Set some external lookups
if "mask" in d and d["mask"] is not None and d["mask"] != "":
path = resolve_path(d["mask"], directory=directory)
with open(path) as infile:
sequence.external_lookup.mask.filename = path
sequence.external_lookup.mask.data = ImageBool(pickle.load(infile))
if "gain" in d and d["gain"] is not None and d["gain"] != "":
path = resolve_path(d["gain"], directory=directory)
with open(path) as infile:
sequence.external_lookup.gain.filename = path
sequence.external_lookup.gain.data = ImageDouble(
pickle.load(infile))
if "pedestal" in d and d["pedestal"] is not None and d["pedestal"] != "":
path = resolve_path(d["pedestal"], directory=directory)
with open(path) as infile:
sequence.external_lookup.pedestal.filename = path
sequence.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile))
return sequence
def test_load_path_deprecated():
os.environ["HELLO_WORLD"] = "EXPANDED"
new_path = os.path.join("~", "$HELLO_WORLD", "path")
with pytest.deprecated_call():
path = load_path(new_path)
assert path == resolve_path(new_path)
def test_load_path_deprecated(monkeypatch):
monkeypatch.setenv("HELLO_WORLD", "EXPANDED")
new_path = os.path.join("~", "$HELLO_WORLD", "path")
with pytest.deprecated_call():
path = load_path(new_path)
assert path == resolve_path(new_path)
def datablocks_from_dict(obj, check_format=True, directory=None):
"""Get the datablocks from the dictionary."""
# If we have a list, extract for each dictionary in the list
if isinstance(obj, list):
return [
datablocks_from_dict(dd, check_format, directory) for dd in obj
]
elif not isinstance(obj, dict):
raise InvalidDataBlockError(
"Unexpected datablock type {} instead of dict".format(type(obj)))
# Make sure the id signature is correct
if not obj.get("__id__") == "DataBlock":
raise InvalidDataBlockError(
"Expected __id__ 'DataBlock', but found {}".format(
repr(obj.get("__id__"))))
# Get the list of models
blist = obj.get("beam", [])
dlist = obj.get("detector", [])
glist = obj.get("goniometer", [])
slist = obj.get("scan", [])
def load_models(obj):
try:
beam = dxtbx.model.BeamFactory.from_dict(blist[obj["beam"]])
except Exception:
beam = None
try:
dobj = dlist[obj["detector"]]
detector = dxtbx.model.DetectorFactory.from_dict(dobj)
except Exception:
detector = None
try:
gonio = dxtbx.model.GoniometerFactory.from_dict(
glist[obj["goniometer"]])
except Exception:
gonio = None
try:
scan = dxtbx.model.ScanFactory.from_dict(slist[obj["scan"]])
except Exception:
scan = None
return beam, detector, gonio, scan
# Loop through all the imagesets
imagesets = []
for imageset in obj["imageset"]:
ident = imageset["__id__"]
if "params" in imageset:
format_kwargs = imageset["params"]
else:
format_kwargs = {}
if ident == "ImageSequence" or ident == "ImageSweep":
beam, detector, gonio, scan = load_models(imageset)
if "template" in imageset:
template = resolve_path(imageset["template"],
directory=directory)
i0, i1 = scan.get_image_range()
iset = dxtbx.imageset.ImageSetFactory.make_sequence(
template,
list(range(i0, i1 + 1)),
None,
beam,
detector,
gonio,
scan,
check_format,
format_kwargs=format_kwargs,
)
if "mask" in imageset and imageset["mask"] is not None:
imageset["mask"] = resolve_path(imageset["mask"],
directory=directory)
iset.external_lookup.mask.filename = imageset["mask"]
if check_format:
with open(imageset["mask"], "rb") as infile:
iset.external_lookup.mask.data = ImageBool(
pickle.load(infile, encoding="bytes"))
if "gain" in imageset and imageset["gain"] is not None:
imageset["gain"] = resolve_path(imageset["gain"],
directory=directory)
iset.external_lookup.gain.filename = imageset["gain"]
if check_format:
with open(imageset["gain"], "rb") as infile:
iset.external_lookup.gain.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "pedestal" in imageset and imageset["pedestal"] is not None:
imageset["pedestal"] = resolve_path(imageset["pedestal"],
directory=directory)
iset.external_lookup.pedestal.filename = imageset[
"pedestal"]
if check_format:
with open(imageset["pedestal"], "rb") as infile:
iset.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dx" in imageset and imageset["dx"] is not None:
imageset["dx"] = resolve_path(imageset["dx"],
directory=directory)
iset.external_lookup.dx.filename = imageset["dx"]
with open(imageset["dx"], "rb") as infile:
iset.external_lookup.dx.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dy" in imageset and imageset["dy"] is not None:
imageset["dy"] = resolve_path(imageset["dy"],
directory=directory)
iset.external_lookup.dy.filename = imageset["dy"]
with open(imageset["dy"], "rb") as infile:
iset.external_lookup.dy.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
iset.update_detector_px_mm_data()
elif "master" in imageset:
template = resolve_path(imageset["master"],
directory=directory)
i0, i1 = scan.get_image_range()
if not check_format:
format_class = FormatMultiImage
else:
format_class = None
iset = dxtbx.imageset.ImageSetFactory.make_sequence(
template,
list(range(i0, i1 + 1)),
format_class=format_class,
beam=beam,
detector=detector,
goniometer=gonio,
scan=scan,
check_format=check_format,
format_kwargs=format_kwargs,
)
if "mask" in imageset and imageset["mask"] is not None:
imageset["mask"] = resolve_path(imageset["mask"],
directory)
iset.external_lookup.mask.filename = imageset["mask"]
if check_format:
with open(imageset["mask"], "rb") as infile:
iset.external_lookup.mask.data = ImageBool(
pickle.load(infile, encoding="bytes"))
if "gain" in imageset and imageset["gain"] is not None:
imageset["gain"] = resolve_path(imageset["gain"],
directory)
iset.external_lookup.gain.filename = imageset["gain"]
if check_format:
with open(imageset["gain"], "rb") as infile:
iset.external_lookup.gain.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "pedestal" in imageset and imageset["pedestal"] is not None:
imageset["pedestal"] = resolve_path(
imageset["pedestal"], directory)
iset.external_lookup.pedestal.filename = imageset[
"pedestal"]
if check_format:
with open(imageset["pedestal"], "rb") as infile:
iset.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dx" in imageset and imageset["dx"] is not None:
imageset["dx"] = resolve_path(imageset["dx"], directory)
iset.external_lookup.dx.filename = imageset["dx"]
with open(imageset["dx"], "rb") as infile:
iset.external_lookup.dx.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dy" in imageset and imageset["dy"] is not None:
imageset["dy"] = resolve_path(imageset["dy"], directory)
iset.external_lookup.dy.filename = imageset["dy"]
with open(imageset["dy"], "rb") as infile:
iset.external_lookup.dy.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
iset.update_detector_px_mm_data()
imagesets.append(iset)
elif ident == "ImageSet" or ident == "ImageGrid":
filenames = [image["filename"] for image in imageset["images"]]
indices = [
image["image"] for image in imageset["images"]
if "image" in image
]
assert len(indices) == 0 or len(indices) == len(filenames)
iset = dxtbx.imageset.ImageSetFactory.make_imageset(
filenames,
None,
check_format,
indices,
format_kwargs=format_kwargs)
if ident == "ImageGrid":
grid_size = imageset["grid_size"]
iset = dxtbx.imageset.ImageGrid.from_imageset(iset, grid_size)
for i, image in enumerate(imageset["images"]):
beam, detector, gonio, scan = load_models(image)
iset.set_beam(beam, i)
iset.set_detector(detector, i)
iset.set_goniometer(gonio, i)
iset.set_scan(scan, i)
if "mask" in imageset and imageset["mask"] is not None:
imageset["mask"] = resolve_path(imageset["mask"], directory)
iset.external_lookup.mask.filename = imageset["mask"]
if check_format:
with open(imageset["mask"], "rb") as infile:
iset.external_lookup.mask.data = ImageBool(
pickle.load(infile, encoding="bytes"))
if "gain" in imageset and imageset["gain"] is not None:
imageset["gain"] = resolve_path(imageset["gain"], directory)
iset.external_lookup.gain.filename = imageset["gain"]
if check_format:
with open(imageset["gain"], "rb") as infile:
iset.external_lookup.gain.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "pedestal" in imageset and imageset["pedestal"] is not None:
imageset["pedestal"] = resolve_path(imageset["pedestal"],
directory)
iset.external_lookup.pedestal.filename = imageset["pedestal"]
if check_format:
with open(imageset["pedestal"], "rb") as infile:
iset.external_lookup.pedestal.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dx" in imageset and imageset["dx"] is not None:
imageset["dx"] = resolve_path(imageset["dx"], directory)
iset.external_lookup.dx.filename = imageset["dx"]
with open(imageset["dx"], "rb") as infile:
iset.external_lookup.dx.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
if "dy" in imageset and imageset["dy"] is not None:
imageset["dy"] = resolve_path(imageset["dy"], directory)
iset.external_lookup.dy.filename = imageset["dy"]
with open(imageset["dy"], "rb") as infile:
iset.external_lookup.dy.data = ImageDouble(
pickle.load(infile, encoding="bytes"))
iset.update_detector_px_mm_data()
imagesets.append(iset)
else:
raise RuntimeError("expected ImageSet/ImageSequence, got %s" %
ident)
return DataBlock(imagesets)
