def it_creates_an_indexed_pickle_and_reloads_it_on_all_properties():
path = "/tmp/__test.ipkl"
testme_orig = PropertiesClass()
utils.indexed_pickler_dump(testme_orig, path)
testme_new = PropertiesClass.__new__(PropertiesClass)
utils.indexed_pickler_load(path, None, testme_new)
for prop in ["a", "b", "c"]:
assert getattr(testme_orig, prop) == getattr(testme_new, prop)
os.unlink(path)
def it_creates_an_indexed_pickle_and_reloads_it():
path = "/tmp/__test.ipkl"
prop_list = ["a", "b"]
testme_orig = PropertiesClass()
utils.indexed_pickler_dump(testme_orig, path, prop_list)
testme_new = PropertiesClass.__new__(PropertiesClass)
utils.indexed_pickler_load(path, prop_list, testme_new)
for prop in prop_list:
assert getattr(testme_orig, prop) == getattr(testme_new, prop)
assert hasattr(testme_orig, "c")
assert not hasattr(testme_new, "c")
os.unlink(path)
def _aln_chcy_ims(self, field_i, key):
cache_key = (field_i, key)
if cache_key not in self._cache_ims:
filename = self._field_filename(field_i, is_debug=True)
self._cache_ims[cache_key] = utils.indexed_pickler_load(
filename, prop_list=key)
return self._cache_ims[cache_key]
def _load_field_prop(self, field_i, prop):
"""Mockpoint"""
if prop.startswith("_"):
name = local.path(self.debug_field_files[field_i]).name
else:
name = local.path(self.field_files[field_i]).name
return utils.indexed_pickler_load(self._folder / name, prop_list=prop)
def _has_prop(self, prop):
# Assume field 0 is representative of all fields
field_i = 0
name = local.path(self.field_files[field_i]).name
props = utils.indexed_pickler_load(self._folder / name,
prop_list=[prop],
skip_missing_props=True)
return prop in props.keys()
def __getattr__(self, key):
"""
Called when the attribute is not found.
This is tricky because this class inherits from Munch
which also overloads __getattr__.
We first ask the munch if it has the attr,
if not then we attempt to load it from the indexed pickle
and then as Munch to try again.
"""
try:
val = Munch.__getattr__(self, key)
if isinstance(val, ArrayResult):
return val.arr()
return val
except AttributeError:
# Bypass munch for these elements...
_folder = object.__getattribute__(self, "_folder")
filename = object.__getattribute__(self, "filename")
with local.cwd(_folder):
try:
loaded = utils.indexed_pickler_load(
filename, prop_list=[key], skip_missing_props=True,
)
if isinstance(loaded[key], ArrayResult):
setattr(self, key, loaded[key].arr())
else:
setattr(self, key, loaded[key])
except FileNotFoundError:
# If the pickle has not yet been saved then return the
# value if it is present
pass
# Try again
try:
return Munch.__getattr__(self, key)
except AttributeError:
# (An attempt to deal with attributes that get added to
# task Results but that can be allowed to be None -
# and gracefully allowing us to load versions of the
# Result klass in which this attribute is missing.)
#
# The attribute is also not in the pickle. If this
# attribute is required and allowed to be None, set
# it so, else raise.
required_props = object.__getattribute__(self, "required_props")
types = required_props.get(key, None)
if isinstance(types, tuple) and type(None) in types:
self[key] = None
return None
# key not found or not allowed to be None:
raise AttributeError
def load_from_folder(cls, folder, prop_list=None):
"""
Used when one task needs the result of a previous stage.
Pass in prop_list to load only certain properties.
The _folder property is treated specially, it is not
part of the Munch
"""
folder = local.path(folder)
with local.cwd(folder):
inst = cls(folder, is_loaded_result=True)
loaded = utils.indexed_pickler_load(folder / cls.filename,
prop_list=prop_list)
for key in loaded:
if isinstance(loaded[key], ArrayResult):
setattr(inst, key, loaded[key].arr())
else:
setattr(inst, key, loaded[key])
object.__setattr__(inst, "_folder", folder)
return inst
def aln_chcy_ims(self, field_i):
if field_i not in self._cache_aln_chcy_ims:
filename = self._field_filename(field_i, is_debug=True)
self._cache_aln_chcy_ims[field_i] = utils.indexed_pickler_load(
filename, prop_list="_aln_chcy_ims")
return self._cache_aln_chcy_ims[field_i]
def _load_field_prop(self, field_i, prop):
"""Mockpoint"""
name = local.path(self.field_files[field_i]).name
return utils.indexed_pickler_load(self._folder / name, prop_list=prop)
def aligned_composite_bg_removed_im(self, field_i):
filename = self._field_filename(field_i, is_debug=True)
return utils.indexed_pickler_load(
filename, prop_list="_aligned_composite_bg_removed_im")
def raw_chcy_ims(self, field_i):
filename = self._field_filename(field_i, is_debug=True)
return utils.indexed_pickler_load(filename, prop_list="_raw_chcy_ims")