def it_creates_an_indexed_pickle_and_reloads_it_on_all_properties(): path = "/tmp/__test.ipkl" testme_orig = PropertiesClass() utils.indexed_pickler_dump(testme_orig, path) testme_new = PropertiesClass.__new__(PropertiesClass) utils.indexed_pickler_load(path, None, testme_new) for prop in ["a", "b", "c"]: assert getattr(testme_orig, prop) == getattr(testme_new, prop) os.unlink(path)
def it_creates_an_indexed_pickle_and_reloads_it(): path = "/tmp/__test.ipkl" prop_list = ["a", "b"] testme_orig = PropertiesClass() utils.indexed_pickler_dump(testme_orig, path, prop_list) testme_new = PropertiesClass.__new__(PropertiesClass) utils.indexed_pickler_load(path, prop_list, testme_new) for prop in prop_list: assert getattr(testme_orig, prop) == getattr(testme_new, prop) assert hasattr(testme_orig, "c") assert not hasattr(testme_new, "c") os.unlink(path)
def _aln_chcy_ims(self, field_i, key): cache_key = (field_i, key) if cache_key not in self._cache_ims: filename = self._field_filename(field_i, is_debug=True) self._cache_ims[cache_key] = utils.indexed_pickler_load( filename, prop_list=key) return self._cache_ims[cache_key]
def _load_field_prop(self, field_i, prop): """Mockpoint""" if prop.startswith("_"): name = local.path(self.debug_field_files[field_i]).name else: name = local.path(self.field_files[field_i]).name return utils.indexed_pickler_load(self._folder / name, prop_list=prop)
def _has_prop(self, prop): # Assume field 0 is representative of all fields field_i = 0 name = local.path(self.field_files[field_i]).name props = utils.indexed_pickler_load(self._folder / name, prop_list=[prop], skip_missing_props=True) return prop in props.keys()
def __getattr__(self, key): """ Called when the attribute is not found. This is tricky because this class inherits from Munch which also overloads __getattr__. We first ask the munch if it has the attr, if not then we attempt to load it from the indexed pickle and then as Munch to try again. """ try: val = Munch.__getattr__(self, key) if isinstance(val, ArrayResult): return val.arr() return val except AttributeError: # Bypass munch for these elements... _folder = object.__getattribute__(self, "_folder") filename = object.__getattribute__(self, "filename") with local.cwd(_folder): try: loaded = utils.indexed_pickler_load( filename, prop_list=[key], skip_missing_props=True, ) if isinstance(loaded[key], ArrayResult): setattr(self, key, loaded[key].arr()) else: setattr(self, key, loaded[key]) except FileNotFoundError: # If the pickle has not yet been saved then return the # value if it is present pass # Try again try: return Munch.__getattr__(self, key) except AttributeError: # (An attempt to deal with attributes that get added to # task Results but that can be allowed to be None - # and gracefully allowing us to load versions of the # Result klass in which this attribute is missing.) # # The attribute is also not in the pickle. If this # attribute is required and allowed to be None, set # it so, else raise. required_props = object.__getattribute__(self, "required_props") types = required_props.get(key, None) if isinstance(types, tuple) and type(None) in types: self[key] = None return None # key not found or not allowed to be None: raise AttributeError
def load_from_folder(cls, folder, prop_list=None): """ Used when one task needs the result of a previous stage. Pass in prop_list to load only certain properties. The _folder property is treated specially, it is not part of the Munch """ folder = local.path(folder) with local.cwd(folder): inst = cls(folder, is_loaded_result=True) loaded = utils.indexed_pickler_load(folder / cls.filename, prop_list=prop_list) for key in loaded: if isinstance(loaded[key], ArrayResult): setattr(inst, key, loaded[key].arr()) else: setattr(inst, key, loaded[key]) object.__setattr__(inst, "_folder", folder) return inst
def aln_chcy_ims(self, field_i): if field_i not in self._cache_aln_chcy_ims: filename = self._field_filename(field_i, is_debug=True) self._cache_aln_chcy_ims[field_i] = utils.indexed_pickler_load( filename, prop_list="_aln_chcy_ims") return self._cache_aln_chcy_ims[field_i]
def _load_field_prop(self, field_i, prop): """Mockpoint""" name = local.path(self.field_files[field_i]).name return utils.indexed_pickler_load(self._folder / name, prop_list=prop)
def aligned_composite_bg_removed_im(self, field_i): filename = self._field_filename(field_i, is_debug=True) return utils.indexed_pickler_load( filename, prop_list="_aligned_composite_bg_removed_im")
def raw_chcy_ims(self, field_i): filename = self._field_filename(field_i, is_debug=True) return utils.indexed_pickler_load(filename, prop_list="_raw_chcy_ims")