def freeze(target_object: object, file_path: str, stage: str = None) -> None: """ Save the given object int the specified file path. When HIDEOUT_ENABLE_CACHE is set to False or HIDEOUT_SKIP_STAGE contains given stage name, the object is not save in to the cache file. Parameters ---------- target_object : object The object is save into the given file path. file_path Path to save given object. stage Stage name in the program. Returns ------- """ if env.HIDEOUT_ENABLE_CACHE is False or stage in env.HIDEOUT_SKIP_STAGES: logger.info("skip saving to cache file ...") return if not os.path.exists(env.HIDEOUT_CACHE_DIR): os.makedirs(env.HIDEOUT_CACHE_DIR, exist_ok=True) logger.info("saving {}".format(file_path)) with open(file_path, mode='wb') as f: return pickle.dump(target_object, f)
def __init__(self, stage: str = None): """ Constructor. Parameters ---------- stage : str Stage of program. """ self.stage = stage logger.info("HIDEOUT_ENABLE_CACHE is set to {}...".format(env.HIDEOUT_ENABLE_CACHE)) logger.info("HIDEOUT_SKIP_STAGES is set to {}...".format(env.HIDEOUT_SKIP_STAGES))
def resume_or_generate(self, func: Callable, func_args: Dict, label: str = None) -> object: """ Returns the object generated from func with func_args parameters. When exist the cache file containing target object, resume method does not given func and load the object from cache file. Parameters ---------- func : Callable function to generate result object func_args : Dict parameters of the func. label : str Prefix of the cache file generated by hideout. Returns ------- object : object """ file_path = generate_path(func, func_args, label) if env.HIDEOUT_ENABLE_CACHE and self.stage not in env.HIDEOUT_SKIP_STAGES: logger.info("specified cache file: {}".format(file_path)) if os.path.exists(file_path): logger.info("found {} ...".format(file_path)) with open(file_path, mode='rb') as f: return pickle.load(f) else: logger.info("not found cache file...") logger.info("generating with func") return self._generate_from_function(file_path, func, func_args)
def remove_all(base_dir: str = env.HIDEOUT_CACHE_DIR): """ Remove all cache files. Parameters ---------- base_dir : str directory containing cache files Returns ------- None : None """ files = glob.glob("{}/*.pickle".format(base_dir)) logger.info("removing all pickles in {}".format(base_dir)) for f in files: os.remove(f)
def _postprocess(self, result, file_path) -> None: if result is not None: logger.info("freezing object to {} ...".format(file_path)) freeze(result, file_path, self.stage) else: raise RuntimeError("Any object is loaded ...")