def __init__(self, fname, cfgstr=None, dpath=None, appname='ubelt', ext='.pkl', meta=None, verbose=None, enabled=True, log=None, hasher='sha1', protocol=2): if verbose is None: verbose = self.VERBOSE if dpath is None: # pragma: no branch from ubelt import util_platform dpath = util_platform.ensure_app_cache_dir(appname) util_path.ensuredir(dpath) self.dpath = dpath self.fname = fname self.cfgstr = cfgstr self.verbose = verbose self.ext = ext self.meta = meta self.enabled = enabled and not self.FORCE_DISABLE self.protocol = protocol self.hasher = hasher self.log = print if log is None else log if len(self.ext) > 0 and self.ext[0] != '.': raise ValueError('Please be explicit and use a dot in ext')
def save(self, data, cfgstr=None): """ Writes data to path specified by `self.fpath(cfgstr)`. Metadata containing information about the cache will also be appended to an adjacent file with the `.meta` suffix. Args: data (object): arbitrary pickleable object to be cached cfgstr (str, optional): overrides the instance-level cfgstr Example: >>> from ubelt.util_cache import * # NOQA >>> # Normal functioning >>> cfgstr = 'long-cfg' * 32 >>> cacher = Cacher('test_enabled_save', cfgstr) >>> cacher.save('data') >>> assert exists(cacher.get_fpath()), 'should be enabeled' >>> assert exists(cacher.get_fpath() + '.meta'), 'missing metadata' >>> # Setting the cacher as enabled=False turns it off >>> cacher2 = Cacher('test_disabled_save', 'params', enabled=False) >>> cacher2.save('data') >>> assert not exists(cacher2.get_fpath()), 'should be disabled' """ from six.moves import cPickle as pickle from ubelt import util_path from ubelt import util_time if not self.enabled: return if self.verbose > 0: self.log('[cacher] ... {} cache save'.format(self.fname)) cfgstr = self._rectify_cfgstr(cfgstr) condensed = self._condense_cfgstr(cfgstr) # Make sure the cache directory exists util_path.ensuredir(self.dpath) data_fpath = self.get_fpath(cfgstr=cfgstr) meta_fpath = data_fpath + '.meta' # Also save metadata file to reconstruct hashing with open(meta_fpath, 'a') as file_: # TODO: maybe append this in json or YML format? file_.write('\n\nsaving {}\n'.format(util_time.timestamp())) file_.write(self.fname + '\n') file_.write(condensed + '\n') file_.write(cfgstr + '\n') file_.write(str(self.meta) + '\n') with open(data_fpath, 'wb') as file_: # Use protocol 2 to support python2 and 3 pickle.dump(data, file_, protocol=self.protocol) if self.verbose > 3: sizestr = _byte_str(os.stat(data_fpath).st_size) self.log('[cacher] ... finish save, size={}'.format(sizestr))
def ensure_app_resource_dir(appname, *args): """ Calls `get_app_resource_dir` but ensures the directory exists. SeeAlso: get_app_resource_dir Example: >>> import ubelt as ub >>> dpath = ub.ensure_app_resource_dir('ubelt') >>> assert exists(dpath) """ dpath = get_app_resource_dir(appname, *args) util_path.ensuredir(dpath) return dpath
def ensure_app_cache_dir(appname, *args): """ Calls `get_app_cache_dir` but ensures the directory exists. Args: appname (str): the name of the application *args: any other subdirectories may be specified SeeAlso: get_app_cache_dir Example: >>> import ubelt as ub >>> dpath = ub.ensure_app_cache_dir('ubelt') >>> assert exists(dpath) """ from ubelt import util_path dpath = get_app_cache_dir(appname, *args) util_path.ensuredir(dpath) return dpath
def _win32_can_symlink(verbose=0, force=0, testing=0): """ Args: verbose (int, default=0): flag force (int, default=0): flag testing (int, default=0): flag Example: >>> # xdoctest: +REQUIRES(WIN32) >>> import ubelt as ub >>> _win32_can_symlink(verbose=1, force=1, testing=1) """ global __win32_can_symlink__ if verbose: print('__win32_can_symlink__ = {!r}'.format(__win32_can_symlink__)) if __win32_can_symlink__ is not None and not force: return __win32_can_symlink__ from ubelt import util_platform tempdir = util_platform.ensure_app_cache_dir('ubelt', '_win32_can_symlink') util_io.delete(tempdir) util_path.ensuredir(tempdir) dpath = join(tempdir, 'dpath') fpath = join(tempdir, 'fpath.txt') dlink = join(tempdir, 'dlink') flink = join(tempdir, 'flink.txt') util_path.ensuredir(dpath) util_io.touch(fpath) # Add broken variants of the links for testing purposes # Its ugly, but so is all this windows code. if testing: broken_dpath = join(tempdir, 'broken_dpath') broken_fpath = join(tempdir, 'broken_fpath.txt') # Create files that we will delete after we link to them util_path.ensuredir(broken_dpath) util_io.touch(broken_fpath) try: _win32_symlink(dpath, dlink) if testing: _win32_symlink(broken_dpath, join(tempdir, 'broken_dlink')) can_symlink_directories = os.path.islink(dlink) except OSError: can_symlink_directories = False if verbose: print('can_symlink_directories = {!r}'.format(can_symlink_directories)) try: _win32_symlink(fpath, flink) if testing: _win32_symlink(broken_fpath, join(tempdir, 'broken_flink')) can_symlink_files = os.path.islink(flink) # os.path.islink(flink) except OSError: can_symlink_files = False if verbose: print('can_symlink_files = {!r}'.format(can_symlink_files)) if int(can_symlink_directories) + int(can_symlink_files) == 1: raise AssertionError( 'can do one but not both. Unexpected {} {}'.format( can_symlink_directories, can_symlink_files)) try: # test that we can create junctions, even if symlinks are disabled djunc = _win32_junction(dpath, join(tempdir, 'djunc')) fjunc = _win32_junction(fpath, join(tempdir, 'fjunc.txt')) if testing: _win32_junction(broken_dpath, join(tempdir, 'broken_djunc')) _win32_junction(broken_fpath, join(tempdir, 'broken_fjunc.txt')) if not _win32_is_junction(djunc): raise AssertionError('expected junction') if not _win32_is_hardlinked(fpath, fjunc): raise AssertionError('expected hardlink') except Exception: warnings.warn('We cannot create junctions either!') raise if testing: # break the links util_io.delete(broken_dpath) util_io.delete(broken_fpath) if verbose: from ubelt import util_links util_links._dirstats(tempdir) try: # Cleanup the test directory util_io.delete(tempdir) except Exception: print('ERROR IN DELETE') from ubelt import util_links util_links._dirstats(tempdir) raise can_symlink = can_symlink_directories and can_symlink_files __win32_can_symlink__ = can_symlink if not can_symlink: warnings.warn('Cannot make real symlink. Falling back to junction') if verbose: print('can_symlink = {!r}'.format(can_symlink)) print('__win32_can_symlink__ = {!r}'.format(__win32_can_symlink__)) return can_symlink