def __init__(self,
fname,
cfgstr=None,
dpath=None,
appname='ubelt',
ext='.pkl',
meta=None,
verbose=None,
enabled=True,
log=None,
hasher='sha1',
protocol=2):
if verbose is None:
verbose = self.VERBOSE
if dpath is None: # pragma: no branch
from ubelt import util_platform
dpath = util_platform.ensure_app_cache_dir(appname)
util_path.ensuredir(dpath)
self.dpath = dpath
self.fname = fname
self.cfgstr = cfgstr
self.verbose = verbose
self.ext = ext
self.meta = meta
self.enabled = enabled and not self.FORCE_DISABLE
self.protocol = protocol
self.hasher = hasher
self.log = print if log is None else log
if len(self.ext) > 0 and self.ext[0] != '.':
raise ValueError('Please be explicit and use a dot in ext')
def save(self, data, cfgstr=None):
"""
Writes data to path specified by `self.fpath(cfgstr)`.
Metadata containing information about the cache will also be appended
to an adjacent file with the `.meta` suffix.
Args:
data (object): arbitrary pickleable object to be cached
cfgstr (str, optional): overrides the instance-level cfgstr
Example:
>>> from ubelt.util_cache import * # NOQA
>>> # Normal functioning
>>> cfgstr = 'long-cfg' * 32
>>> cacher = Cacher('test_enabled_save', cfgstr)
>>> cacher.save('data')
>>> assert exists(cacher.get_fpath()), 'should be enabeled'
>>> assert exists(cacher.get_fpath() + '.meta'), 'missing metadata'
>>> # Setting the cacher as enabled=False turns it off
>>> cacher2 = Cacher('test_disabled_save', 'params', enabled=False)
>>> cacher2.save('data')
>>> assert not exists(cacher2.get_fpath()), 'should be disabled'
"""
from six.moves import cPickle as pickle
from ubelt import util_path
from ubelt import util_time
if not self.enabled:
return
if self.verbose > 0:
self.log('[cacher] ... {} cache save'.format(self.fname))
cfgstr = self._rectify_cfgstr(cfgstr)
condensed = self._condense_cfgstr(cfgstr)
# Make sure the cache directory exists
util_path.ensuredir(self.dpath)
data_fpath = self.get_fpath(cfgstr=cfgstr)
meta_fpath = data_fpath + '.meta'
# Also save metadata file to reconstruct hashing
with open(meta_fpath, 'a') as file_:
# TODO: maybe append this in json or YML format?
file_.write('\n\nsaving {}\n'.format(util_time.timestamp()))
file_.write(self.fname + '\n')
file_.write(condensed + '\n')
file_.write(cfgstr + '\n')
file_.write(str(self.meta) + '\n')
with open(data_fpath, 'wb') as file_:
# Use protocol 2 to support python2 and 3
pickle.dump(data, file_, protocol=self.protocol)
if self.verbose > 3:
sizestr = _byte_str(os.stat(data_fpath).st_size)
self.log('[cacher] ... finish save, size={}'.format(sizestr))
def ensure_app_resource_dir(appname, *args):
"""
Calls `get_app_resource_dir` but ensures the directory exists.
SeeAlso:
get_app_resource_dir
Example:
>>> import ubelt as ub
>>> dpath = ub.ensure_app_resource_dir('ubelt')
>>> assert exists(dpath)
"""
dpath = get_app_resource_dir(appname, *args)
util_path.ensuredir(dpath)
return dpath
def ensure_app_cache_dir(appname, *args):
"""
Calls `get_app_cache_dir` but ensures the directory exists.
Args:
appname (str): the name of the application
*args: any other subdirectories may be specified
SeeAlso:
get_app_cache_dir
Example:
>>> import ubelt as ub
>>> dpath = ub.ensure_app_cache_dir('ubelt')
>>> assert exists(dpath)
"""
from ubelt import util_path
dpath = get_app_cache_dir(appname, *args)
util_path.ensuredir(dpath)
return dpath
def _win32_can_symlink(verbose=0, force=0, testing=0):
"""
Args:
verbose (int, default=0): flag
force (int, default=0): flag
testing (int, default=0): flag
Example:
>>> # xdoctest: +REQUIRES(WIN32)
>>> import ubelt as ub
>>> _win32_can_symlink(verbose=1, force=1, testing=1)
"""
global __win32_can_symlink__
if verbose:
print('__win32_can_symlink__ = {!r}'.format(__win32_can_symlink__))
if __win32_can_symlink__ is not None and not force:
return __win32_can_symlink__
from ubelt import util_platform
tempdir = util_platform.ensure_app_cache_dir('ubelt', '_win32_can_symlink')
util_io.delete(tempdir)
util_path.ensuredir(tempdir)
dpath = join(tempdir, 'dpath')
fpath = join(tempdir, 'fpath.txt')
dlink = join(tempdir, 'dlink')
flink = join(tempdir, 'flink.txt')
util_path.ensuredir(dpath)
util_io.touch(fpath)
# Add broken variants of the links for testing purposes
# Its ugly, but so is all this windows code.
if testing:
broken_dpath = join(tempdir, 'broken_dpath')
broken_fpath = join(tempdir, 'broken_fpath.txt')
# Create files that we will delete after we link to them
util_path.ensuredir(broken_dpath)
util_io.touch(broken_fpath)
try:
_win32_symlink(dpath, dlink)
if testing:
_win32_symlink(broken_dpath, join(tempdir, 'broken_dlink'))
can_symlink_directories = os.path.islink(dlink)
except OSError:
can_symlink_directories = False
if verbose:
print('can_symlink_directories = {!r}'.format(can_symlink_directories))
try:
_win32_symlink(fpath, flink)
if testing:
_win32_symlink(broken_fpath, join(tempdir, 'broken_flink'))
can_symlink_files = os.path.islink(flink)
# os.path.islink(flink)
except OSError:
can_symlink_files = False
if verbose:
print('can_symlink_files = {!r}'.format(can_symlink_files))
if int(can_symlink_directories) + int(can_symlink_files) == 1:
raise AssertionError(
'can do one but not both. Unexpected {} {}'.format(
can_symlink_directories, can_symlink_files))
try:
# test that we can create junctions, even if symlinks are disabled
djunc = _win32_junction(dpath, join(tempdir, 'djunc'))
fjunc = _win32_junction(fpath, join(tempdir, 'fjunc.txt'))
if testing:
_win32_junction(broken_dpath, join(tempdir, 'broken_djunc'))
_win32_junction(broken_fpath, join(tempdir, 'broken_fjunc.txt'))
if not _win32_is_junction(djunc):
raise AssertionError('expected junction')
if not _win32_is_hardlinked(fpath, fjunc):
raise AssertionError('expected hardlink')
except Exception:
warnings.warn('We cannot create junctions either!')
raise
if testing:
# break the links
util_io.delete(broken_dpath)
util_io.delete(broken_fpath)
if verbose:
from ubelt import util_links
util_links._dirstats(tempdir)
try:
# Cleanup the test directory
util_io.delete(tempdir)
except Exception:
print('ERROR IN DELETE')
from ubelt import util_links
util_links._dirstats(tempdir)
raise
can_symlink = can_symlink_directories and can_symlink_files
__win32_can_symlink__ = can_symlink
if not can_symlink:
warnings.warn('Cannot make real symlink. Falling back to junction')
if verbose:
print('can_symlink = {!r}'.format(can_symlink))
print('__win32_can_symlink__ = {!r}'.format(__win32_can_symlink__))
return can_symlink
