def file_by_mapping(self, srcdirpath):
''' Examine the `{TAGGER_TAG_PREFIX_DEFAULT}.file_by` tag for `srcdirpath`.
Return a mapping of specific tag values to filing locations
derived via `per_tag_auto_file_map`.
The file location specification in the tag may be a list or a string
(for convenient single locations).
For example, I might tag my downloads directory with:
{TAGGER_TAG_PREFIX_DEFAULT}.file_by={{"abn":"~/them/me"}}
indicating that files with an `abn` tag may be filed in the `~/them/me` directory.
That directory is then walked looking for the tag `abn`,
and wherever some tag `abn=`*value*` is found on a subdirectory
a mapping entry for `abn=`*value*=>*subdirectory* is added.
This results in a direct mapping of specific tag values to filing locations,
such as:
{{ Tag('abn','***********') => ['/path/to/them/me/abn-**-***-***-***'] }}
because the target subdirectory has been tagged with `abn="***********"`.
'''
assert isdirpath(srcdirpath)
assert not srcdirpath.startswith('~')
assert '~' not in srcdirpath
fstags = self.fstags
tagged = fstags[srcdirpath]
key = tagged.filepath
try:
mapping = self._file_by_mappings[key]
except KeyError:
mapping = defaultdict(set)
file_by = self.conf_tag(fstags[srcdirpath].all_tags, 'file_by', {})
# group the tags by file_by target path
grouped = defaultdict(set)
for tag_name, file_to in file_by.items():
if isinstance(file_to, str):
file_to = (file_to, )
for file_to_path in file_to:
if not isabspath(file_to_path):
if file_to_path.startswith('~'):
file_to_path = expanduser(file_to_path)
assert isabspath(file_to_path)
else:
file_to_path = joinpath(srcdirpath, file_to_path)
file_to_path = realpath(file_to_path)
grouped[file_to_path].add(tag_name)
# walk each path for its tag_names of interest
for file_to_path, tag_names in sorted(grouped.items()):
with Pfx("%r:%r", file_to_path, tag_names):
# accrue destination paths by tag values
for bare_key, dstpaths in self.per_tag_auto_file_map(
file_to_path, tag_names).items():
mapping[bare_key].update(dstpaths)
self._file_by_mappings[key] = mapping
return mapping
def expand_path(path, basedir=None):
''' Expand a path specification.
'''
path = expanduser(path)
if basedir and not isabspath(path):
path = joinpath(basedir, path)
return path
def _run(self, *calargv, subp_options=None):
''' Run a Calibre utility command.
Parameters:
* `calargv`: an iterable of the calibre command to issue;
if the command name is not an absolute path
it is expected to come from `self.CALIBRE_BINDIR_DEFAULT`
* `subp_options`: optional mapping of keyword arguments
to pass to `subprocess.run`
'''
X("calargv=%r", calargv)
if subp_options is None:
subp_options = {}
subp_options.setdefault('check', True)
cmd, *calargv = calargv
if not isabspath(cmd):
cmd = joinpath(self.CALIBRE_BINDIR_DEFAULT, cmd)
print("RUN", cmd, *calargv)
try:
cp = pfx_call(run, [cmd, *calargv], **subp_options)
except CalledProcessError as cpe:
error(
"run fails, exit code %s:\n %s",
cpe.returncode,
' '.join(map(repr, cpe.cmd)),
)
if cpe.stderr:
print(cpe.stderr.replace('\n', ' \n'), file=sys.stderr)
raise
return cp
def datafile_Store(
self,
store_name,
clause_name,
*,
path=None,
basedir=None,
hashclass=None,
):
''' Construct a VTDStore from a "datafile" clause.
'''
if basedir is None:
basedir = self.get_default('basedir')
if path is None:
path = clause_name
path = longpath(path)
if not isabspath(path):
if path.startswith('./'):
path = abspath(path)
else:
if basedir is None:
raise ValueError('relative path %r but no basedir' %
(path, ))
basedir = longpath(basedir)
path = joinpath(basedir, path)
return VTDStore(store_name, path, hashclass=hashclass)
def datadir_Store(
self,
store_name,
clause_name,
*,
path=None,
basedir=None,
hashclass=None,
raw=False,
):
''' Construct a DataDirStore from a "datadir" clause.
'''
if basedir is None:
basedir = self.get_default('basedir')
if path is None:
path = clause_name
path = longpath(path)
if not isabspath(path):
if path.startswith('./'):
path = abspath(path)
else:
if basedir is None:
raise ValueError('relative path %r but no basedir' %
(path, ))
basedir = longpath(basedir)
path = joinpath(basedir, path)
if isinstance(raw, str):
raw = truthy_word(raw)
return DataDirStore(store_name, path, hashclass=hashclass, raw=raw)
class HasFSPath:
''' An object with a `.fspath` attribute representing a filesystem location.
'''
@require(lambda fspath: isabspath(fspath))
def __init__(self, fspath):
self.fspath = fspath
def pathto(self, subpath):
''' The full path to `subpath`, a relative path below `self.fspath`.
'''
return joinpath(self.fspath, subpath)
def platonic_Store(
self,
store_name,
clause_name,
*,
path=None,
basedir=None,
follow_symlinks=False,
meta=None,
archive=None,
hashclass=None,
):
''' Construct a PlatonicStore from a "datadir" clause.
'''
if basedir is None:
basedir = self.get_default('basedir')
if path is None:
path = clause_name
debug("path from clausename: %r", path)
path = longpath(path)
debug("longpath(path) ==> %r", path)
if not isabspath(path):
if path.startswith('./'):
path = abspath(path)
debug("abspath ==> %r", path)
else:
if basedir is None:
raise ValueError('relative path %r but no basedir' %
(path, ))
basedir = longpath(basedir)
debug("longpath(basedir) ==> %r", basedir)
path = joinpath(basedir, path)
debug("path ==> %r", path)
if follow_symlinks is None:
follow_symlinks = False
if meta is None:
meta_store = None
elif isinstance(meta, str):
meta_store = Store(meta, self)
if isinstance(archive, str):
archive = longpath(archive)
return PlatonicStore(
store_name,
path,
hashclass=hashclass,
indexclass=None,
follow_symlinks=follow_symlinks,
meta_store=meta_store,
archive=archive,
flags_prefix='VT_' + clause_name,
)
def filecache_Store(
self,
store_name,
clause_name,
*,
path=None,
max_files=None,
max_file_size=None,
basedir=None,
backend=None,
hashclass=None,
):
''' Construct a FileCacheStore from a "filecache" clause.
'''
if basedir is None:
basedir = self.get_default('basedir')
if path is None:
path = clause_name
debug("path from clausename: %r", path)
path = longpath(path)
debug("longpath(path) ==> %r", path)
if isinstance(max_files, str):
max_files = scaled_value(max_files)
if isinstance(max_file_size, str):
max_file_size = scaled_value(max_file_size)
if backend is None:
backend_store = None
else:
backend_store = self.Store_from_spec(backend)
if not isabspath(path):
if path.startswith('./'):
path = abspath(path)
debug("abspath ==> %r", path)
else:
if basedir is None:
raise ValueError('relative path %r but no basedir' %
(path, ))
basedir = longpath(basedir)
debug("longpath(basedir) ==> %r", basedir)
path = joinpath(basedir, path)
debug("path ==> %r", path)
return FileCacheStore(
store_name,
backend_store,
path,
max_cachefile_size=max_file_size,
max_cachefiles=max_files,
hashclass=hashclass,
)
def cmd_fileby(self, argv):
''' Usage: {cmd} [-d dirpath] tag_name paths...
Add paths to the tagger.file_by mapping for the current directory.
-d dirpath Adjust the mapping for a different directory.
'''
dirpath = '.'
opts, argv = getopt(argv, 'd:')
for opt, val in opts:
with Pfx(opt):
if opt == '-d':
dirpath = val
else:
raise RuntimeError("unhandled option")
if not argv:
raise GetoptError("missing tag_name")
tag_name = argv.pop(0)
if not Tag.is_valid_name(tag_name):
raise GetoptError("invalid tag_name: %r" % (tag_name, ))
if not argv:
raise GetoptError("missing paths")
tagged = self.options.fstags[dirpath]
file_by = tagged.get('tagger.file_by', {})
paths = file_by.get(tag_name, ())
if isinstance(paths, str):
paths = [paths]
paths = set(paths)
paths.update(argv)
homedir = os.environ.get('HOME')
if homedir and isabspath(homedir):
homedir_ = homedir + os.sep
paths = set(
('~/' +
path[len(homedir_):] if path.startswith(homedir_) else path)
for path in paths)
file_by[tag_name] = sorted(paths)
tagged['tagger.file_by'] = file_by
print("tagger.file_by =", repr(file_by))
def file_by_tags(self,
path: str,
prune_inherited=False,
no_link=False,
do_remove=False):
''' Examine a file's tags.
Where those tags imply a location, link the file to that location.
Return the list of links made.
Parameters:
* `path`: the source path to file
* `prune_inherited`: optional, default `False`:
prune the inherited tags from the direct tags on the target
* `no_link`: optional, default `False`;
do not actually make the hard link, just report the target
* `do_remove`: optional, default `False`;
remove source files if successfully linked
Note: if `path` is already linked to an implied location
that location is also included in the returned list.
The filing process is as follows:
- for each target directory, initially `dirname(path)`,
look for a filing map on tag `file_by_mapping`
- for each directory in that mapping which matches a tag from `path`,
queue it as an additional target directory
- if there were no matching directories, file `path` at the current
target directory under the filename
returned by `{TAGGER_TAG_PREFIX_DEFAULT}.auto_name`
'''
if do_remove and no_link:
raise ValueError("do_remove and no_link may not both be true")
fstags = self.fstags
# start the queue with the resolved `path`
tagged = fstags[path]
srcpath = tagged.filepath
tags = tagged.all_tags
# a queue of reference directories
q = ListQueue((dirname(srcpath), ))
linked_to = []
seen = set()
for refdirpath in unrepeated(q, signature=abspath, seen=seen):
with Pfx(refdirpath):
# places to redirect this file
mapping = self.file_by_mapping(refdirpath)
interesting_tag_names = {tag.name for tag in mapping.keys()}
# locate specific filing locations in the refdirpath
refile_to = set()
for tag_name in sorted(interesting_tag_names):
with Pfx("tag_name %r", tag_name):
if tag_name not in tags:
continue
bare_tag = Tag(tag_name, tags[tag_name])
try:
target_dirs = mapping.get(bare_tag, ())
except TypeError as e:
warning(" %s not mapped (%s), skipping", bare_tag,
e)
continue
if not target_dirs:
continue
# collect other filing locations
refile_to.update(target_dirs)
# queue further locations if they are new
if refile_to:
new_refile_to = set(map(abspath, refile_to)) - seen
if new_refile_to:
q.extend(new_refile_to)
continue
# file locally (no new locations)
dstbase = self.auto_name(srcpath, refdirpath, tags)
with Pfx("%s => %s", refdirpath, dstbase):
dstpath = dstbase if isabspath(dstbase) else joinpath(
refdirpath, dstbase)
if existspath(dstpath):
if not samefile(srcpath, dstpath):
warning("already exists, skipping")
continue
if no_link:
linked_to.append(dstpath)
else:
linkto_dirpath = dirname(dstpath)
if not isdirpath(linkto_dirpath):
pfx_call(os.mkdir, linkto_dirpath)
try:
pfx_call(os.link, srcpath, dstpath)
except OSError as e:
warning("cannot link to %r: %s", dstpath, e)
else:
linked_to.append(dstpath)
fstags[dstpath].update(tags)
if prune_inherited:
fstags[dstpath].prune_inherited()
if linked_to and do_remove:
S = os.stat(srcpath)
if S.st_nlink < 2:
warning("not removing %r, unsufficient hard links (%s)",
srcpath, S.st_nlink)
else:
pfx_call(os.remove, srcpath)
return linked_to