def dump_Store(S, indent=''):
''' Dump a description of a Store.
'''
from .cache import FileCacheStore
from .store import MappingStore, ProxyStore, DataDirStore
X("%s%s:%s", indent, type(S).__name__, S.name)
indent += ' '
if isinstance(S, DataDirStore):
X("%sdir = %s", indent, shortpath(S._datadir.topdirpath))
elif isinstance(S, FileCacheStore):
X("%sdatadir = %s", indent, shortpath(S.cache.dirpath))
elif isinstance(S, ProxyStore):
for attr in 'save', 'read', 'save2', 'read2', 'copy2':
backends = getattr(S, attr)
if backends:
backends = sorted(backends, key=lambda S: S.name)
X(
"%s%s = %s", indent, attr,
','.join(backend.name for backend in backends)
)
for backend in backends:
dump_Store(backend, indent + ' ')
elif isinstance(S, MappingStore):
mapping = S.mapping
X("%smapping = %s", indent, type(mapping))
else:
X("%sUNRECOGNISED Store type", indent)
def _autofile(path, *, tagger, no_link, do_remove):
''' Wrapper for `Tagger.file_by_tags` which reports actions.
'''
if not no_link and not existspath(path):
warning("no such path, skipped")
linked_to = []
else:
fstags = tagger.fstags
# apply inferred tags if not already present
tagged = fstags[path]
all_tags = tagged.merged_tags()
for tag_name, tag_value in tagger.infer(path).items():
if tag_name not in all_tags:
tagged[tag_name] = tag_value
linked_to = tagger.file_by_tags(path,
no_link=no_link,
do_remove=do_remove)
if linked_to:
for linked in linked_to:
printpath = linked
if basename(path) == basename(printpath):
printpath = dirname(printpath) + '/'
pfxprint('=>', shortpath(printpath))
else:
pfxprint('not filed')
return linked_to
def __init__(self, parent, *, path, **kw):
''' Initialise the image widget to display `path`.
'''
kw.setdefault('bitmap', 'gray25')
kw.setdefault('text', shortpath(path) if path else "NONE")
super().__init__(parent, **kw)
self.fspath = path
self._image_for = None
def fspath(self, new_fspath):
''' Switch the preview to look at a new filesystem path.
'''
print("SET fspath =", repr(new_fspath))
self._fspath = new_fspath
self._tag_widgets = {}
self.update(value=shortpath(new_fspath) if new_fspath else "NONE")
self.preview.fspath = new_fspath
tags = self.tagged.merged_tags()
self.tagsview.set_tags(tags)
self.tagsview.set_size(size=(1920, 120))
print("tag suggestions =", repr(self.suggested_tags))
def assimilate(self, other, no_action=False):
''' Link our primary path to all the paths from `other`. Return success.
'''
ok = True
path = self.path
opaths = other.paths
pathprefix = common_path_prefix(path, *opaths)
vpathprefix = shortpath(pathprefix)
pathsuffix = path[len(pathprefix):] # pylint: disable=unsubscriptable-object
with UpdProxy() as proxy:
proxy(
"%s%s <= %r", vpathprefix, pathsuffix,
list(map(lambda opath: opath[len(pathprefix):], sorted(opaths)))
)
with Pfx(path):
if self is other or self.same_file(other):
# already assimilated
return ok
assert self.same_dev(other)
for opath in sorted(opaths):
with Pfx(opath):
if opath in self.paths:
warning("already assimilated")
continue
if vpathprefix:
print(
"%s: %s => %s" %
(vpathprefix, opath[len(pathprefix):], pathsuffix)
)
else:
print("%s => %s" % (opath[len(pathprefix):], pathsuffix))
if no_action:
continue
odir = dirname(opath)
with NamedTemporaryFile(dir=odir) as tfp:
with Pfx("unlink(%s)", tfp.name):
os.unlink(tfp.name)
with Pfx("rename(%s, %s)", opath, tfp.name):
os.rename(opath, tfp.name)
with Pfx("link(%s, %s)", path, opath):
try:
os.link(path, opath)
except OSError as e:
error("%s", e)
ok = False
# try to restore the previous file
with Pfx("restore: link(%r, %r)", tfp.name, opath):
os.link(tfp.name, opath)
else:
self.paths.add(opath)
opaths.remove(opath)
return ok
def make_treedata(self, fspaths):
treedata = sg.TreeData()
for fspath in fspaths:
with Pfx(fspath):
fullpath = realpath(fspath)
pathinfo = IndexedMapping(pk='fullpath')
top_record = UUIDedDict(fullpath=fullpath)
pathinfo.add(top_record)
treedata.insert(
"",
top_record.uuid,
shortpath(top_record.fullpath),
[basename(top_record.fullpath)],
icon=None,
)
if isdirpath(fullpath):
for dirpath, dirnames, filenames in os.walk(fullpath):
with Pfx("walk %r", dirpath):
record = pathinfo.by_fullpath[dirpath]
parent_node = treedata.tree_dict[record.uuid]
for dirname in sorted(dirnames):
with Pfx(dirname):
if dirname.startswith('.'):
continue
subdir_path = joinpath(dirpath, dirname)
subdir_record = UUIDedDict(fullpath=subdir_path)
pathinfo.add(subdir_record)
treedata.insert(
record.uuid,
subdir_record.uuid,
dirname,
[dirname],
icon=None,
)
for filename in sorted(filenames):
with Pfx(filenames):
if filename.startswith('.'):
continue
filepath = joinpath(dirpath, filename)
file_record = UUIDedDict(fullpath=filepath)
pathinfo.add(file_record)
treedata.insert(
record.uuid,
file_record.uuid,
filename,
[filename],
icon=None,
)
return treedata, pathinfo
def fspath(self, new_fspath):
''' Switch the preview to look at a new filesystem path.
'''
print("SET fspath =", repr(new_fspath))
self._fspath = new_fspath
self._tag_widgets = {}
self.config(text=shortpath(new_fspath) or "NONE")
self.preview.fspath = new_fspath
tagged = self.tagged
all_tags = TagSet(tagged.merged_tags())
suggested_tags = self.suggested_tags
for sg_name in suggested_tags.keys():
if sg_name not in all_tags:
all_tags[sg_name] = None
self.tagsview.set_tags(
tagged, lambda tag: suggested_tags.get(tag.name), bg_tags=all_tags
)
print("tag suggestions =", repr(self.suggested_tags))
def cmd_test(self, argv):
''' Usage: {cmd} path
Run a test against path.
Current we try out the suggestions.
'''
tagger = self.options.tagger
fstags = self.options.fstags
if not argv:
raise GetoptError("missing path")
path = argv.pop(0)
if argv:
raise GetoptError("extra arguments: %r" % (argv, ))
tagged = fstags[path]
changed = True
while True:
print(path, *tagged)
if changed:
changed = False
suggestions = tagger.suggested_tags(path)
for tag_name, values in sorted(suggestions.items()):
print(" ", tag_name, values)
for file_to in tagger.file_by_tags(path, no_link=True):
print("=>", shortpath(file_to))
print("inferred:", repr(tagger.infer(path)))
try:
action = input("Action? ").strip()
except EOFError:
break
if action:
with Pfx(repr(action)):
try:
if action.startswith('-'):
tag = Tag.from_str(action[1:].lstrip())
tagged.discard(tag)
changed = True
elif action.startswith('+'):
tag = Tag.from_str(action[1:].lstrip())
tagged.add(tag)
changed = True
else:
raise ValueError("unrecognised action")
except ValueError as e:
warning("action fails: %s", e)
def add_path(self, new_path: str, indexed_to=0) -> DataFileState:
''' Insert a new path into the map.
Return its `DataFileState`.
'''
info("new path %r", shortpath(new_path))
with self._lock:
c = self._modify(
'INSERT INTO filemap(`path`, `indexed_to`) VALUES (?, ?)',
(new_path, 0),
return_cursor=True
)
if c:
filenum = c.lastrowid
self._map(new_path, filenum, indexed_to=indexed_to)
c.close()
DFstate = self.n_to_DFstate[filenum]
else:
# already mapped
DFState = self.path_to_DFstate[new_path]
return DFstate
def __str__(self):
return "%s:%s" % (type(self).__name__, shortpath(self.fspath))
def __str__(self):
return "%s:%s:%s(%r,index=%s)" % (
type(self).__name__, self.hashclass.HASHNAME,
self.data_record_class.__name__, shortpath(self.path), self.index)
def __str__(self): return "PlatonicFile(%s)" % (shortpath(self.path,))
def parse_special(self, special, readonly):
''' Parse the mount command's special device from `special`.
Return `(fsname,readonly,Store,Dir,basename,archive)`.
Supported formats:
* `D{...}`: a raw `Dir` transcription.
* `[`*clause*`]`: a config clause name.
* `[`*clause*`]`*archive*: a config clause name
and a reference to a named archive associates with that clause.
* *archive_file*`.vt`: a path to a `.vt` archive file.
'''
fsname = special
specialD = None
special_store = None
archive = None
if special.startswith('D{') and special.endswith('}'):
# D{dir}
specialD, offset = parse(special)
if offset != len(special):
raise ValueError("unparsed text: %r" % (special[offset:], ))
if not isinstance(specialD, Dir):
raise ValueError(
"does not seem to be a Dir transcription, looks like a %s"
% (type(specialD), ))
special_basename = specialD.name
if not readonly:
warning("setting readonly")
readonly = True
elif special.startswith('['):
if special.endswith(']'):
# expect "[clause]"
clause_name, offset = get_ini_clausename(special)
archive_name = ''
special_basename = clause_name
else:
# expect "[clause]archive"
# TODO: just pass to Archive(special,config=self)?
# what about special_basename then?
clause_name, archive_name, offset = get_ini_clause_entryname(
special)
special_basename = archive_name
if offset < len(special):
raise ValueError("unparsed text: %r" % (special[offset:], ))
fsname = str(self) + special
try:
special_store = self[clause_name]
except KeyError:
raise ValueError("unknown config clause [%s]" %
(clause_name, ))
if archive_name is None or not archive_name:
special_basename = clause_name
else:
special_basename = archive_name
archive = special_store.get_Archive(archive_name)
else:
# pathname to archive file
arpath = special
if not isfilepath(arpath):
raise ValueError("not a file")
fsname = shortpath(realpath(arpath))
spfx, sext = splitext(basename(arpath))
if spfx and sext == '.vt':
special_basename = spfx
else:
special_basename = special
archive = FilePathArchive(arpath)
return fsname, readonly, special_store, specialD, special_basename, archive
def __str__(self): return '%s(%s)' % (self.__class__.__name__, shortpath(self.topdirpath))
def _monitor_datafiles(self):
''' Thread body to poll the ideal tree for new or changed files.
'''
proxy = upd_state.proxy
proxy.prefix = str(self) + " monitor "
meta_store = self.meta_store
filemap = self._filemap
datadirpath = self.pathto('data')
if meta_store is not None:
topdir = self.topdir
else:
warning("%s: no meta_store!", self)
updated = False
disabled = False
while not self.cancelled:
sleep(self.DELAY_INTERSCAN)
if self.flag_scan_disable:
if not disabled:
info("scan %r DISABLED", shortpath(datadirpath))
disabled = True
continue
if disabled:
info("scan %r ENABLED", shortpath(datadirpath))
disabled = False
# scan for new datafiles
with Pfx("%r", datadirpath):
seen = set()
info("scan tree...")
with proxy.extend_prefix(" scan"):
for dirpath, dirnames, filenames in os.walk(datadirpath,
followlinks=True):
dirnames[:] = sorted(dirnames)
filenames = sorted(filenames)
sleep(self.DELAY_INTRASCAN)
if self.cancelled or self.flag_scan_disable:
break
rdirpath = relpath(dirpath, datadirpath)
with Pfx(rdirpath):
with (proxy.extend_prefix(" " + rdirpath)
if filenames else nullcontext()):
# this will be the subdirectories into which to recurse
pruned_dirnames = []
for dname in dirnames:
if self.exclude_dir(joinpath(rdirpath, dname)):
# unwanted
continue
subdirpath = joinpath(dirpath, dname)
try:
S = os.stat(subdirpath)
except OSError as e:
# inaccessable
warning("stat(%r): %s, skipping", subdirpath, e)
continue
ino = S.st_dev, S.st_ino
if ino in seen:
# we have seen this subdir before, probably via a symlink
# TODO: preserve symlinks? attach alter ego directly as a Dir?
debug(
"seen %r (dev=%s,ino=%s), skipping", subdirpath,
ino[0], ino[1]
)
continue
seen.add(ino)
pruned_dirnames.append(dname)
dirnames[:] = pruned_dirnames
if meta_store is None:
warning("no meta_store")
D = None
else:
with meta_store:
D = topdir.makedirs(rdirpath, force=True)
# prune removed names
names = list(D.keys())
for name in names:
if name not in dirnames and name not in filenames:
info("del %r", name)
del D[name]
for filename in filenames:
with Pfx(filename):
if self.cancelled or self.flag_scan_disable:
break
rfilepath = joinpath(rdirpath, filename)
if self.exclude_file(rfilepath):
continue
filepath = joinpath(dirpath, filename)
if not isfilepath(filepath):
continue
# look up this file in our file state index
DFstate = filemap.get(rfilepath)
if (DFstate is not None and D is not None
and filename not in D):
# in filemap, but not in dir: start again
warning("in filemap but not in Dir, rescanning")
filemap.del_path(rfilepath)
DFstate = None
if DFstate is None:
DFstate = filemap.add_path(rfilepath)
try:
new_size = DFstate.stat_size(self.follow_symlinks)
except OSError as e:
if e.errno == errno.ENOENT:
warning("forgetting missing file")
self._del_datafilestate(DFstate)
else:
warning("stat: %s", e)
continue
if new_size is None:
# skip non files
debug("SKIP non-file")
continue
if meta_store:
try:
E = D[filename]
except KeyError:
E = FileDirent(filename)
D[filename] = E
else:
if not E.isfile:
info(
"new FileDirent replacing previous nonfile: %s",
E
)
E = FileDirent(filename)
D[filename] = E
if new_size > DFstate.scanned_to:
with proxy.extend_prefix(
" scan %s[%d:%d]" %
(filename, DFstate.scanned_to, new_size)):
if DFstate.scanned_to > 0:
info("scan from %d", DFstate.scanned_to)
if meta_store is not None:
blockQ = IterableQueue()
R = meta_store._defer(
lambda B, Q: top_block_for(spliced_blocks(B, Q)),
E.block, blockQ
)
scan_from = DFstate.scanned_to
scan_start = time()
scanner = DFstate.scanfrom(offset=DFstate.scanned_to)
if defaults.show_progress:
scanner = progressbar(
DFstate.scanfrom(offset=DFstate.scanned_to),
"scan " + rfilepath,
position=DFstate.scanned_to,
total=new_size,
units_scale=BINARY_BYTES_SCALE,
itemlenfunc=lambda t3: t3[2] - t3[0],
update_frequency=128,
)
for pre_offset, data, post_offset in scanner:
hashcode = self.hashclass.from_chunk(data)
entry = FileDataIndexEntry(
filenum=DFstate.filenum,
data_offset=pre_offset,
data_length=len(data),
flags=0,
)
entry_bs = bytes(entry)
with self._lock:
index[hashcode] = entry_bs
if meta_store is not None:
B = Block(data=data, hashcode=hashcode, added=True)
blockQ.put((pre_offset, B))
DFstate.scanned_to = post_offset
if self.cancelled or self.flag_scan_disable:
break
if meta_store is not None:
blockQ.close()
try:
top_block = R()
except MissingHashcodeError as e:
error("missing data, forcing rescan: %s", e)
DFstate.scanned_to = 0
else:
E.block = top_block
D.changed = True
updated = True
elapsed = time() - scan_start
scanned = DFstate.scanned_to - scan_from
if elapsed > 0:
scan_rate = scanned / elapsed
else:
scan_rate = None
if scan_rate is None:
info(
"scanned to %d: %s", DFstate.scanned_to,
transcribe_bytes_geek(scanned)
)
else:
info(
"scanned to %d: %s at %s/s", DFstate.scanned_to,
transcribe_bytes_geek(scanned),
transcribe_bytes_geek(scan_rate)
)
# stall after a file scan, briefly, to limit impact
if elapsed > 0:
sleep(min(elapsed, self.DELAY_INTRASCAN))
# update the archive after updating from a directory
if updated and meta_store is not None:
self.sync_meta()
updated = False
self.flush()
def cmd_import_from_calibre(self, argv):
''' Usage: {cmd} other-library [identifier-name] [identifier-values...]
Import formats from another Calibre library.
other-library: the path to another Calibre library tree
identifier-name: the key on which to link matching books;
the default is {DEFAULT_LINK_IDENTIFIER}
identifier-values: specific book identifiers to import
'''
options = self.options
calibre = options.calibre
if not argv:
raise GetoptError("missing other-library")
other_library = CalibreTree(argv.pop(0))
with Pfx(shortpath(other_library.fspath)):
if other_library is calibre:
raise GetoptError("cannot import from the same library")
if argv:
identifier_name = argv.pop(0)
else:
identifier_name = self.DEFAULT_LINK_IDENTIFIER
if argv:
identifier_values = argv
else:
identifier_values = sorted(
set(
filter(
lambda idv: idv is not None,
(cbook.identifiers_as_dict().get(identifier_name)
for cbook in other_library))))
xit = 0
for identifier_value in identifier_values:
with Pfx("%s:%s", identifier_name, identifier_value):
obooks = list(
other_library.by_identifier(identifier_name,
identifier_value))
if not obooks:
error("no books with this identifier")
xit = 1
continue
if len(obooks) > 1:
warning(" \n".join([
"multiple \"other\" books with this identifier:",
*map(str, obooks)
]))
xit = 1
continue
obook, = obooks
cbooks = list(
calibre.by_identifier(identifier_name,
identifier_value))
if not cbooks:
print("NEW BOOK", obook)
elif len(cbooks) > 1:
warning(" \n".join([
"multiple \"local\" books with this identifier:",
*map(str, cbooks)
]))
print("PULL", obook, "AS NEW BOOK")
else:
cbook, = cbooks
print("MERGE", obook, "INTO", cbook)
return xit
def __str__(self): return "%s(%s)" % (type(self).__name__, shortpath(self.path))
def __str__(self):
if self.path is None:
return repr(self)
return "Config(%s)" % (shortpath(self.path), )