def dump_Store(S, indent=''): ''' Dump a description of a Store. ''' from .cache import FileCacheStore from .store import MappingStore, ProxyStore, DataDirStore X("%s%s:%s", indent, type(S).__name__, S.name) indent += ' ' if isinstance(S, DataDirStore): X("%sdir = %s", indent, shortpath(S._datadir.topdirpath)) elif isinstance(S, FileCacheStore): X("%sdatadir = %s", indent, shortpath(S.cache.dirpath)) elif isinstance(S, ProxyStore): for attr in 'save', 'read', 'save2', 'read2', 'copy2': backends = getattr(S, attr) if backends: backends = sorted(backends, key=lambda S: S.name) X( "%s%s = %s", indent, attr, ','.join(backend.name for backend in backends) ) for backend in backends: dump_Store(backend, indent + ' ') elif isinstance(S, MappingStore): mapping = S.mapping X("%smapping = %s", indent, type(mapping)) else: X("%sUNRECOGNISED Store type", indent)
def _autofile(path, *, tagger, no_link, do_remove): ''' Wrapper for `Tagger.file_by_tags` which reports actions. ''' if not no_link and not existspath(path): warning("no such path, skipped") linked_to = [] else: fstags = tagger.fstags # apply inferred tags if not already present tagged = fstags[path] all_tags = tagged.merged_tags() for tag_name, tag_value in tagger.infer(path).items(): if tag_name not in all_tags: tagged[tag_name] = tag_value linked_to = tagger.file_by_tags(path, no_link=no_link, do_remove=do_remove) if linked_to: for linked in linked_to: printpath = linked if basename(path) == basename(printpath): printpath = dirname(printpath) + '/' pfxprint('=>', shortpath(printpath)) else: pfxprint('not filed') return linked_to
def __init__(self, parent, *, path, **kw): ''' Initialise the image widget to display `path`. ''' kw.setdefault('bitmap', 'gray25') kw.setdefault('text', shortpath(path) if path else "NONE") super().__init__(parent, **kw) self.fspath = path self._image_for = None
def fspath(self, new_fspath): ''' Switch the preview to look at a new filesystem path. ''' print("SET fspath =", repr(new_fspath)) self._fspath = new_fspath self._tag_widgets = {} self.update(value=shortpath(new_fspath) if new_fspath else "NONE") self.preview.fspath = new_fspath tags = self.tagged.merged_tags() self.tagsview.set_tags(tags) self.tagsview.set_size(size=(1920, 120)) print("tag suggestions =", repr(self.suggested_tags))
def assimilate(self, other, no_action=False): ''' Link our primary path to all the paths from `other`. Return success. ''' ok = True path = self.path opaths = other.paths pathprefix = common_path_prefix(path, *opaths) vpathprefix = shortpath(pathprefix) pathsuffix = path[len(pathprefix):] # pylint: disable=unsubscriptable-object with UpdProxy() as proxy: proxy( "%s%s <= %r", vpathprefix, pathsuffix, list(map(lambda opath: opath[len(pathprefix):], sorted(opaths))) ) with Pfx(path): if self is other or self.same_file(other): # already assimilated return ok assert self.same_dev(other) for opath in sorted(opaths): with Pfx(opath): if opath in self.paths: warning("already assimilated") continue if vpathprefix: print( "%s: %s => %s" % (vpathprefix, opath[len(pathprefix):], pathsuffix) ) else: print("%s => %s" % (opath[len(pathprefix):], pathsuffix)) if no_action: continue odir = dirname(opath) with NamedTemporaryFile(dir=odir) as tfp: with Pfx("unlink(%s)", tfp.name): os.unlink(tfp.name) with Pfx("rename(%s, %s)", opath, tfp.name): os.rename(opath, tfp.name) with Pfx("link(%s, %s)", path, opath): try: os.link(path, opath) except OSError as e: error("%s", e) ok = False # try to restore the previous file with Pfx("restore: link(%r, %r)", tfp.name, opath): os.link(tfp.name, opath) else: self.paths.add(opath) opaths.remove(opath) return ok
def make_treedata(self, fspaths): treedata = sg.TreeData() for fspath in fspaths: with Pfx(fspath): fullpath = realpath(fspath) pathinfo = IndexedMapping(pk='fullpath') top_record = UUIDedDict(fullpath=fullpath) pathinfo.add(top_record) treedata.insert( "", top_record.uuid, shortpath(top_record.fullpath), [basename(top_record.fullpath)], icon=None, ) if isdirpath(fullpath): for dirpath, dirnames, filenames in os.walk(fullpath): with Pfx("walk %r", dirpath): record = pathinfo.by_fullpath[dirpath] parent_node = treedata.tree_dict[record.uuid] for dirname in sorted(dirnames): with Pfx(dirname): if dirname.startswith('.'): continue subdir_path = joinpath(dirpath, dirname) subdir_record = UUIDedDict(fullpath=subdir_path) pathinfo.add(subdir_record) treedata.insert( record.uuid, subdir_record.uuid, dirname, [dirname], icon=None, ) for filename in sorted(filenames): with Pfx(filenames): if filename.startswith('.'): continue filepath = joinpath(dirpath, filename) file_record = UUIDedDict(fullpath=filepath) pathinfo.add(file_record) treedata.insert( record.uuid, file_record.uuid, filename, [filename], icon=None, ) return treedata, pathinfo
def fspath(self, new_fspath): ''' Switch the preview to look at a new filesystem path. ''' print("SET fspath =", repr(new_fspath)) self._fspath = new_fspath self._tag_widgets = {} self.config(text=shortpath(new_fspath) or "NONE") self.preview.fspath = new_fspath tagged = self.tagged all_tags = TagSet(tagged.merged_tags()) suggested_tags = self.suggested_tags for sg_name in suggested_tags.keys(): if sg_name not in all_tags: all_tags[sg_name] = None self.tagsview.set_tags( tagged, lambda tag: suggested_tags.get(tag.name), bg_tags=all_tags ) print("tag suggestions =", repr(self.suggested_tags))
def cmd_test(self, argv): ''' Usage: {cmd} path Run a test against path. Current we try out the suggestions. ''' tagger = self.options.tagger fstags = self.options.fstags if not argv: raise GetoptError("missing path") path = argv.pop(0) if argv: raise GetoptError("extra arguments: %r" % (argv, )) tagged = fstags[path] changed = True while True: print(path, *tagged) if changed: changed = False suggestions = tagger.suggested_tags(path) for tag_name, values in sorted(suggestions.items()): print(" ", tag_name, values) for file_to in tagger.file_by_tags(path, no_link=True): print("=>", shortpath(file_to)) print("inferred:", repr(tagger.infer(path))) try: action = input("Action? ").strip() except EOFError: break if action: with Pfx(repr(action)): try: if action.startswith('-'): tag = Tag.from_str(action[1:].lstrip()) tagged.discard(tag) changed = True elif action.startswith('+'): tag = Tag.from_str(action[1:].lstrip()) tagged.add(tag) changed = True else: raise ValueError("unrecognised action") except ValueError as e: warning("action fails: %s", e)
def add_path(self, new_path: str, indexed_to=0) -> DataFileState: ''' Insert a new path into the map. Return its `DataFileState`. ''' info("new path %r", shortpath(new_path)) with self._lock: c = self._modify( 'INSERT INTO filemap(`path`, `indexed_to`) VALUES (?, ?)', (new_path, 0), return_cursor=True ) if c: filenum = c.lastrowid self._map(new_path, filenum, indexed_to=indexed_to) c.close() DFstate = self.n_to_DFstate[filenum] else: # already mapped DFState = self.path_to_DFstate[new_path] return DFstate
def __str__(self): return "%s:%s" % (type(self).__name__, shortpath(self.fspath))
def __str__(self): return "%s:%s:%s(%r,index=%s)" % ( type(self).__name__, self.hashclass.HASHNAME, self.data_record_class.__name__, shortpath(self.path), self.index)
def __str__(self): return "PlatonicFile(%s)" % (shortpath(self.path,))
def parse_special(self, special, readonly): ''' Parse the mount command's special device from `special`. Return `(fsname,readonly,Store,Dir,basename,archive)`. Supported formats: * `D{...}`: a raw `Dir` transcription. * `[`*clause*`]`: a config clause name. * `[`*clause*`]`*archive*: a config clause name and a reference to a named archive associates with that clause. * *archive_file*`.vt`: a path to a `.vt` archive file. ''' fsname = special specialD = None special_store = None archive = None if special.startswith('D{') and special.endswith('}'): # D{dir} specialD, offset = parse(special) if offset != len(special): raise ValueError("unparsed text: %r" % (special[offset:], )) if not isinstance(specialD, Dir): raise ValueError( "does not seem to be a Dir transcription, looks like a %s" % (type(specialD), )) special_basename = specialD.name if not readonly: warning("setting readonly") readonly = True elif special.startswith('['): if special.endswith(']'): # expect "[clause]" clause_name, offset = get_ini_clausename(special) archive_name = '' special_basename = clause_name else: # expect "[clause]archive" # TODO: just pass to Archive(special,config=self)? # what about special_basename then? clause_name, archive_name, offset = get_ini_clause_entryname( special) special_basename = archive_name if offset < len(special): raise ValueError("unparsed text: %r" % (special[offset:], )) fsname = str(self) + special try: special_store = self[clause_name] except KeyError: raise ValueError("unknown config clause [%s]" % (clause_name, )) if archive_name is None or not archive_name: special_basename = clause_name else: special_basename = archive_name archive = special_store.get_Archive(archive_name) else: # pathname to archive file arpath = special if not isfilepath(arpath): raise ValueError("not a file") fsname = shortpath(realpath(arpath)) spfx, sext = splitext(basename(arpath)) if spfx and sext == '.vt': special_basename = spfx else: special_basename = special archive = FilePathArchive(arpath) return fsname, readonly, special_store, specialD, special_basename, archive
def __str__(self): return '%s(%s)' % (self.__class__.__name__, shortpath(self.topdirpath))
def _monitor_datafiles(self): ''' Thread body to poll the ideal tree for new or changed files. ''' proxy = upd_state.proxy proxy.prefix = str(self) + " monitor " meta_store = self.meta_store filemap = self._filemap datadirpath = self.pathto('data') if meta_store is not None: topdir = self.topdir else: warning("%s: no meta_store!", self) updated = False disabled = False while not self.cancelled: sleep(self.DELAY_INTERSCAN) if self.flag_scan_disable: if not disabled: info("scan %r DISABLED", shortpath(datadirpath)) disabled = True continue if disabled: info("scan %r ENABLED", shortpath(datadirpath)) disabled = False # scan for new datafiles with Pfx("%r", datadirpath): seen = set() info("scan tree...") with proxy.extend_prefix(" scan"): for dirpath, dirnames, filenames in os.walk(datadirpath, followlinks=True): dirnames[:] = sorted(dirnames) filenames = sorted(filenames) sleep(self.DELAY_INTRASCAN) if self.cancelled or self.flag_scan_disable: break rdirpath = relpath(dirpath, datadirpath) with Pfx(rdirpath): with (proxy.extend_prefix(" " + rdirpath) if filenames else nullcontext()): # this will be the subdirectories into which to recurse pruned_dirnames = [] for dname in dirnames: if self.exclude_dir(joinpath(rdirpath, dname)): # unwanted continue subdirpath = joinpath(dirpath, dname) try: S = os.stat(subdirpath) except OSError as e: # inaccessable warning("stat(%r): %s, skipping", subdirpath, e) continue ino = S.st_dev, S.st_ino if ino in seen: # we have seen this subdir before, probably via a symlink # TODO: preserve symlinks? attach alter ego directly as a Dir? debug( "seen %r (dev=%s,ino=%s), skipping", subdirpath, ino[0], ino[1] ) continue seen.add(ino) pruned_dirnames.append(dname) dirnames[:] = pruned_dirnames if meta_store is None: warning("no meta_store") D = None else: with meta_store: D = topdir.makedirs(rdirpath, force=True) # prune removed names names = list(D.keys()) for name in names: if name not in dirnames and name not in filenames: info("del %r", name) del D[name] for filename in filenames: with Pfx(filename): if self.cancelled or self.flag_scan_disable: break rfilepath = joinpath(rdirpath, filename) if self.exclude_file(rfilepath): continue filepath = joinpath(dirpath, filename) if not isfilepath(filepath): continue # look up this file in our file state index DFstate = filemap.get(rfilepath) if (DFstate is not None and D is not None and filename not in D): # in filemap, but not in dir: start again warning("in filemap but not in Dir, rescanning") filemap.del_path(rfilepath) DFstate = None if DFstate is None: DFstate = filemap.add_path(rfilepath) try: new_size = DFstate.stat_size(self.follow_symlinks) except OSError as e: if e.errno == errno.ENOENT: warning("forgetting missing file") self._del_datafilestate(DFstate) else: warning("stat: %s", e) continue if new_size is None: # skip non files debug("SKIP non-file") continue if meta_store: try: E = D[filename] except KeyError: E = FileDirent(filename) D[filename] = E else: if not E.isfile: info( "new FileDirent replacing previous nonfile: %s", E ) E = FileDirent(filename) D[filename] = E if new_size > DFstate.scanned_to: with proxy.extend_prefix( " scan %s[%d:%d]" % (filename, DFstate.scanned_to, new_size)): if DFstate.scanned_to > 0: info("scan from %d", DFstate.scanned_to) if meta_store is not None: blockQ = IterableQueue() R = meta_store._defer( lambda B, Q: top_block_for(spliced_blocks(B, Q)), E.block, blockQ ) scan_from = DFstate.scanned_to scan_start = time() scanner = DFstate.scanfrom(offset=DFstate.scanned_to) if defaults.show_progress: scanner = progressbar( DFstate.scanfrom(offset=DFstate.scanned_to), "scan " + rfilepath, position=DFstate.scanned_to, total=new_size, units_scale=BINARY_BYTES_SCALE, itemlenfunc=lambda t3: t3[2] - t3[0], update_frequency=128, ) for pre_offset, data, post_offset in scanner: hashcode = self.hashclass.from_chunk(data) entry = FileDataIndexEntry( filenum=DFstate.filenum, data_offset=pre_offset, data_length=len(data), flags=0, ) entry_bs = bytes(entry) with self._lock: index[hashcode] = entry_bs if meta_store is not None: B = Block(data=data, hashcode=hashcode, added=True) blockQ.put((pre_offset, B)) DFstate.scanned_to = post_offset if self.cancelled or self.flag_scan_disable: break if meta_store is not None: blockQ.close() try: top_block = R() except MissingHashcodeError as e: error("missing data, forcing rescan: %s", e) DFstate.scanned_to = 0 else: E.block = top_block D.changed = True updated = True elapsed = time() - scan_start scanned = DFstate.scanned_to - scan_from if elapsed > 0: scan_rate = scanned / elapsed else: scan_rate = None if scan_rate is None: info( "scanned to %d: %s", DFstate.scanned_to, transcribe_bytes_geek(scanned) ) else: info( "scanned to %d: %s at %s/s", DFstate.scanned_to, transcribe_bytes_geek(scanned), transcribe_bytes_geek(scan_rate) ) # stall after a file scan, briefly, to limit impact if elapsed > 0: sleep(min(elapsed, self.DELAY_INTRASCAN)) # update the archive after updating from a directory if updated and meta_store is not None: self.sync_meta() updated = False self.flush()
def cmd_import_from_calibre(self, argv): ''' Usage: {cmd} other-library [identifier-name] [identifier-values...] Import formats from another Calibre library. other-library: the path to another Calibre library tree identifier-name: the key on which to link matching books; the default is {DEFAULT_LINK_IDENTIFIER} identifier-values: specific book identifiers to import ''' options = self.options calibre = options.calibre if not argv: raise GetoptError("missing other-library") other_library = CalibreTree(argv.pop(0)) with Pfx(shortpath(other_library.fspath)): if other_library is calibre: raise GetoptError("cannot import from the same library") if argv: identifier_name = argv.pop(0) else: identifier_name = self.DEFAULT_LINK_IDENTIFIER if argv: identifier_values = argv else: identifier_values = sorted( set( filter( lambda idv: idv is not None, (cbook.identifiers_as_dict().get(identifier_name) for cbook in other_library)))) xit = 0 for identifier_value in identifier_values: with Pfx("%s:%s", identifier_name, identifier_value): obooks = list( other_library.by_identifier(identifier_name, identifier_value)) if not obooks: error("no books with this identifier") xit = 1 continue if len(obooks) > 1: warning(" \n".join([ "multiple \"other\" books with this identifier:", *map(str, obooks) ])) xit = 1 continue obook, = obooks cbooks = list( calibre.by_identifier(identifier_name, identifier_value)) if not cbooks: print("NEW BOOK", obook) elif len(cbooks) > 1: warning(" \n".join([ "multiple \"local\" books with this identifier:", *map(str, cbooks) ])) print("PULL", obook, "AS NEW BOOK") else: cbook, = cbooks print("MERGE", obook, "INTO", cbook) return xit
def __str__(self): return "%s(%s)" % (type(self).__name__, shortpath(self.path))
def __str__(self): if self.path is None: return repr(self) return "Config(%s)" % (shortpath(self.path), )