def _sync(self):
with Pfx("_sync"):
if defaults.S is None:
raise RuntimeError("RUNTIME: defaults.S is None!")
archive = self.archive
if not self.readonly and archive is not None:
with self._lock:
E = self.E
updated = False
X("snapshot %r ...", E)
E.snapshot()
X("snapshot: afterwards E=%r", E)
fs_inode_dirents = self._inodes.get_fs_inode_dirents()
X("_SYNC: FS_INODE_DIRENTS:")
dump_Dirent(fs_inode_dirents)
X("set meta.fs_inode_dirents")
if fs_inode_dirents.size > 0:
E.meta['fs_inode_dirents'] = str(fs_inode_dirents)
else:
E.meta['fs_inode_dirents'] = ''
new_state = bytes(E)
if new_state != self._last_sync_state:
archive.update(E)
self._last_sync_state = new_state
updated = True
# debugging
if updated:
dump_Dirent(E, recurse=False)
def noexc_gen_wrapper(*args, **kwargs):
try:
it = iter(func(*args, **kwargs))
except Exception as e0:
try:
exception("exception calling %s(*%s, **(%s)): %s",
func.__name__, args, kwargs, e0)
except Exception as e2:
try:
X("exception calling %s(*%s, **(%s)): %s", func.__name__,
args, kwargs, e2)
except Exception:
pass
return
while True:
try:
item = next(it)
except StopIteration:
raise
except Exception as e:
try:
exception("exception calling next(%s(*%s, **(%s))): %s",
func.__name__, args, kwargs, e)
except Exception:
try:
X("exception calling next(%s(*%s, **(%s))): %s",
func.__name__, args, kwargs, e)
except Exception:
pass
return
else:
yield item
def rename(self, new_name):
if self.name == new_name:
warning("rename tag %r: no change", self.name)
return
X("Tag[%d:%r].rename(new_name=%r)", self.id, self.name, new_name)
T = self._table
try:
otag = T[new_name]
except KeyError:
# name not in use, rename current tag
T.update('name', new_name, 'id = %d' % (self.id, ))
else:
X("other tag for new_name = %d:%r", otag.id, otag.name)
if otag.id == self.id:
# case insensitive or the like: update the name in place
T.update('name', new_name, 'id = %d' % (self.id, ))
else:
# update related objects (books?)
# to point at the other tag
for B in self.books:
X(" update Book[%d:%r]{%s} ...", B.id, B.name,
','.join(T.name for T in B.tags))
B.add_tag(new_name)
B.remove_tag(self.name)
# delete our tag, become the other tag
T.delete('id = ?', self.id)
self.ns.id = otag.id
self.name = new_name
def XP(msg, *args, **kwargs):
''' Variation on `cs.x.X`
which prefixes the message with the current Pfx prefix.
'''
if args:
return X("%s: " + msg, prefix(), *args, **kwargs)
return X(prefix() + DEFAULT_SEPARATOR + msg, **kwargs)
def _row_of_tags(self):
''' Make a row of tag widgets.
'''
X("TagsView_Frame: TAGS=%s", self.tags)
##row = [self._get_tag_widget(tag) for tag in self.tags]
row = [sg.Text(str(tag)) for tag in self.tags]
X("ROW OF TAGS: %r", row)
return row
def set_tags(self, tags):
super().set_tags(tags)
X("%s.set_tags: self.tags=%s", type(self).__name__, self.tags)
X("CALL SELF.LAYOUT(SELF.LAYOUT_TAGS)")
self.layout(self.layout_tags(self.tags))
if self.Widget is not None:
X("CONTENTS_CHANGED")
self.contents_changed()
def handle(self, *a, **kw):
''' Wrapper for FUSE handler methods.
'''
syscall = method.__name__
if syscall == 'write':
fh, offset, bs = a
arg_desc = [
str(a[0]),
str(a[1]),
"%d bytes:%r..." % (len(bs), bytes(bs[:16]))
]
else:
arg_desc = [(("<%s>" % (type(arg).__name__, )) if isinstance(
arg, llfuse.RequestContext) else repr(arg)) for arg in a]
arg_desc.extend("%s=%r" % (kw_name, kw_value)
for kw_name, kw_value in kw.items())
arg_desc = ','.join(arg_desc)
with Pfx("%s.%s(%s)", type(self).__name__, syscall, arg_desc):
trace = syscall in (
##'getxattr',
##'setxattr',
##'statfs',
)
if trace:
X("CALL %s(%s)", syscall, arg_desc)
fs = self._vtfs
try:
with stackattrs(defaults, fs=fs):
with fs.S:
with LogTime("SLOW SYSCALL", threshold=5.0):
result = method(self, *a, **kw)
if trace:
if isinstance(result, bytes):
X("CALL %s result => %d bytes, %r...", syscall,
len(result), result[:16])
else:
X("CALL %s result => %s", syscall, result)
return result
##except FuseOSError as e:
## warning("=> FuseOSError %s", e, exc_info=False)
## raise
except OSError as e:
##warning("=> OSError %s => FuseOSError", e, exc_info=False)
raise FuseOSError(e.errno) from e
except MissingHashcodeError as e:
error("raising IOError from missing hashcode: %s", e)
raise FuseOSError(errno.EIO) from e
except Exception as e:
exception("unexpected exception, raising EINVAL %s:%s",
type(e), e)
raise FuseOSError(errno.EINVAL) from e
except BaseException as e:
error("UNCAUGHT EXCEPTION: %s", e)
raise RuntimeError("UNCAUGHT EXCEPTION") from e
except:
error("=> EXCEPTION %r", sys.exc_info())
def __getattribute__(self, attr):
X("TracingObject.__getattribute__(attr=%r)", attr)
_proxied = Proxy.__getattribute__(self, '_proxied')
try:
value = object.__getattribute__(_proxied, attr)
except AttributeError:
X("no .%s attribute", attr)
raise
else:
X("getattr .%s", attr)
return TracingObject(value)
def mount(mnt,
E,
*,
S=None,
archive=None,
subpath=None,
readonly=None,
append_only=False,
fsname=None):
''' Run a FUSE filesystem, return the Thread running the filesystem.
Parameters:
* `mnt`: mount point
* `E`: Dirent of root Store directory
* `S`: optional backing Store, default from defaults.S
* `archive`: if not None, an Archive or similar, with a
`.update(Dirent[,when])` method
* `subpath`: relative path from `E` to the directory to attach
to the mountpoint
* `readonly`: forbid data modification operations
* `append_only`: files may not be truncated or overwritten
* `fsname`: optional filesystem name for use by llfuse
'''
if readonly is None:
readonly = S.readonly
else:
if not readonly and S.readonly:
warning(
"Store %s is readonly, using readonly option for mount (was %r)",
S, readonly)
readonly = True
# forget the archive if readonly
if readonly:
if archive is not None:
warning("readonly, forgetting archive %s", archive)
archive = None
log = getLogger(LOGGER_NAME)
log.propagate = False
log_handler = LogFileHandler(LOGGER_FILENAME)
log_formatter = LogFormatter(DEFAULT_BASE_FORMAT)
log_handler.setFormatter(log_formatter)
log.addHandler(log_handler)
X("mount: S=%s", S)
X("mount: E=%s", E)
##dump_Dirent(E, recurse=True)
FS = StoreFS(E,
S=S,
archive=archive,
subpath=subpath,
readonly=readonly,
append_only=append_only,
show_prev_dirent=True)
return FS._vt_runfuse(mnt, fsname=fsname)
def dump_Block(block, indent=''):
''' Dump a Block.
'''
X("%s%s", indent, block)
if block.indirect:
indent += ' '
subblocks = block.subblocks
X(
"%sindirect %d subblocks, span %d bytes", indent, len(subblocks),
len(block)
)
for B in subblocks:
dump_Block(B, indent=indent)
def datetime_from_http_date(s):
''' Parse an HTTP-date from a string, return a datetime object.
See RFC2616 section 3.3.1.
'''
try:
return datetime_from_rfc1123_date(s)
except ValueError as e:
X("datetime_from_rfc1123_date(%r): %s", s, e)
try:
return datetime_from_rfc850_date(s)
except ValueError as e:
X("datetime_from_rfc850_date(%r): %s", s, e)
return datetime_from_asctime_date(s)
def load_fs_inode_dirents(self, D):
''' Load entries from an `fs_inode_dirents` Dir into the Inode table.
'''
X("LOAD FS INODE DIRENTS:")
dump_Dirent(D)
for name, E in D.entries.items():
X(" name=%r, E=%r", name, E)
with Pfx(name):
# get the refcount from the :uuid:refcount" name
_, refcount_s = name.split(':')[:2]
I = self.add(E)
I.refcount = int(refcount_s)
X(" I=%s", I)
def setxattr(self, inum, xattr_name, xattr_value):
''' Set the extended attribute `xattr_name` to `xattr_value`
on inode `inum`.
'''
if self.readonly:
OS_EROFS("fs is read only")
E = self.i2E(inum)
xattr_name = Meta.xattrify(xattr_name)
if not xattr_name.startswith(XATTR_VT_PREFIX):
# ordinary attribute, set it and return
E.meta.setxattr(xattr_name, xattr_value)
return
# process special attribute names
with Pfx("%s.setxattr(%d,%r,%r)", self, inum, xattr_name, xattr_value):
suffix = xattr_name[len(XATTR_VT_PREFIX):]
with Pfx(suffix):
if suffix == 'block':
# update the Dirent's content directly
if not E.isfile:
OS_EINVAL("tried to update the data content of a nonfile: %s", E)
block_s = Meta.xattrify(xattr_value)
B, offset = parse(block_s)
if offset < len(block_s):
OS_EINVAL("unparsed text after trancription: %r", block_s[offset:])
if not isBlock(B):
OS_EINVAL("not a Block transcription")
info("%s: update .block directly to %r", E, str(B))
E.block = B
return
if suffix == 'control':
argv = shlex.split(xattr_value.decode('utf-8'))
if not argv:
OS_EINVAL("no control command")
op = argv.pop(0)
with Pfx(op):
if op == 'cache':
if argv:
OS_EINVAL("extra arguments: %r", argv)
B = E.block
if B.indirect:
X("ADD BLOCK CACHE FOR %s", B)
bm = self.block_cache.get_blockmap(B)
X("==> BLOCKMAP: %s", bm)
else:
X("IGNORE BLOCK CACHE for %s: not indirect", B)
return
OS_EINVAL("unrecognised control command")
OS_EINVAL("invalid %r prefixed name", XATTR_VT_PREFIX)
def preprocess(context, row):
''' Convert some row columns before assimilation.
'''
if context.index > 0:
for i, attr in enumerate(context.cls.attrs_):
if attr in (
'bit_rate',
'disc_count',
'disc_number',
'my_rating',
'plays',
'sample_rate',
'size',
'time',
'track_count',
'track_number',
'year',
):
row[i] = int(row[i]) if row[i] else None
elif attr in (
'date_added',
'date_modified',
'last_played',
):
row[i] = playlist_date(row[i]) if row[i] else None
X("row = %r", row)
return row
def db_book(self):
''' Return a cached reference to the database book record.
'''
db = self.tree.db
with db.db_session() as session:
X("FETCH BOOK %r", self.dbid)
return db.books.by_id(self.dbid, session=session)
def _run(self, *calargv, subp_options=None):
''' Run a Calibre utility command.
Parameters:
* `calargv`: an iterable of the calibre command to issue;
if the command name is not an absolute path
it is expected to come from `self.CALIBRE_BINDIR_DEFAULT`
* `subp_options`: optional mapping of keyword arguments
to pass to `subprocess.run`
'''
X("calargv=%r", calargv)
if subp_options is None:
subp_options = {}
subp_options.setdefault('check', True)
cmd, *calargv = calargv
if not isabspath(cmd):
cmd = joinpath(self.CALIBRE_BINDIR_DEFAULT, cmd)
print("RUN", cmd, *calargv)
try:
cp = pfx_call(run, [cmd, *calargv], **subp_options)
except CalledProcessError as cpe:
error(
"run fails, exit code %s:\n %s",
cpe.returncode,
' '.join(map(repr, cpe.cmd)),
)
if cpe.stderr:
print(cpe.stderr.replace('\n', ' \n'), file=sys.stderr)
raise
return cp
def acquire(self, timeout=-1, _caller=None):
''' Acquire the lock and note the caller who takes it.
'''
if _caller is None:
_caller = caller()
lock = self._lock
hold = _LockContext(_caller, current_thread())
if timeout != -1:
warning(
"%s:%d: lock %s: timeout=%s", hold.caller.filename,
hold.caller.lineno, lock, timeout
)
contended = False
if True:
if lock.acquire(0):
lock.release()
else:
contended = True
held = self._held
warning(
"%s:%d: lock %s: waiting for contended lock, held by %s:%s:%d",
hold.caller.filename, hold.caller.lineno, lock, held.thread,
held.caller.filename, held.caller.lineno
)
acquired = lock.acquire(timeout=timeout)
if contended:
warning(
"%s:%d: lock %s: %s", hold.caller.filename, hold.caller.lineno, lock,
"acquired" if acquired else "timed out"
)
self._held = hold
if acquired and self.trace_acquire:
X("ACQUIRED %r", self)
stack_dump()
return acquired
def layout_tags(self, tags):
''' Create a layout for `tags`.
'''
##layout = [[self._get_tag_widget(tag)] for tag in sorted(tags)]
layout = [[sg.Text(str(tag))] for tag in sorted(tags)]
X("%s.layout_tags => %r", type(self).__name__, layout)
return layout
def statusline(self, new_status):
''' Set the status line value.
'''
assert new_status is None or isinstance(new_status, str)
self._statusline = new_status
X("statusbar => %r", new_status)
self.widget.statusBar().showMessage(new_status if new_status else '')
def test(self):
if False:
# to help with debugging:
# print the first 16 sync points - some _may_ be in the audio data
bfr = CornuCopyBuffer.from_filename(TESTFILE)
count = 16
while not bfr.at_eof() and count > 0:
bs = b''.join(MP3AudioFrame.scan_for_sync(bfr))
X("AUDIO at %d after %d bytes", bfr.offset, len(bs))
bfr.take(1)
count -= 1
S = os.stat(TESTFILE)
mp3_size = S.st_size
bfr = CornuCopyBuffer.from_filename(TESTFILE)
for offset, frame, post_offset in MP3Frame.scan_with_offsets(bfr):
frame_size = post_offset - offset
frame_bs = bytes(frame)
##frame2 = MP3Frame.from_bytes(frame_bs)
##self.assertIs(type(frame), type(frame2))
# There used to be a round trip size check, but we repair
# some input data and write it out correctly, so the size can
# change. Example: a USC-2 text field missing its BOM.
self.assertEqual(
bfr.offset, mp3_size,
"file size = %d, buffer offset = %d" % (mp3_size, bfr.offset))
self.assertTrue(bfr.at_eof())
bfr.close()
def set_tags(self, tags):
super().set_tags(tags)
tag_row = self._row_of_tags()
self.__layout[1:] = tag_row
X("SET TAG 0: __layout=%r", self.__layout)
##self.update()
##self.contents_changed()
##self.layout(self.__layout)
canvas = self.__canvas.tk_canvas
canvas.delete(canvas.find_all())
for tag in self.tags:
for tag_item in tag_row:
canvas.create_window(20, 20, window=tag_item.Widget)
self.__layout[1:] = []
X("SET TAG 0: __layout=%r", self.__layout)
##self.update()
self.contents_changed()
def flush(self):
''' Commit file contents to Store.
Chooses a scanner based on the Dirent.name.
'''
X("FileHandle.flush: self.E.name=%r", self.E.name)
mime_type = self.E.meta.mime_type
if mime_type is None:
scanner = None
else:
X("look up scanner from mime_type %r", mime_type)
scanner = scanner_from_mime_type(mime_type)
if scanner is None:
X("look up scanner from filename %r", self.E.name)
scanner = scanner_from_filename(self.E.name)
self.E.flush(scanner, dispatch=self.bg)
## no touch, already done by any writes
X("FileHandle.Flush DONE")
def pixmap(self):
''' The image pixmap.
'''
pm = self._pixmap
if not pm:
X("load pixmap from %r", self.image_path)
pm = self._pixmap = QPixmap(self.image_path)
return pm
def add(self, new_tab):
''' Add an element to the tab set.
'''
assert isinstance(new_tab, _Element)
self.tabs.append(new_tab)
X("add qt %s to tab set %s", new_tab.widget, self.widget)
self.widget.addTab(new_tab.widget, new_tab.title)
return new_tab
def access(self,
access_mode,
access_uid=None,
access_gid=None,
default_uid=None,
default_gid=None):
''' POSIX like access call, accepting os.access `access_mode`.
Parameters:
* `access_mode`: a bitmask of os.{R_OK,W_OK,X_OK} as for
the os.access function.
* `access_uid`: the effective uid of the querying user.
* `access_gid`: the effective gid of the querying user.
* `default_uid`: the reference uid to use if this Meta.uid is None
* `default_gid`: the reference gid to use if this Meta.gid is None
If the Meta has no uid or `access_uid == Meta.uid`,
use the owner permissions.
Otherwise if the Meta has no gid or `access_gid == Meta.gid`,
use the group permissions.
Otherwise use the "other" permissions.
'''
X("META.ACCESS...")
u = self.uid
if u is None:
u = default_uid
g = self.gid
if g is None:
g = default_gid
perms = self.unix_perm_bits
if access_mode & os.R_OK:
if u is None or (access_uid is not None and access_uid == u):
if not (perms >> 6) & 4:
return False
elif g is None or (access_gid is not None and access_gid == g):
if not (perms >> 3) & 4:
return False
elif not perms & 4:
return False
if access_mode & os.W_OK:
if u is None or (access_uid is not None and access_uid == u):
if not (perms >> 6) & 2:
return False
elif g is None or (access_gid is not None and access_gid == g):
if not (perms >> 3) & 2:
return False
elif not perms & 2:
return False
if access_mode & os.X_OK:
if u is None or (access_uid is not None and access_uid == u):
if not (perms >> 6) & 1:
return False
elif g is None or (access_gid is not None and access_gid == g):
if not (perms >> 3) & 1:
return False
elif not perms & 1:
return False
return True
def dump_Dirent(E, indent='', recurse=False, not_dir=False):
''' Dump a Dirent.
'''
X("%s%r", indent, E)
if E.isdir and not not_dir:
indent += ' '
for name in sorted(E.keys()):
E2 = E[name]
dump_Dirent(E2, indent, recurse=recurse, not_dir=not recurse)
def line(self):
line = "%-15s %d/%s" % (self.name if self.name else '',
self.portnum,
self.proto)
if self.aliases:
X("line=%s, aliases=%s", line, self.aliases)
line += ' ' + ' '.join(self.aliases)
if self.comment:
line += ' # ' + self.comment
return line
def insert1(self, **kw):
''' Insert a single row, return the new row id.
'''
column_names = []
values = []
for column_name, value in kw.items():
column_names.append(column_name)
values.append(value)
X("insert1: column_names=%r, values=%r", column_names, values)
return self.insert(column_names, (values,))
def __init__(self, ont_path: str):
self.ont_path = ont_path
tagsets, ont_pfx_map = self.tagsetses_from_path(ont_path)
super().__init__(tagsets)
# apply any prefix TagSetses
for prefix, subtagsets in sorted(ont_pfx_map.items(), key=lambda item:
(len(item[0]), item[0])):
prefix_ = prefix + '.'
X("add %r => %s", prefix_, subtagsets)
self.add_tagsets(subtagsets, prefix_)
def default(self, line):
''' Default command action.
'''
if line == 'EOF':
return True
try:
exec_code(line, globals(), self.vars)
except Exception as e:
X("Exception: %s", e)
self.stdout.flush()
return False