def start_listeners():
log.info("Listening for file changes")
listen_for_file_changes()
db_listener = Thread(target=listen_for_db_changes)
db_listener.name = "Database change listener"
db_listener.daemon = True
db_listener.start()
def _load_file(xmpf: Path, db: sqlite3.Connection):
meta: CuteMeta = CuteMeta.from_file(xmpf)
timestamp = meta.last_updated
if not meta.uid: return
if not meta.hash: return
log.info("Loading %r", str(xmpf))
# Sync data
if meta.generate_keywords():
log.info("Updated autogenerated keywords")
timestamp = datetime.utcnow() # make sure we set the correct timestamp
with __lock:
try:
__hashes.add(meta.hash)
except KeyError:
log.warn("Possible duplicate %r", str(xmpf))
db.execute(
f"""
INSERT INTO Metadata (
last_updated, uid, hash, caption, author, source, group_id, rating, source_other, source_via
) VALUES (
?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)""", (timestamp, meta.uid, meta.hash, meta.caption, meta.author,
meta.source, meta.group_id, meta.rating, meta.source_other,
meta.source_via))
if meta.date:
db.execute(
"""
UPDATE Metadata SET date = ? WHERE uid is ?
""", (meta.date, meta.uid))
if meta.keywords:
db.executemany(
f"""
INSERT INTO Metadata_Keywords VALUES (
?, ?
)
""", [(meta.uid, keyword) for keyword in meta.keywords])
if meta.collections:
db.executemany(
f"""
INSERT INTO Metadata_Collections VALUES (
?, ?
)
""", [(meta.uid, collection) for collection in meta.collections])
def _save_file(xmpf: Path, db: sqlite3.Connection):
with __lock:
meta = CuteMeta.from_file(xmpf)
f_last_updated = meta.last_updated
uid = UUID(xmpf.stem)
db_last_updated = db.execute(
"""
select last_updated from Metadata where uid is ?
""", (uid, )).fetchone()[0]
if f_last_updated > db_last_updated:
log.info("Reading from file %r", str(xmpf))
log.debug("file: %s database: %s", f_last_updated, db_last_updated)
_save_meta(meta, f_last_updated, db)
db.commit()
def _save_meta(meta: CuteMeta, timestamp: datetime, db: sqlite3.Connection):
# Sync data
if meta.generate_keywords():
log.info("Updated autogenerated keywords")
timestamp = datetime.utcnow() # make sure we set the correct timestamp
db.execute(
"""
DELETE FROM Metadata_Keywords WHERE uid is ?
""", (meta.uid, ))
db.execute(
"""
DELETE FROM Metadata_Collections WHERE uid is ?
""", (meta.uid, ))
if meta.keywords:
db.executemany(
f"""
INSERT INTO Metadata_Keywords VALUES (
?, ?
)
""", [(meta.uid, keyword) for keyword in meta.keywords])
if meta.collections:
db.executemany(
f"""
INSERT INTO Metadata_Collections VALUES (
?, ?
)
""", [(meta.uid, collection) for collection in meta.collections])
db.execute(
"""
UPDATE Metadata SET
last_updated = ?,
hash = ?,
caption = ?,
author = ?,
source = ?,
group_id = ?,
rating = ?,
source_other = ?,
source_via = ?
WHERE
uid is ?
""", (timestamp, meta.hash, meta.caption, meta.author, meta.source,
meta.group_id, meta.rating, meta.source_other, meta.source_via,
meta.uid))
def listen_for_db_changes():
last_updated = datetime.utcfromtimestamp(
os.path.getmtime(config.metadbf.resolve()))
# We need our own connection since this is on a different thread
db = connect_db()
while True:
time.sleep(10)
modified = db.execute(
"""
select * from Metadata where last_updated > ?
""", (last_updated, )).fetchall()
last_updated = datetime.utcnow()
for data in modified:
with __lock:
filename = xmp_file_for_uid(data["uid"])
meta = CuteMeta.from_file(filename)
f_last_updated = meta.last_updated
db_last_updated = data["last_updated"]
if db_last_updated > f_last_updated:
log.info("Writing to file %r", str(filename))
log.debug("file: %s database: %s", f_last_updated,
db_last_updated)
for name, v in zip(data.keys(), data):
setattr(meta, name, v)
keywords = db.execute(
"""
select keyword from Metadata_Keywords where uid = ?
""", (data["uid"], )).fetchmany()
collections = db.execute(
"""
select collection from Metadata_Collections where uid = ?
""", (data["uid"], )).fetchmany()
meta.keywords = set(k[0] for k in keywords)
meta.collections = set(c[0] for c in collections)
meta.last_updated = db_last_updated # Make sure that the entry in the database stays the same as the file
meta.write()
def _remove_image(uid: UUID, db: sqlite3.Connection):
log.info("Removing %s", uid)
imghash = db.execute(
"""
SELECT hash FROM Metadata WHERE uid = ?
""", (uid, )).fetchone()["hash"]
cnthash = db.execute(
"""
SELECT count(uid) FROM Metadata where hash = ?
""", (imghash, )).fetchone()[0]
db.execute("DELETE FROM Metadata_Keywords WHERE uid = ?", (uid, ))
db.execute("DELETE FROM Metadata_Collections WHERE uid = ?", (uid, ))
db.execute("DELETE FROM Metadata WHERE uid = ?", (uid, ))
if cnthash == 1:
with __lock:
try:
__hashes.remove(
imghash
) # Only one hash by this name, it doesnt exist anymore now
except KeyError:
pass
def exit():
log.info("Closing database connection")
__db.commit()
__db.close()
with open(config.hashdbf, "wb") as hashdbfp, __lock:
log.info("Writing hashes to file...")
__hashes.write_to_file(hashdbfp)
log.info("Done!")
def init_db():
global __db, __hashes
log.info("Scanning database")
refresh_cache = False
if not config.metadbf.exists() or not config.hashdbf.exists():
config.metadbf.touch(exist_ok=True)
refresh_cache = True
log.info("Setting up database %r", str(config.metadbf))
__db = connect_db()
__db.executescript(f"""
CREATE TABLE IF not EXISTS Metadata (
uid UUID PRIMARY KEY not null,
last_updated timestamp not null DEFAULT(strftime('%Y-%m-%d %H:%M:%f', 'now')),
hash TEXT not null,
caption TEXT,
author TEXT,
source TEXT,
group_id UUID,
date timestamp not null DEFAULT(strftime('%Y-%m-%d %H:%M:%f', 'now')),
rating Rating,
source_other PSet,
source_via PSet
) WITHOUT ROWID;
CREATE TABLE IF not EXISTS Metadata_Keywords (
uid UUID not null,
keyword TEXT NOT NULL CHECK (keyword REGEXP '{config.tag_regex}')
);
CREATE TABLE IF not EXISTS Metadata_Collections (
uid UUID not null,
collection TEXT NOT NULL CHECK (collection REGEXP '{config.tag_regex}')
);
""")
if refresh_cache:
__hashes = HashTree(config.hash_length)
log.info("Loading folder %r into database", str(config.image_folder))
for xmpf in config.image_folder.glob("*.xmp"):
if not xmpf.is_file(): continue
if xmpf.name.startswith("."): continue
_load_file(xmpf, __db)
__db.commit()
with open(config.hashdbf, "wb") as hashdbfp, __lock:
log.info("Writing hashes to file...")
__hashes.write_to_file(hashdbfp)
else:
with open(config.hashdbf, "rb") as hashdbfp, __lock:
log.info("Loading hashes from cache %r", str(config.hashdbf))
__hashes = HashTree.read_from_file(hashdbfp, config.hash_length)
log.info("Catching up with image folder")
uuids_in_folder = set()
for xmpf in config.image_folder.glob("*.xmp"):
if not xmpf.is_file(): continue
if xmpf.name.startswith("."): continue
try:
uuid = UUID(xmpf.stem)
uuids_in_folder.add(uuid)
except:
continue
uuids_in_database = set(
d[0] for d in __db.execute("select uid from Metadata").fetchall())
for uid in uuids_in_folder - uuids_in_database: # recently added
_load_file(xmp_file_for_uid(uid), __db)
for uid in uuids_in_database - uuids_in_folder: # recently deleted
_remove_image(uid, __db)
for uid in uuids_in_database:
try:
_save_file(xmp_file_for_uid(uid), __db)
except FileNotFoundError:
pass # was deleted earlier
__db.commit()
log.info("Done!")
def exit():
log.info("Closing database connection")
__db.commit()
__db.close()
with open(config.hashdbf, "wb") as hashdbfp, __lock:
log.info("Writing hashes to file...")
__hashes.write_to_file(hashdbfp)
log.info("Done!")
atexit.register(exit)