def shutdown(cls):
"""Stop the librarian process."""
global QUEUE
QUEUE.put(LibrarianDaemon.SHUTDOWN_TASK)
if cls.proc:
LOG.debug("Waiting to shut down...")
cls.proc.join()
def obj_moved(src_path, dest_path, cls):
"""Move a comic in the db."""
if cls == Folder and not does_folder_contain_comics(dest_path):
LOG.debug(f"Ignoring {dest_path} folder without comics.")
return
obj = cls.objects.only("path", "library", "parent_folder", "folder").get(
path=src_path
)
if cls.objects.filter(path=dest_path).exists():
"""
Files get moved before folders which means they create new folders
before we can move the old one, leaving the folder orphaned. So we
delete it. This means we create a lot of new folder objects. The
solution for this is to implement an event aggregation queue in
watcherd and handle a big folder move as one event.
"""
obj_deleted(src_path, cls)
LOG.debug(
f"Deleted {cls.__name__} {src_path} becuase {dest_path} already exists."
)
return
obj.path = dest_path
folders = Importer.get_folders(obj.library, obj.path)
obj.parent_folder = folders[-1] if folders else None
obj.folder.set(folders)
obj.save()
LOG.info(f"Moved {cls.__name__} from {src_path} to {dest_path}")
QUEUE.put(LibraryChangedTask())
def delete_queryset(self, request, queryset):
"""Bulk delete."""
for obj in queryset:
purge_all_covers(obj)
super().delete_queryset(request, queryset)
QUEUE.put(WatcherCronTask(sleep=1))
QUEUE.put(LibraryChangedTask())
def get_x_cover_path(self, obj):
"""Ensure comic cover exists for any cover_path we send."""
comic_path = obj.get("x_path")
cover_path = obj.get("x_cover_path")
task = ComicCoverCreateTask(comic_path, cover_path, False)
QUEUE.put(task)
return cover_path
def scan_existing(library, cls, force=False):
"""
Scan existing comics and folders.
Remove the missing ones. Update the outated comics.
"""
num_removed = 0
num_updated = 0
num_passed = 0
objs = cls.objects.filter(library=library).only("path")
for obj in objs:
path = Path(obj.path)
task = None
if cls == Folder and not path.is_dir():
task = FolderDeletedTask(obj.path)
elif cls == Comic and not path.is_file():
task = ComicDeletedTask(obj.path)
if task:
num_removed += 1
elif cls == Comic and (force or is_obj_outdated(obj)):
task = ComicModifiedTask(obj.path, library.id)
num_updated += 1
if task:
QUEUE.put(task)
else:
num_passed += 1
cls_name = cls.__name__.lower()
LOG.info(f"Queued {num_removed} {cls_name}s for removal.")
if cls == Comic:
LOG.info(f"Queued {num_updated} {cls_name}s for re-import.")
LOG.debug(f"Ignored {num_passed} {cls_name}s that are up to date.")
def _on_change(self):
"""Signal UI that its out of date."""
# Heavy handed refresh everything, but simple.
# Folder View could only change the group view and let the ui decide
# Registration only needs to change the enable flag
task = LibraryChangedTask()
QUEUE.put(task)
def scan_for_new(library):
"""Add comics from a library that aren't in the db already."""
# Will hafta check each comic's updated_at once I allow writing
LOG.info(f"Scanning {library.path}")
num_new = 0
num_skipped = 0
num_passed = 0
for root, _, filenames in os.walk(library.path):
walk_root = Path(root)
LOG.debug(f"Scanning {walk_root}")
for filename in sorted(filenames):
path = walk_root / filename
if COMIC_MATCHER.search(str(path)) is not None:
if not Comic.objects.filter(path=path).exists():
task = ComicModifiedTask(path, library.id)
QUEUE.put(task)
num_new += 1
else:
num_passed += 1
else:
num_skipped += 1
LOG.info(f"Queued {num_new} comics for import.")
LOG.debug(f"Skipped {num_skipped} non-comic files.")
LOG.debug(f"Ignored {num_passed} comics already in the db.")
def create_comic_cover(comic_path, db_cover_path, force=False):
"""Create a comic cover thumnail and save it to disk."""
try:
if db_cover_path == MISSING_COVER_FN and not force:
LOG.debug(f"Cover for {comic_path} missing.")
return
fs_cover_path = COVER_ROOT / db_cover_path
if fs_cover_path.exists() and not force:
LOG.debug(f"Cover already exists {comic_path} {db_cover_path}")
return
fs_cover_path.parent.mkdir(exist_ok=True, parents=True)
if comic_path is None:
comic = Comic.objects.only("path").get(cover_path=db_cover_path)
comic_path = comic.path
# Reopens the car, so slightly inefficient.
car = ComicArchive(comic_path)
im = Image.open(BytesIO(car.get_cover_image()))
im.thumbnail(THUMBNAIL_SIZE)
im.save(fs_cover_path, im.format)
LOG.info(f"Created cover thumbnail for: {comic_path}")
QUEUE.put(LibraryChangedTask())
except Exception as exc:
LOG.error(f"Failed to create cover thumb for {comic_path}")
LOG.exception(exc)
Comic.objects.filter(comic_path=comic_path).update(
cover_path=MISSING_COVER_FN)
LOG.warn(f"Marked cover for {comic_path} missing.")
def scan_all_roots(force):
"""Scan all the librarys."""
LOG.info("Scanning all librarys...")
rps = Library.objects.all().only("pk")
for library in rps:
task = ScanRootTask(library.pk, force)
QUEUE.put(task)
def scan_root(pk, force=False):
"""Scan a library."""
library = Library.objects.only("scan_in_progress", "last_scan",
"path").get(pk=pk)
if library.scan_in_progress:
LOG.info(f"Scan in progress for {library.path}. Not rescanning")
return
LOG.info(f"Scanning {library.path}...")
library.scan_in_progress = True
library.save()
try:
if Path(library.path).is_dir():
force = force or library.last_scan is None
scan_existing(library, Comic, force)
scan_existing(library, Folder)
scan_for_new(library)
cleanup_failed_imports(library)
if force or library.last_scan is None or library.schema_version is None:
library.schema_version = SCHEMA_VERSION
library.last_scan = timezone.now()
else:
LOG.warning(f"Could not find {library.path}. Not scanning.")
except Exception as exc:
LOG.exception(exc)
finally:
library.scan_in_progress = False
library.save()
is_failed_imports = FailedImport.objects.exists()
QUEUE.put(ScanDoneTask(failed_imports=is_failed_imports, sleep=0))
LOG.info(f"Scan for {library.path} finished.")
def on_modified(self, event):
"""Put a comic modified task on the queue."""
if event.is_directory or self.is_ignored(event.is_directory, event.src_path):
return
src_path = Path(event.src_path)
self._wait_for_copy(src_path, event.is_directory)
task = ComicModifiedTask(event.src_path, self.pk)
QUEUE.put(task)
def process_task(self, task):
"""Process an individual task popped off the queue."""
run = True
try:
if isinstance(task, ScanRootTask):
msg = WS_MSGS["SCAN_LIBRARY"]
self.send_json(MessageType.ADMIN_BROADCAST, msg)
# no retry
scan_root(task.library_id, task.force)
elif isinstance(task, ScanDoneTask):
if task.failed_imports:
msg = WS_MSGS["FAILED_IMPORTS"]
else:
msg = WS_MSGS["SCAN_DONE"]
if not self.send_json(MessageType.ADMIN_BROADCAST, msg):
QUEUE.put(task)
elif isinstance(task, ComicModifiedTask):
import_comic(task.library_id, task.src_path)
elif isinstance(task, ComicCoverCreateTask):
# Cover creation is cpu bound, farm it out.
args = (task.src_path, task.db_cover_path, task.force)
self.pool.apply_async(create_comic_cover, args=args)
elif isinstance(task, FolderMovedTask):
obj_moved(task.src_path, task.dest_path, Folder)
elif isinstance(task, ComicMovedTask):
obj_moved(task.src_path, task.dest_path, Comic)
elif isinstance(task, ComicDeletedTask):
obj_deleted(task.src_path, Comic)
elif isinstance(task, FolderDeletedTask):
obj_deleted(task.src_path, Folder)
elif isinstance(task, LibraryChangedTask):
msg = WS_MSGS["LIBRARY_CHANGED"]
if not self.send_json(MessageType.BROADCAST, msg):
QUEUE.put(task)
elif isinstance(task, WatcherCronTask):
sleep(task.sleep)
self.watcher.set_all_library_watches()
elif isinstance(task, ScannerCronTask):
sleep(task.sleep)
scan_cron()
elif isinstance(task, UpdateCronTask):
sleep(task.sleep)
update_codex(task.force)
elif isinstance(task, RestartTask):
sleep(task.sleep)
restart_codex()
elif task == self.SHUTDOWN_TASK:
LOG.info("Shutting down Librarian...")
run = False
else:
LOG.warning(f"Unhandled task popped: {task}")
except (Comic.DoesNotExist, Folder.DoesNotExist) as exc:
LOG.warning(exc)
except Exception as exc:
LOG.exception(exc)
return run
def on_deleted(self, event):
"""Put a comic deleted task on the queue."""
if self.is_ignored(event.is_directory, event.src_path):
return
self._wait_for_delete(event.src_path)
if event.is_directory:
task = FolderDeletedTask(event.src_path)
else:
task = ComicDeletedTask(event.src_path)
QUEUE.put(task)
def scan_cron():
"""Regular cron for scanning."""
librarys = Library.objects.filter(enable_scan_cron=True).only(
"pk", "schema_version", "last_scan")
for library in librarys:
force_import = library.schema_version < SCHEMA_VERSION
if is_time_to_scan(library) or force_import:
try:
task = ScanRootTask(library.pk, force_import)
QUEUE.put(task)
except Exception as exc:
LOG.error(exc)
def obj_deleted(src_path, cls):
"""Delete a class."""
try:
obj = cls.objects.get(path=src_path)
except cls.DoesNotExist:
LOG.debug(f"{cls.__name__} {src_path} does not exist. Can't delete.")
return
if cls == Comic:
purge_cover(obj)
obj.delete()
LOG.info(f"Deleted {cls.__name__} {src_path}")
QUEUE.put(LibraryChangedTask())
def import_metadata(self):
"""Import a comic into the db, from a path, using Comicbox."""
# Buld the catalogue tree BEFORE we clean the metadata
created = False
(
self.md["publisher"],
self.md["imprint"],
self.md["series"],
self.md["volume"],
) = self.get_browser_group_tree()
library = Library.objects.only("pk", "path").get(pk=self.library_id)
self.set_locales()
credits = self.get_credits()
m2m_fields = self.get_many_to_many_instances()
self.clean_metadata() # the order of this is pretty important
foreign_keys = self.get_foreign_keys()
self.md.update(foreign_keys)
folders = self.get_folders(library, self.path)
if folders:
self.md["parent_folder"] = folders[-1]
self.md["library"] = library
self.md["path"] = self.path
self.md["size"] = Path(self.path).stat().st_size
self.md["max_page"] = max(self.md["page_count"] - 1, 0)
cover_path = get_cover_path(self.path)
self.md["cover_path"] = cover_path
if credits:
m2m_fields["credits"] = credits
m2m_fields["folder"] = folders
comic, created = Comic.objects.update_or_create(
defaults=self.md, path=self.path
)
comic.myself = comic
# Add the m2m2 instances afterwards, can't initialize with them
for attr, instance_list in m2m_fields.items():
getattr(comic, attr).set(instance_list)
comic.save()
QUEUE.put(ComicCoverCreateTask(self.path, cover_path, True))
# If it works, clear the failed import
FailedImport.objects.filter(path=self.path).delete()
if created:
verb = "Created"
else:
verb = "Updated"
LOG.info(f"{verb} comic {str(comic.volume.series.name)} #{comic.issue:03}")
QUEUE.put(LibraryChangedTask())
return created
def get(self, request, *args, **kwargs):
"""Return if any libraries are scanning."""
any_in_progress = Library.objects.filter(scan_in_progress=True).exists()
any_in_progress |= not QUEUE.empty()
data = {"scanInProgress": any_in_progress}
serializer = ScanNotifySerializer(data)
return Response(serializer.data)
def on_moved(self, event):
"""
Put a comic moved task on the queue.
Watchdog events can arrive in any order, but often file events
occur before folder events. This ends up leading us to create
new folders and delete old ones on move instead of moving the
folders. The solution is to implement lazydog in a cross platform
manner. Make a delay queue for all events and see if they can
be bundled as a single top-level folder move event.
For the future.
"""
if self.is_ignored(event.is_directory, event.dest_path):
return
self._wait_for_copy(event.dest_path, event.is_directory)
if event.is_directory:
task = FolderMovedTask(event.src_path, event.dest_path)
else:
task = ComicMovedTask(event.src_path, event.dest_path)
QUEUE.put(task)
def _on_change(self, obj, created=False):
"""Events for when the library has changed."""
# XXX These sleep values are for waiting for db consistency
# between processes. Klugey.
if created:
QUEUE.put(LibraryChangedTask())
QUEUE.put(ScannerCronTask(sleep=1))
QUEUE.put(WatcherCronTask(sleep=1))
def run(self):
"""Watch a path and log the events."""
LOG.info("Started cron")
with self.COND:
while self.run_thread:
QUEUE.put(ScannerCronTask(sleep=0))
QUEUE.put(WatcherCronTask(sleep=0))
QUEUE.put(UpdateCronTask(sleep=0, force=False))
self.COND.wait(timeout=self.WAIT_INTERVAL)
LOG.info("Stopped cron.")
def run(self):
"""
Process tasks from the queue.
This proces also runs the crond thread and the Watchdog Observer
threads.
"""
try:
LOG.debug("Started Librarian.")
self.start_threads()
run = True
LOG.info("Librarian started threads and waiting for tasks.")
while run:
task = QUEUE.get()
run = self.process_task(task)
self.stop_threads()
except Exception as exc:
LOG.error("Librarian crashed.")
LOG.exception(exc)
LOG.info("Stopped Librarian.")
def restart_now(self, request, queryset):
"""Send a restart task immediately."""
QUEUE.put(RestartTask(sleep=0))
def update_now(self, request, queryset):
"""Trigger an update task immediately."""
QUEUE.put(UpdateCronTask(sleep=0, force=True))
def _scan(self, request, queryset, force):
"""Queue a scan task for the library."""
pks = queryset.values_list("pk", flat=True)
for pk in pks:
task = ScanRootTask(pk, force)
QUEUE.put(task)
def delete_model(self, request, obj):
"""Stop watching on delete."""
purge_all_covers(obj)
super().delete_model(request, obj)
QUEUE.put(WatcherCronTask(sleep=1))
QUEUE.put(LibraryChangedTask())
