def shutdown(cls): """Stop the librarian process.""" global QUEUE QUEUE.put(LibrarianDaemon.SHUTDOWN_TASK) if cls.proc: LOG.debug("Waiting to shut down...") cls.proc.join()
def obj_moved(src_path, dest_path, cls): """Move a comic in the db.""" if cls == Folder and not does_folder_contain_comics(dest_path): LOG.debug(f"Ignoring {dest_path} folder without comics.") return obj = cls.objects.only("path", "library", "parent_folder", "folder").get( path=src_path ) if cls.objects.filter(path=dest_path).exists(): """ Files get moved before folders which means they create new folders before we can move the old one, leaving the folder orphaned. So we delete it. This means we create a lot of new folder objects. The solution for this is to implement an event aggregation queue in watcherd and handle a big folder move as one event. """ obj_deleted(src_path, cls) LOG.debug( f"Deleted {cls.__name__} {src_path} becuase {dest_path} already exists." ) return obj.path = dest_path folders = Importer.get_folders(obj.library, obj.path) obj.parent_folder = folders[-1] if folders else None obj.folder.set(folders) obj.save() LOG.info(f"Moved {cls.__name__} from {src_path} to {dest_path}") QUEUE.put(LibraryChangedTask())
def delete_queryset(self, request, queryset): """Bulk delete.""" for obj in queryset: purge_all_covers(obj) super().delete_queryset(request, queryset) QUEUE.put(WatcherCronTask(sleep=1)) QUEUE.put(LibraryChangedTask())
def get_x_cover_path(self, obj): """Ensure comic cover exists for any cover_path we send.""" comic_path = obj.get("x_path") cover_path = obj.get("x_cover_path") task = ComicCoverCreateTask(comic_path, cover_path, False) QUEUE.put(task) return cover_path
def scan_existing(library, cls, force=False): """ Scan existing comics and folders. Remove the missing ones. Update the outated comics. """ num_removed = 0 num_updated = 0 num_passed = 0 objs = cls.objects.filter(library=library).only("path") for obj in objs: path = Path(obj.path) task = None if cls == Folder and not path.is_dir(): task = FolderDeletedTask(obj.path) elif cls == Comic and not path.is_file(): task = ComicDeletedTask(obj.path) if task: num_removed += 1 elif cls == Comic and (force or is_obj_outdated(obj)): task = ComicModifiedTask(obj.path, library.id) num_updated += 1 if task: QUEUE.put(task) else: num_passed += 1 cls_name = cls.__name__.lower() LOG.info(f"Queued {num_removed} {cls_name}s for removal.") if cls == Comic: LOG.info(f"Queued {num_updated} {cls_name}s for re-import.") LOG.debug(f"Ignored {num_passed} {cls_name}s that are up to date.")
def _on_change(self): """Signal UI that its out of date.""" # Heavy handed refresh everything, but simple. # Folder View could only change the group view and let the ui decide # Registration only needs to change the enable flag task = LibraryChangedTask() QUEUE.put(task)
def scan_for_new(library): """Add comics from a library that aren't in the db already.""" # Will hafta check each comic's updated_at once I allow writing LOG.info(f"Scanning {library.path}") num_new = 0 num_skipped = 0 num_passed = 0 for root, _, filenames in os.walk(library.path): walk_root = Path(root) LOG.debug(f"Scanning {walk_root}") for filename in sorted(filenames): path = walk_root / filename if COMIC_MATCHER.search(str(path)) is not None: if not Comic.objects.filter(path=path).exists(): task = ComicModifiedTask(path, library.id) QUEUE.put(task) num_new += 1 else: num_passed += 1 else: num_skipped += 1 LOG.info(f"Queued {num_new} comics for import.") LOG.debug(f"Skipped {num_skipped} non-comic files.") LOG.debug(f"Ignored {num_passed} comics already in the db.")
def create_comic_cover(comic_path, db_cover_path, force=False): """Create a comic cover thumnail and save it to disk.""" try: if db_cover_path == MISSING_COVER_FN and not force: LOG.debug(f"Cover for {comic_path} missing.") return fs_cover_path = COVER_ROOT / db_cover_path if fs_cover_path.exists() and not force: LOG.debug(f"Cover already exists {comic_path} {db_cover_path}") return fs_cover_path.parent.mkdir(exist_ok=True, parents=True) if comic_path is None: comic = Comic.objects.only("path").get(cover_path=db_cover_path) comic_path = comic.path # Reopens the car, so slightly inefficient. car = ComicArchive(comic_path) im = Image.open(BytesIO(car.get_cover_image())) im.thumbnail(THUMBNAIL_SIZE) im.save(fs_cover_path, im.format) LOG.info(f"Created cover thumbnail for: {comic_path}") QUEUE.put(LibraryChangedTask()) except Exception as exc: LOG.error(f"Failed to create cover thumb for {comic_path}") LOG.exception(exc) Comic.objects.filter(comic_path=comic_path).update( cover_path=MISSING_COVER_FN) LOG.warn(f"Marked cover for {comic_path} missing.")
def scan_all_roots(force): """Scan all the librarys.""" LOG.info("Scanning all librarys...") rps = Library.objects.all().only("pk") for library in rps: task = ScanRootTask(library.pk, force) QUEUE.put(task)
def scan_root(pk, force=False): """Scan a library.""" library = Library.objects.only("scan_in_progress", "last_scan", "path").get(pk=pk) if library.scan_in_progress: LOG.info(f"Scan in progress for {library.path}. Not rescanning") return LOG.info(f"Scanning {library.path}...") library.scan_in_progress = True library.save() try: if Path(library.path).is_dir(): force = force or library.last_scan is None scan_existing(library, Comic, force) scan_existing(library, Folder) scan_for_new(library) cleanup_failed_imports(library) if force or library.last_scan is None or library.schema_version is None: library.schema_version = SCHEMA_VERSION library.last_scan = timezone.now() else: LOG.warning(f"Could not find {library.path}. Not scanning.") except Exception as exc: LOG.exception(exc) finally: library.scan_in_progress = False library.save() is_failed_imports = FailedImport.objects.exists() QUEUE.put(ScanDoneTask(failed_imports=is_failed_imports, sleep=0)) LOG.info(f"Scan for {library.path} finished.")
def on_modified(self, event): """Put a comic modified task on the queue.""" if event.is_directory or self.is_ignored(event.is_directory, event.src_path): return src_path = Path(event.src_path) self._wait_for_copy(src_path, event.is_directory) task = ComicModifiedTask(event.src_path, self.pk) QUEUE.put(task)
def process_task(self, task): """Process an individual task popped off the queue.""" run = True try: if isinstance(task, ScanRootTask): msg = WS_MSGS["SCAN_LIBRARY"] self.send_json(MessageType.ADMIN_BROADCAST, msg) # no retry scan_root(task.library_id, task.force) elif isinstance(task, ScanDoneTask): if task.failed_imports: msg = WS_MSGS["FAILED_IMPORTS"] else: msg = WS_MSGS["SCAN_DONE"] if not self.send_json(MessageType.ADMIN_BROADCAST, msg): QUEUE.put(task) elif isinstance(task, ComicModifiedTask): import_comic(task.library_id, task.src_path) elif isinstance(task, ComicCoverCreateTask): # Cover creation is cpu bound, farm it out. args = (task.src_path, task.db_cover_path, task.force) self.pool.apply_async(create_comic_cover, args=args) elif isinstance(task, FolderMovedTask): obj_moved(task.src_path, task.dest_path, Folder) elif isinstance(task, ComicMovedTask): obj_moved(task.src_path, task.dest_path, Comic) elif isinstance(task, ComicDeletedTask): obj_deleted(task.src_path, Comic) elif isinstance(task, FolderDeletedTask): obj_deleted(task.src_path, Folder) elif isinstance(task, LibraryChangedTask): msg = WS_MSGS["LIBRARY_CHANGED"] if not self.send_json(MessageType.BROADCAST, msg): QUEUE.put(task) elif isinstance(task, WatcherCronTask): sleep(task.sleep) self.watcher.set_all_library_watches() elif isinstance(task, ScannerCronTask): sleep(task.sleep) scan_cron() elif isinstance(task, UpdateCronTask): sleep(task.sleep) update_codex(task.force) elif isinstance(task, RestartTask): sleep(task.sleep) restart_codex() elif task == self.SHUTDOWN_TASK: LOG.info("Shutting down Librarian...") run = False else: LOG.warning(f"Unhandled task popped: {task}") except (Comic.DoesNotExist, Folder.DoesNotExist) as exc: LOG.warning(exc) except Exception as exc: LOG.exception(exc) return run
def on_deleted(self, event): """Put a comic deleted task on the queue.""" if self.is_ignored(event.is_directory, event.src_path): return self._wait_for_delete(event.src_path) if event.is_directory: task = FolderDeletedTask(event.src_path) else: task = ComicDeletedTask(event.src_path) QUEUE.put(task)
def scan_cron(): """Regular cron for scanning.""" librarys = Library.objects.filter(enable_scan_cron=True).only( "pk", "schema_version", "last_scan") for library in librarys: force_import = library.schema_version < SCHEMA_VERSION if is_time_to_scan(library) or force_import: try: task = ScanRootTask(library.pk, force_import) QUEUE.put(task) except Exception as exc: LOG.error(exc)
def obj_deleted(src_path, cls): """Delete a class.""" try: obj = cls.objects.get(path=src_path) except cls.DoesNotExist: LOG.debug(f"{cls.__name__} {src_path} does not exist. Can't delete.") return if cls == Comic: purge_cover(obj) obj.delete() LOG.info(f"Deleted {cls.__name__} {src_path}") QUEUE.put(LibraryChangedTask())
def import_metadata(self): """Import a comic into the db, from a path, using Comicbox.""" # Buld the catalogue tree BEFORE we clean the metadata created = False ( self.md["publisher"], self.md["imprint"], self.md["series"], self.md["volume"], ) = self.get_browser_group_tree() library = Library.objects.only("pk", "path").get(pk=self.library_id) self.set_locales() credits = self.get_credits() m2m_fields = self.get_many_to_many_instances() self.clean_metadata() # the order of this is pretty important foreign_keys = self.get_foreign_keys() self.md.update(foreign_keys) folders = self.get_folders(library, self.path) if folders: self.md["parent_folder"] = folders[-1] self.md["library"] = library self.md["path"] = self.path self.md["size"] = Path(self.path).stat().st_size self.md["max_page"] = max(self.md["page_count"] - 1, 0) cover_path = get_cover_path(self.path) self.md["cover_path"] = cover_path if credits: m2m_fields["credits"] = credits m2m_fields["folder"] = folders comic, created = Comic.objects.update_or_create( defaults=self.md, path=self.path ) comic.myself = comic # Add the m2m2 instances afterwards, can't initialize with them for attr, instance_list in m2m_fields.items(): getattr(comic, attr).set(instance_list) comic.save() QUEUE.put(ComicCoverCreateTask(self.path, cover_path, True)) # If it works, clear the failed import FailedImport.objects.filter(path=self.path).delete() if created: verb = "Created" else: verb = "Updated" LOG.info(f"{verb} comic {str(comic.volume.series.name)} #{comic.issue:03}") QUEUE.put(LibraryChangedTask()) return created
def get(self, request, *args, **kwargs): """Return if any libraries are scanning.""" any_in_progress = Library.objects.filter(scan_in_progress=True).exists() any_in_progress |= not QUEUE.empty() data = {"scanInProgress": any_in_progress} serializer = ScanNotifySerializer(data) return Response(serializer.data)
def on_moved(self, event): """ Put a comic moved task on the queue. Watchdog events can arrive in any order, but often file events occur before folder events. This ends up leading us to create new folders and delete old ones on move instead of moving the folders. The solution is to implement lazydog in a cross platform manner. Make a delay queue for all events and see if they can be bundled as a single top-level folder move event. For the future. """ if self.is_ignored(event.is_directory, event.dest_path): return self._wait_for_copy(event.dest_path, event.is_directory) if event.is_directory: task = FolderMovedTask(event.src_path, event.dest_path) else: task = ComicMovedTask(event.src_path, event.dest_path) QUEUE.put(task)
def _on_change(self, obj, created=False): """Events for when the library has changed.""" # XXX These sleep values are for waiting for db consistency # between processes. Klugey. if created: QUEUE.put(LibraryChangedTask()) QUEUE.put(ScannerCronTask(sleep=1)) QUEUE.put(WatcherCronTask(sleep=1))
def run(self): """Watch a path and log the events.""" LOG.info("Started cron") with self.COND: while self.run_thread: QUEUE.put(ScannerCronTask(sleep=0)) QUEUE.put(WatcherCronTask(sleep=0)) QUEUE.put(UpdateCronTask(sleep=0, force=False)) self.COND.wait(timeout=self.WAIT_INTERVAL) LOG.info("Stopped cron.")
def run(self): """ Process tasks from the queue. This proces also runs the crond thread and the Watchdog Observer threads. """ try: LOG.debug("Started Librarian.") self.start_threads() run = True LOG.info("Librarian started threads and waiting for tasks.") while run: task = QUEUE.get() run = self.process_task(task) self.stop_threads() except Exception as exc: LOG.error("Librarian crashed.") LOG.exception(exc) LOG.info("Stopped Librarian.")
def restart_now(self, request, queryset): """Send a restart task immediately.""" QUEUE.put(RestartTask(sleep=0))
def update_now(self, request, queryset): """Trigger an update task immediately.""" QUEUE.put(UpdateCronTask(sleep=0, force=True))
def _scan(self, request, queryset, force): """Queue a scan task for the library.""" pks = queryset.values_list("pk", flat=True) for pk in pks: task = ScanRootTask(pk, force) QUEUE.put(task)
def delete_model(self, request, obj): """Stop watching on delete.""" purge_all_covers(obj) super().delete_model(request, obj) QUEUE.put(WatcherCronTask(sleep=1)) QUEUE.put(LibraryChangedTask())