def test_get_random_page(): car = ComicArchive(ARCHIVE_PATH) page = car.get_page_by_index(4) with open(PAGE_FIVE, "rb") as cif: image = cif.read() assert image == page
def create_comic_cover(comic_path, db_cover_path, force=False): """Create a comic cover thumnail and save it to disk.""" try: if db_cover_path == MISSING_COVER_FN and not force: LOG.debug(f"Cover for {comic_path} missing.") return fs_cover_path = COVER_ROOT / db_cover_path if fs_cover_path.exists() and not force: LOG.debug(f"Cover already exists {comic_path} {db_cover_path}") return fs_cover_path.parent.mkdir(exist_ok=True, parents=True) if comic_path is None: comic = Comic.objects.only("path").get(cover_path=db_cover_path) comic_path = comic.path # Reopens the car, so slightly inefficient. car = ComicArchive(comic_path) im = Image.open(BytesIO(car.get_cover_image())) im.thumbnail(THUMBNAIL_SIZE) im.save(fs_cover_path, im.format) LOG.info(f"Created cover thumbnail for: {comic_path}") QUEUE.put(LibraryChangedTask()) except Exception as exc: LOG.error(f"Failed to create cover thumb for {comic_path}") LOG.exception(exc) Comic.objects.filter(comic_path=comic_path).update( cover_path=MISSING_COVER_FN) LOG.warn(f"Marked cover for {comic_path} missing.")
def test_get_covers(): car = ComicArchive(ARCHIVE_PATH) page = car.get_cover_image() with open(COVER_IMAGE, "rb") as cif: image = cif.read() assert image == page
def test_get_pages_after(): car = ComicArchive(ARCHIVE_PATH) page_num = 33 pages = car.get_pages(page_num) for page in pages: fn = Path(PAGE_TMPL.format(page_num=page_num + 1)) with open(fn, "rb") as cif: image = cif.read() assert image == page page_num += 1
def get(self, request, *args, **kwargs): """Get the comic page from the archive.""" pk = self.kwargs.get("pk") comic = Comic.objects.only("path").get(pk=pk) try: car = ComicArchive(comic.path) page = self.kwargs.get("page") page_image = car.get_page_by_index(page) return HttpResponse(page_image, content_type="image/jpeg") except Exception as exc: LOG.exception(exc) raise NotFound(detail="comic page not found")
def read_metadata(archive_path, metadata): """Read metadata and compare to dict fixture.""" disk_car = ComicArchive(archive_path) md = ComicBaseMetadata(metadata=metadata) pprint(disk_car.metadata.metadata) pprint(md.metadata) assert disk_car.metadata == md
def write_metadata(tmp_path, new_test_cbz_path, metadata, md_type): """Create a test metadata file, read it back and compare the original.""" create_test_file(tmp_path, new_test_cbz_path, metadata, md_type) # read data back from the test file and then cleanup disk_car = ComicArchive(new_test_cbz_path) shutil.rmtree(tmp_path) # comparison metadata direct from example data md = ComicBaseMetadata(metadata=metadata) assert disk_car.metadata == md
def create_test_file(tmp_path, new_test_cbz_path, metadata, md_type): """Create a test file and write metadata to it.""" # Create an minimal file to write to extract_path = tmp_path / "extract" extract_path.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(SOURCE_ARCHIVE_PATH) as zf: zf.extractall(extract_path) with zipfile.ZipFile(new_test_cbz_path, mode="w") as zf: for root, _, filenames in os.walk(extract_path): root_path = Path(root) for fn in sorted(filenames): if fn.endswith(".xml"): continue full_fn = root_path / fn relative_path = full_fn.relative_to(extract_path) zf.write(full_fn, arcname=relative_path) shutil.rmtree(extract_path) # Create an archive object with the fixture data car = ComicArchive(new_test_cbz_path, metadata) # write the metadata to the empty archive car.write_metadata(md_type)
def run_on_file(self, path): """Run operations on one file.""" if path.is_dir() and self.config.recurse: self.recurse(path) if not path.is_file(): print(f"{path} is not a file.") return car = ComicArchive(path, config=self.config) if self.config.raw: car.print_raw() if self.config.metadata: pprint(car.get_metadata()) if self.config.covers: car.extract_cover_as(self.config.dest_path) if self.config.index_from: car.extract_pages(self.config.index_from, self.config.dest_path) if self.config.export: car.export_files() if self.config.cbz or self.config.delete_tags: car.recompress() if self.config.import_fn: car.import_file(self.config.import_fn) if self.config.rename: car.rename_file()
def test_get_covers(): car = ComicArchive(ARCHIVE_PATH, config=CONFIG) with open(COVER_IMAGE, "rb") as cif: image = cif.read() assert image == car.cover_image_data
class Importer: """import ComicBox metadata into Codex database.""" BROWSER_GROUP_TREE = (Publisher, Imprint, Series, Volume) BROWSER_GROUP_TREE_COUNT_FIELDS = {Series: "volume_count", Volume: "issue_count"} SPECIAL_FKS = ("myself", "volume", "series", "imprint", "publisher", "library") EXCLUDED_MODEL_KEYS = ("id", "parent_folder") FIELD_PREFIX_LEN = len("codex.Comic.") def __init__(self, library_id, path): """Set the state for this import.""" self.library_id = library_id self.path = path self.car = ComicArchive(path) self.md = self.car.get_metadata() @staticmethod def get_or_create_simple_model(name, cls): """Update or create a SimpleModel that just has a name.""" if not name: return # find deleted models and undelete them too defaults = {} defaults["name"] = name inst, created = cls.objects.get_or_create(defaults=defaults, name__iexact=name) if created: LOG.info(f"Created {cls.__name__} {inst.name}") return inst def get_foreign_keys(self): """Set all the foreign keys in the comic model.""" foreign_keys = {} for field in Comic._meta.get_fields(): if not isinstance(field, ForeignKey): continue if field.name in self.SPECIAL_FKS: continue name = self.md.get(field.name) inst = self.get_or_create_simple_model(name, field.related_model) foreign_keys[field.name] = inst return foreign_keys def get_many_to_many_instances(self): """Get or Create all the ManyToMany fields.""" m2m_fields = {} # for key, cls, in M2M_KEYS.items(): for field in Comic._meta.get_fields(): if not isinstance(field, ManyToManyField): continue if field.name == "credits": continue names = self.md.get(field.name) if names: instances = set() for name in names: inst = self.get_or_create_simple_model(name, field.related_model) if inst: instances.add(inst) if instances: m2m_fields[field.name] = instances del self.md[field.name] return m2m_fields def get_credits(self): """Get or Create credits, a complicated ManyToMany Field.""" credits_md = self.md.get("credits") if not credits_md: return credits = [] for credit_md in credits_md: if not credit_md: continue # Default to people credited without roles being ok. search = {"role": None} for field in Credit._meta.get_fields(): if not isinstance(field, ForeignKey): continue name = credit_md.get(field.name) if name: inst = self.get_or_create_simple_model(name, field.related_model) if inst: search[field.name] = inst if len(search) != 2: LOG.warn(f"Invalid credit not creating: {credits_md}") continue defaults = {} defaults.update(search) credit, created = Credit.objects.get_or_create(defaults=defaults, **search) credits.append(credit) if created: if credit.role: credit_name = credit.role.name else: credit_name = None LOG.info(f"Created credit {credit_name}: {credit.person.name}") del self.md["credits"] return credits def get_or_create_browser_group_tree(self, cls, tree): """Get or create a single level of the comic's publish tree.""" md_key = cls.__name__.lower() # current level name = self.md.get(md_key) # name of the current level defaults = {} if name: for key, val in self.BROWSER_GROUP_TREE_COUNT_FIELDS.items(): # set special total count fields for the level we're at if isinstance(cls, key): defaults[val] = self.md.get(val) break search = {"is_default": name is not None} # Set all the parents from the tree we've alredy created for index, inst in enumerate(tree): field = self.BROWSER_GROUP_TREE[index].__name__.lower() search[field] = inst defaults.update(search) if name: defaults["name"] = name search["name__iexact"] = name else: search["name__iexact"] = cls._meta.get_field("name").get_default() inst, created = cls.objects.get_or_create(defaults=defaults, **search) log_str = f"{cls.__name__} {inst.name}" if created: LOG.info(f"Created {log_str}") else: LOG.debug(f"Updated {log_str}") return inst def get_browser_group_tree(self): """Create the browse tree from publisher down to volume.""" tree = [] for cls in self.BROWSER_GROUP_TREE: inst = self.get_or_create_browser_group_tree(cls, tree) tree.append(inst) return tree @staticmethod def get_folders(library, child_path): """Create db folder tree from the library on down.""" library_p = Path(library.path) relative_path = Path(child_path).relative_to(library_p) parents = relative_path.parents parent_folder = None folder = None folders = [] for parent in reversed(parents): if str(parent) == ".": continue path = library_p / parent defaults = {"name": path.name, "parent_folder": parent_folder} search_kwargs = { "library": library, "path": str(path), } defaults.update(search_kwargs) folder, created = Folder.objects.get_or_create( defaults=defaults, **search_kwargs ) if created: LOG.info(f"Created {path} db folder.") parent_folder = folder folders.append(folder) return folders def clean_metadata(self): """Create whitelisted metdata object with only Comic model fields.""" fields = Comic._meta.fields clean_md = {} for field in fields: key = str(field)[self.FIELD_PREFIX_LEN :] if key not in self.EXCLUDED_MODEL_KEYS and key in self.md.keys(): clean_md[key] = self.md[key] self.md = clean_md def set_locales(self): """Explode locale info from alpha2 into long names.""" # Could do i8n here later lang = self.md.get("language") if lang: pc_lang = pycountry.languages.lookup(lang) if pc_lang: self.md["language"] = pc_lang.name country = self.md.get("country") if country: py_country = pycountry.countries.lookup(country) if py_country: self.md["country"] = py_country.name def import_metadata(self): """Import a comic into the db, from a path, using Comicbox.""" # Buld the catalogue tree BEFORE we clean the metadata created = False ( self.md["publisher"], self.md["imprint"], self.md["series"], self.md["volume"], ) = self.get_browser_group_tree() library = Library.objects.only("pk", "path").get(pk=self.library_id) self.set_locales() credits = self.get_credits() m2m_fields = self.get_many_to_many_instances() self.clean_metadata() # the order of this is pretty important foreign_keys = self.get_foreign_keys() self.md.update(foreign_keys) folders = self.get_folders(library, self.path) if folders: self.md["parent_folder"] = folders[-1] self.md["library"] = library self.md["path"] = self.path self.md["size"] = Path(self.path).stat().st_size self.md["max_page"] = max(self.md["page_count"] - 1, 0) cover_path = get_cover_path(self.path) self.md["cover_path"] = cover_path if credits: m2m_fields["credits"] = credits m2m_fields["folder"] = folders comic, created = Comic.objects.update_or_create( defaults=self.md, path=self.path ) comic.myself = comic # Add the m2m2 instances afterwards, can't initialize with them for attr, instance_list in m2m_fields.items(): getattr(comic, attr).set(instance_list) comic.save() QUEUE.put(ComicCoverCreateTask(self.path, cover_path, True)) # If it works, clear the failed import FailedImport.objects.filter(path=self.path).delete() if created: verb = "Created" else: verb = "Updated" LOG.info(f"{verb} comic {str(comic.volume.series.name)} #{comic.issue:03}") QUEUE.put(LibraryChangedTask()) return created
def __init__(self, library_id, path): """Set the state for this import.""" self.library_id = library_id self.path = path self.car = ComicArchive(path) self.md = self.car.get_metadata()