def test_get_random_page():
car = ComicArchive(ARCHIVE_PATH)
page = car.get_page_by_index(4)
with open(PAGE_FIVE, "rb") as cif:
image = cif.read()
assert image == page
def create_comic_cover(comic_path, db_cover_path, force=False):
"""Create a comic cover thumnail and save it to disk."""
try:
if db_cover_path == MISSING_COVER_FN and not force:
LOG.debug(f"Cover for {comic_path} missing.")
return
fs_cover_path = COVER_ROOT / db_cover_path
if fs_cover_path.exists() and not force:
LOG.debug(f"Cover already exists {comic_path} {db_cover_path}")
return
fs_cover_path.parent.mkdir(exist_ok=True, parents=True)
if comic_path is None:
comic = Comic.objects.only("path").get(cover_path=db_cover_path)
comic_path = comic.path
# Reopens the car, so slightly inefficient.
car = ComicArchive(comic_path)
im = Image.open(BytesIO(car.get_cover_image()))
im.thumbnail(THUMBNAIL_SIZE)
im.save(fs_cover_path, im.format)
LOG.info(f"Created cover thumbnail for: {comic_path}")
QUEUE.put(LibraryChangedTask())
except Exception as exc:
LOG.error(f"Failed to create cover thumb for {comic_path}")
LOG.exception(exc)
Comic.objects.filter(comic_path=comic_path).update(
cover_path=MISSING_COVER_FN)
LOG.warn(f"Marked cover for {comic_path} missing.")
def test_get_covers():
car = ComicArchive(ARCHIVE_PATH)
page = car.get_cover_image()
with open(COVER_IMAGE, "rb") as cif:
image = cif.read()
assert image == page
def test_get_pages_after():
car = ComicArchive(ARCHIVE_PATH)
page_num = 33
pages = car.get_pages(page_num)
for page in pages:
fn = Path(PAGE_TMPL.format(page_num=page_num + 1))
with open(fn, "rb") as cif:
image = cif.read()
assert image == page
page_num += 1
def get(self, request, *args, **kwargs):
"""Get the comic page from the archive."""
pk = self.kwargs.get("pk")
comic = Comic.objects.only("path").get(pk=pk)
try:
car = ComicArchive(comic.path)
page = self.kwargs.get("page")
page_image = car.get_page_by_index(page)
return HttpResponse(page_image, content_type="image/jpeg")
except Exception as exc:
LOG.exception(exc)
raise NotFound(detail="comic page not found")
def read_metadata(archive_path, metadata):
"""Read metadata and compare to dict fixture."""
disk_car = ComicArchive(archive_path)
md = ComicBaseMetadata(metadata=metadata)
pprint(disk_car.metadata.metadata)
pprint(md.metadata)
assert disk_car.metadata == md
def write_metadata(tmp_path, new_test_cbz_path, metadata, md_type):
"""Create a test metadata file, read it back and compare the original."""
create_test_file(tmp_path, new_test_cbz_path, metadata, md_type)
# read data back from the test file and then cleanup
disk_car = ComicArchive(new_test_cbz_path)
shutil.rmtree(tmp_path)
# comparison metadata direct from example data
md = ComicBaseMetadata(metadata=metadata)
assert disk_car.metadata == md
def create_test_file(tmp_path, new_test_cbz_path, metadata, md_type):
"""Create a test file and write metadata to it."""
# Create an minimal file to write to
extract_path = tmp_path / "extract"
extract_path.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(SOURCE_ARCHIVE_PATH) as zf:
zf.extractall(extract_path)
with zipfile.ZipFile(new_test_cbz_path, mode="w") as zf:
for root, _, filenames in os.walk(extract_path):
root_path = Path(root)
for fn in sorted(filenames):
if fn.endswith(".xml"):
continue
full_fn = root_path / fn
relative_path = full_fn.relative_to(extract_path)
zf.write(full_fn, arcname=relative_path)
shutil.rmtree(extract_path)
# Create an archive object with the fixture data
car = ComicArchive(new_test_cbz_path, metadata)
# write the metadata to the empty archive
car.write_metadata(md_type)
def run_on_file(self, path):
"""Run operations on one file."""
if path.is_dir() and self.config.recurse:
self.recurse(path)
if not path.is_file():
print(f"{path} is not a file.")
return
car = ComicArchive(path, config=self.config)
if self.config.raw:
car.print_raw()
if self.config.metadata:
pprint(car.get_metadata())
if self.config.covers:
car.extract_cover_as(self.config.dest_path)
if self.config.index_from:
car.extract_pages(self.config.index_from, self.config.dest_path)
if self.config.export:
car.export_files()
if self.config.cbz or self.config.delete_tags:
car.recompress()
if self.config.import_fn:
car.import_file(self.config.import_fn)
if self.config.rename:
car.rename_file()
def test_get_covers():
car = ComicArchive(ARCHIVE_PATH, config=CONFIG)
with open(COVER_IMAGE, "rb") as cif:
image = cif.read()
assert image == car.cover_image_data
class Importer:
"""import ComicBox metadata into Codex database."""
BROWSER_GROUP_TREE = (Publisher, Imprint, Series, Volume)
BROWSER_GROUP_TREE_COUNT_FIELDS = {Series: "volume_count", Volume: "issue_count"}
SPECIAL_FKS = ("myself", "volume", "series", "imprint", "publisher", "library")
EXCLUDED_MODEL_KEYS = ("id", "parent_folder")
FIELD_PREFIX_LEN = len("codex.Comic.")
def __init__(self, library_id, path):
"""Set the state for this import."""
self.library_id = library_id
self.path = path
self.car = ComicArchive(path)
self.md = self.car.get_metadata()
@staticmethod
def get_or_create_simple_model(name, cls):
"""Update or create a SimpleModel that just has a name."""
if not name:
return
# find deleted models and undelete them too
defaults = {}
defaults["name"] = name
inst, created = cls.objects.get_or_create(defaults=defaults, name__iexact=name)
if created:
LOG.info(f"Created {cls.__name__} {inst.name}")
return inst
def get_foreign_keys(self):
"""Set all the foreign keys in the comic model."""
foreign_keys = {}
for field in Comic._meta.get_fields():
if not isinstance(field, ForeignKey):
continue
if field.name in self.SPECIAL_FKS:
continue
name = self.md.get(field.name)
inst = self.get_or_create_simple_model(name, field.related_model)
foreign_keys[field.name] = inst
return foreign_keys
def get_many_to_many_instances(self):
"""Get or Create all the ManyToMany fields."""
m2m_fields = {}
# for key, cls, in M2M_KEYS.items():
for field in Comic._meta.get_fields():
if not isinstance(field, ManyToManyField):
continue
if field.name == "credits":
continue
names = self.md.get(field.name)
if names:
instances = set()
for name in names:
inst = self.get_or_create_simple_model(name, field.related_model)
if inst:
instances.add(inst)
if instances:
m2m_fields[field.name] = instances
del self.md[field.name]
return m2m_fields
def get_credits(self):
"""Get or Create credits, a complicated ManyToMany Field."""
credits_md = self.md.get("credits")
if not credits_md:
return
credits = []
for credit_md in credits_md:
if not credit_md:
continue
# Default to people credited without roles being ok.
search = {"role": None}
for field in Credit._meta.get_fields():
if not isinstance(field, ForeignKey):
continue
name = credit_md.get(field.name)
if name:
inst = self.get_or_create_simple_model(name, field.related_model)
if inst:
search[field.name] = inst
if len(search) != 2:
LOG.warn(f"Invalid credit not creating: {credits_md}")
continue
defaults = {}
defaults.update(search)
credit, created = Credit.objects.get_or_create(defaults=defaults, **search)
credits.append(credit)
if created:
if credit.role:
credit_name = credit.role.name
else:
credit_name = None
LOG.info(f"Created credit {credit_name}: {credit.person.name}")
del self.md["credits"]
return credits
def get_or_create_browser_group_tree(self, cls, tree):
"""Get or create a single level of the comic's publish tree."""
md_key = cls.__name__.lower() # current level
name = self.md.get(md_key) # name of the current level
defaults = {}
if name:
for key, val in self.BROWSER_GROUP_TREE_COUNT_FIELDS.items():
# set special total count fields for the level we're at
if isinstance(cls, key):
defaults[val] = self.md.get(val)
break
search = {"is_default": name is not None}
# Set all the parents from the tree we've alredy created
for index, inst in enumerate(tree):
field = self.BROWSER_GROUP_TREE[index].__name__.lower()
search[field] = inst
defaults.update(search)
if name:
defaults["name"] = name
search["name__iexact"] = name
else:
search["name__iexact"] = cls._meta.get_field("name").get_default()
inst, created = cls.objects.get_or_create(defaults=defaults, **search)
log_str = f"{cls.__name__} {inst.name}"
if created:
LOG.info(f"Created {log_str}")
else:
LOG.debug(f"Updated {log_str}")
return inst
def get_browser_group_tree(self):
"""Create the browse tree from publisher down to volume."""
tree = []
for cls in self.BROWSER_GROUP_TREE:
inst = self.get_or_create_browser_group_tree(cls, tree)
tree.append(inst)
return tree
@staticmethod
def get_folders(library, child_path):
"""Create db folder tree from the library on down."""
library_p = Path(library.path)
relative_path = Path(child_path).relative_to(library_p)
parents = relative_path.parents
parent_folder = None
folder = None
folders = []
for parent in reversed(parents):
if str(parent) == ".":
continue
path = library_p / parent
defaults = {"name": path.name, "parent_folder": parent_folder}
search_kwargs = {
"library": library,
"path": str(path),
}
defaults.update(search_kwargs)
folder, created = Folder.objects.get_or_create(
defaults=defaults, **search_kwargs
)
if created:
LOG.info(f"Created {path} db folder.")
parent_folder = folder
folders.append(folder)
return folders
def clean_metadata(self):
"""Create whitelisted metdata object with only Comic model fields."""
fields = Comic._meta.fields
clean_md = {}
for field in fields:
key = str(field)[self.FIELD_PREFIX_LEN :]
if key not in self.EXCLUDED_MODEL_KEYS and key in self.md.keys():
clean_md[key] = self.md[key]
self.md = clean_md
def set_locales(self):
"""Explode locale info from alpha2 into long names."""
# Could do i8n here later
lang = self.md.get("language")
if lang:
pc_lang = pycountry.languages.lookup(lang)
if pc_lang:
self.md["language"] = pc_lang.name
country = self.md.get("country")
if country:
py_country = pycountry.countries.lookup(country)
if py_country:
self.md["country"] = py_country.name
def import_metadata(self):
"""Import a comic into the db, from a path, using Comicbox."""
# Buld the catalogue tree BEFORE we clean the metadata
created = False
(
self.md["publisher"],
self.md["imprint"],
self.md["series"],
self.md["volume"],
) = self.get_browser_group_tree()
library = Library.objects.only("pk", "path").get(pk=self.library_id)
self.set_locales()
credits = self.get_credits()
m2m_fields = self.get_many_to_many_instances()
self.clean_metadata() # the order of this is pretty important
foreign_keys = self.get_foreign_keys()
self.md.update(foreign_keys)
folders = self.get_folders(library, self.path)
if folders:
self.md["parent_folder"] = folders[-1]
self.md["library"] = library
self.md["path"] = self.path
self.md["size"] = Path(self.path).stat().st_size
self.md["max_page"] = max(self.md["page_count"] - 1, 0)
cover_path = get_cover_path(self.path)
self.md["cover_path"] = cover_path
if credits:
m2m_fields["credits"] = credits
m2m_fields["folder"] = folders
comic, created = Comic.objects.update_or_create(
defaults=self.md, path=self.path
)
comic.myself = comic
# Add the m2m2 instances afterwards, can't initialize with them
for attr, instance_list in m2m_fields.items():
getattr(comic, attr).set(instance_list)
comic.save()
QUEUE.put(ComicCoverCreateTask(self.path, cover_path, True))
# If it works, clear the failed import
FailedImport.objects.filter(path=self.path).delete()
if created:
verb = "Created"
else:
verb = "Updated"
LOG.info(f"{verb} comic {str(comic.volume.series.name)} #{comic.issue:03}")
QUEUE.put(LibraryChangedTask())
return created
def __init__(self, library_id, path):
"""Set the state for this import."""
self.library_id = library_id
self.path = path
self.car = ComicArchive(path)
self.md = self.car.get_metadata()