def save_author_file(author, static_folder, books, project_id, force=False): fpath = os.path.join(static_folder, "{}.html".format(author.fname())) if path(fpath).exists() and not force: logger.debug("\t\tSkipping author file {}".format(fpath)) return logger.debug("\t\tSaving author file {}".format(fpath)) save_file(author_html_content_for(author, books, project_id), fpath, UTF8)
def save_author_file(author, static_folder, books, project_id, force=False): fpath = os.path.join(static_folder, "{}.html".format(author.fname())) if path(fpath).exists() and not force: logger.debug("\t\tSkipping author file {}".format(fpath)) return logger.debug("\t\tSaving author file {}".format(fpath)) save_file( author_html_content_for(author, static_folder, books, project_id), fpath, UTF8)
def exec_cmd(cmd): if isinstance(cmd, (tuple, list)): args = cmd else: args = cmd.split(' ') logger.debug(" ".join(args)) if six.PY3: return subprocess.run(args).returncode else: return subprocess.call(args)
def exec_cmd(cmd): if isinstance(cmd, (tuple, list)): args = cmd else: args = cmd.split(" ") logger.debug(" ".join(args)) if six.PY3: return subprocess.run(args).returncode else: return subprocess.call(args)
def setup_database(wipe=False): logger.info("Setting up the database") for model in (License, Format, Author, Book, BookFormat, Url): if wipe: model.drop_table(fail_silently=True) if not model.table_exists(): model.create_table() logger.debug("Created table for {}".format(model._meta.name)) load_fixtures(model) else: logger.debug("{} table already exists.".format(model._meta.name))
def setup_database(wipe=False): logger.info("Setting up the database") for model in (License, Format, Author, Book, BookFormat, Url): if wipe: model.drop_table(fail_silently=True) if not model.table_exists(): model.create_table() logger.debug("Created table for {}".format(model._meta.name)) load_fixtures(model) else: logger.debug("{} table already exists.".format(model._meta.name))
def download_covers(book, download_cache): cover = "{}_cover.jpg".format(book.id) fpath = os.path.join(download_cache, cover) has_cover = Book.select(Book.cover_page).where(Book.id == book.id) if has_cover: title = "{}{}/pg{}.cover.medium.jpg".format(IMAGE_BASE, book.id, book.id) logger.debug("Downloading {}".format(title)) download_file(title, fpath) else: logger.debug("No Book Cover found for Book #{}".format(book.id)) return True
def handle_companion_file(fname, dstfname=None, book=None, force=False, as_ext=None): ext = path(fname).ext if as_ext is None else as_ext src = os.path.join(path(download_cache).abspath(), fname) if dstfname is None: dstfname = fname dst = os.path.join(path(static_folder).abspath(), dstfname) if path(dst).exists() and not force: logger.debug("\t\tSkipping existing companion {}".format(dstfname)) return # optimization based on mime/extension if ext in (".png", ".jpg", ".jpeg", ".gif"): logger.info( "\t\tCopying and optimizing image companion {}".format(fname)) # copy_from_cache(src, dst) optimize_image(src, dst) elif ext == ".epub": logger.info("\t\tCreating optimized EPUB file {}".format(fname)) tmp_epub = tempfile.NamedTemporaryFile(suffix=".epub", dir=TMP_FOLDER) tmp_epub.close() try: optimize_epub(src, tmp_epub.name) except zipfile.BadZipFile: logger.warn("\t\tBad zip file. " "Copying as it might be working{}".format(fname)) handle_companion_file(fname, dstfname, book, force, as_ext="zip") else: path(tmp_epub.name).move(dst) else: # excludes files created by Windows Explorer if src.endswith("_Thumbs.db"): return # copy otherwise (PDF mostly) logger.debug("\t\tshitty ext: {}".format(dst)) logger.info("\t\tCopying companion file to {}".format(fname)) copy_from_cache(src, dst)
def download_cover(book, book_dir, s3_storage, optimizer_version): has_cover = Book.select(Book.cover_page).where(Book.id == book.id) if has_cover: # try to download optimized cover from cache if s3_storage url = "{}{}/pg{}.cover.medium.jpg".format(IMAGE_BASE, book.id, book.id) etag = get_etag_from_url(url) downloaded_from_cache = False cover = "{}_cover_image.jpg".format(book.id) if (book_dir.joinpath("optimized").joinpath(cover).exists() or book_dir.joinpath("unoptimized").joinpath(cover).exists()): logger.debug(f"Cover already exists for book #{book.id}") return if s3_storage: logger.info( f"Trying to download cover for {book.id} from optimization cache" ) downloaded_from_cache = download_from_cache( book=book, etag=etag, book_format="cover", dest_dir=book_dir.joinpath("optimized"), s3_storage=s3_storage, optimizer_version=optimizer_version, ) if not downloaded_from_cache: logger.debug("Downloading {}".format(url)) if download_file(url, book_dir.joinpath("unoptimized").joinpath(cover)): book.cover_etag = etag book.save() else: logger.debug("No Book Cover found for Book #{}".format(book.id))
def handle_companion_file(fname, dstfname=None, book=None, force=False, as_ext=None): ext = path(fname).ext if as_ext is None else as_ext src = os.path.join(path(download_cache).abspath(), fname) if dstfname is None: dstfname = fname dst = os.path.join(path(static_folder).abspath(), dstfname) if path(dst).exists() and not force: logger.debug("\t\tSkipping existing companion {}".format(dstfname)) return # optimization based on mime/extension if ext in ('.png', '.jpg', '.jpeg', '.gif'): logger.info("\t\tCopying and optimizing image companion {}" .format(fname)) # copy_from_cache(src, dst) optimize_image(src, dst) elif ext == '.epub': logger.info("\t\tCreating optimized EPUB file {}".format(fname)) tmp_epub = tempfile.NamedTemporaryFile(suffix='.epub', dir=TMP_FOLDER) tmp_epub.close() try: optimize_epub(src, tmp_epub.name) except zipfile.BadZipFile: logger.warn("\t\tBad zip file. " "Copying as it might be working{}".format(fname)) handle_companion_file(fname, dstfname, book, force, as_ext='zip') else: path(tmp_epub.name).move(dst) else: # excludes files created by Windows Explorer if src.endswith('_Thumbs.db'): return # copy otherwise (PDF mostly) logger.debug("\t\tshitty ext: {}".format(dst)) logger.info("\t\tCopying companion file to {}".format(fname)) copy_from_cache(src, dst)
def load_fixtures(model): logger.info("Loading fixtures for {}".format(model._meta.name)) for fixture in getattr(model._meta, "fixtures", []): f = model.create(**fixture) logger.debug("[fixtures] Created {}".format(f))
def load_fixtures(model): logger.info("Loading fixtures for {}".format(model._meta.name)) for fixture in getattr(model._meta, 'fixtures', []): f = model.create(**fixture) logger.debug("[fixtures] Created {}".format(f))
def download_book(book, download_cache, languages, formats, force): logger.info( "\tDownloading content files for Book #{id}".format(id=book.id)) # apply filters if not formats: formats = FORMAT_MATRIX.keys() # HTML is our base for ZIM for add it if not present if "html" not in formats: formats.append("html") for format in formats: fpath = os.path.join(download_cache, fname_for(book, format)) # check if already downloaded if path(fpath).exists() and not force: logger.debug("\t\t{fmt} already exists at {path}".format( fmt=format, path=fpath)) continue # retrieve corresponding BookFormat bfs = BookFormat.filter(book=book) if format == "html": patterns = [ "mnsrb10h.htm", "8ledo10h.htm", "tycho10f.htm", "8ledo10h.zip", "salme10h.htm", "8nszr10h.htm", "{id}-h.html", "{id}.html.gen", "{id}-h.htm", "8regr10h.zip", "{id}.html.noimages", "8lgme10h.htm", "tycho10h.htm", "tycho10h.zip", "8lgme10h.zip", "8indn10h.zip", "8resp10h.zip", "20004-h.htm", "8indn10h.htm", "8memo10h.zip", "fondu10h.zip", "{id}-h.zip", "8mort10h.zip", ] bfso = bfs bfs = bfs.join(Format).filter(Format.pattern << patterns) if not bfs.count(): pp( list([(b.format.mime, b.format.images, b.format.pattern) for b in bfs])) pp( list([(b.format.mime, b.format.images, b.format.pattern) for b in bfso])) logger.error("html not found") continue else: bfs = bfs.filter(BookFormat.format << Format.filter( mime=FORMAT_MATRIX.get(format))) if not bfs.count(): logger.debug("[{}] not avail. for #{}# {}".format( format, book.id, book.title).encode("utf-8")) continue if bfs.count() > 1: try: bf = bfs.join(Format).filter(Format.images).get() except Exception: bf = bfs.get() else: bf = bfs.get() logger.debug("[{}] Requesting URLs for #{}# {}".format( format, book.id, book.title).encode("utf-8")) # retrieve list of URLs for format unless we have it in DB if bf.downloaded_from and not force: urls = [bf.downloaded_from] else: urld = get_urls(book) urls = list(reversed(urld.get(FORMAT_MATRIX.get(format)))) import copy allurls = copy.copy(urls) while urls: url = urls.pop() if len(allurls) != 1: if not resource_exists(url): continue # HTML files are *sometime* available as ZIP files if url.endswith(".zip"): zpath = "{}.zip".format(fpath) if not download_file(url, zpath): logger.error("ZIP file donwload failed: {}".format(zpath)) continue # extract zipfile handle_zipped_epub(zippath=zpath, book=book, download_cache=download_cache) else: if not download_file(url, fpath): logger.error("file donwload failed: {}".format(fpath)) continue # store working URL in DB bf.downloaded_from = url bf.save() if not bf.downloaded_from: logger.error("NO FILE FOR #{}/{}".format(book.id, format)) pp(allurls) continue
def export_all_books(static_folder, download_cache, concurrency, languages=[], formats=[], only_books=[], force=False): project_id = get_project_id(languages=languages, formats=formats, only_books=only_books) # ensure dir exist path(static_folder).mkdir_p() books = get_list_of_filtered_books(languages=languages, formats=formats, only_books=only_books) if not len(get_langs_with_count(books=books)): critical_error("Unable to proceed. Combination of lamguages, " "books and formats has no result.") # sz = len(list(books)) # logger.debug("\tFiltered book collection size: {}".format(sz)) def nb_by_fmt(fmt): return sum([1 for book in books if BookFormat.select(BookFormat, Book, Format) .join(Book).switch(BookFormat) .join(Format) .where(Book.id == book.id) .where(Format.mime == FORMAT_MATRIX.get(fmt)) .count()]) logger.debug("\tFiltered book collection, PDF: {}" .format(nb_by_fmt('pdf'))) logger.debug("\tFiltered book collection, ePUB: {}" .format(nb_by_fmt('epub'))) logger.debug("\tFiltered book collection, HTML: {}" .format(nb_by_fmt('html'))) # export to JSON helpers export_to_json_helpers(books=books, static_folder=static_folder, languages=languages, formats=formats, project_id=project_id) # export HTML index and other static files export_skeleton(static_folder=static_folder, dev_mode=False, languages=languages, formats=formats, only_books=only_books) # Compute popularity popbooks = books.order_by(Book.downloads.desc()) popbooks_count = popbooks.count() stars_limits = [0] * NB_POPULARITY_STARS stars = NB_POPULARITY_STARS nb_downloads = popbooks[0].downloads for ibook in range(0, popbooks.count(), 1): if ibook > float(NB_POPULARITY_STARS-stars+1)/NB_POPULARITY_STARS*popbooks_count \ and popbooks[ibook].downloads < nb_downloads: stars_limits[stars-1] = nb_downloads stars = stars - 1 nb_downloads = popbooks[ibook].downloads # export to HTML cached_files = os.listdir(download_cache) for book in books: book.popularity = sum( [int(book.downloads >= stars_limits[i]) for i in range(NB_POPULARITY_STARS)]) dlb = lambda b: export_book_to(b, static_folder=static_folder, download_cache=download_cache, cached_files=cached_files, languages=languages, formats=formats, books=books, project_id=project_id, force=force) Pool(concurrency).map(dlb, books)
def handle_unoptimized_files( book, static_folder, src_dir, languages, formats, books, project_id, optimizer_version, force=False, title_search=False, add_bookshelves=False, s3_storage=None, ): def copy_file(src, dst): logger.info("\t\tCopying {}".format(dst)) try: shutil.copy2(src, dst) except IOError: logger.error("/!\\ Unable to copy missing file {}".format(src)) return def update_download_cache(unoptimized_file, optimized_file): book_dir = unoptimized_file.parents[1] optimized_dir = book_dir.joinpath("optimized") unoptimized_dir = book_dir.joinpath("unoptimized") if not optimized_dir.exists(): optimized_dir.mkdir() dst = optimized_dir.joinpath(optimized_file.name) os.unlink(unoptimized_file) copy_file(optimized_file.resolve(), dst.resolve()) if not [fpath for fpath in unoptimized_dir.iterdir()]: unoptimized_dir.rmdir() logger.info("\tExporting Book #{id}.".format(id=book.id)) # actual book content, as HTML html, _ = html_content_for(book=book, src_dir=src_dir) html_book_optimized_files = [] if html: article_fpath = static_folder.joinpath(article_name_for(book)) if not article_fpath.exists() or force: logger.info("\t\tExporting to {}".format(article_fpath)) try: new_html = update_html_for_static(book=book, html_content=html) except Exception: raise save_bs_output(new_html, article_fpath, UTF8) html_book_optimized_files.append(article_fpath) update_download_cache( src_dir.joinpath(fname_for(book, "html")), article_fpath ) if not src_dir.exists(): return else: logger.info("\t\tSkipping HTML article {}".format(article_fpath)) def optimize_image(src, dst, force=False): if dst.exists() and not force: logger.info("\tSkipping image optimization for {}".format(dst)) return dst logger.info("\tOptimizing image {}".format(dst)) if src.suffix == ".png": return optimize_png(str(src.resolve()), str(dst.resolve())) if src.suffix in (".jpg", ".jpeg"): return optimize_jpeg(str(src.resolve()), str(dst.resolve())) if src.suffix == ".gif": return optimize_gif(str(src.resolve()), str(dst.resolve())) return dst def optimize_gif(src, dst): exec_cmd(["gifsicle", "-O3", src, "-o", dst]) def optimize_png(src, dst): exec_cmd(["pngquant", "--nofs", "--force", "--output", dst, src]) exec_cmd(["advdef", "-z", "-4", "-i", "5", dst]) def optimize_jpeg(src, dst): if src != dst: copy_file(src, dst) exec_cmd(["jpegoptim", "--strip-all", "-m50", dst]) def optimize_epub(src, dst): logger.info("\t\tCreating ePUB off {} at {}".format(src, dst)) zipped_files = [] # create temp directory to extract to tmpd = tempfile.mkdtemp(dir=TMP_FOLDER) try: with zipfile.ZipFile(src, "r") as zf: zipped_files = zf.namelist() zf.extractall(tmpd) except zipfile.BadZipFile as exc: shutil.rmtree(tmpd) raise exc remove_cover = False for fname in zipped_files: fnp = os.path.join(tmpd, fname) if path(fname).ext in (".png", ".jpeg", ".jpg", ".gif"): # special case to remove ugly cover if fname.endswith("cover.jpg") and is_bad_cover(fnp): zipped_files.remove(fname) remove_cover = True else: optimize_image(pathlib.Path(fnp), pathlib.Path(fnp), force=True) if path(fname).ext in (".htm", ".html"): html_content, _ = read_file(fnp) html = update_html_for_static( book=book, html_content=html_content, epub=True ) save_bs_output(html, fnp, UTF8) if path(fname).ext == ".ncx": pattern = "*** START: FULL LICENSE ***" ncx, _ = read_file(fnp) soup = BeautifulSoup(ncx, "lxml-xml") for tag in soup.findAll("text"): if pattern in tag.text: s = tag.parent.parent s.decompose() for s in s.next_siblings: s.decompose() s.next_sibling save_bs_output(soup, fnp, UTF8) # delete {id}/cover.jpg if exist and update {id}/content.opf if remove_cover: # remove cover path(os.path.join(tmpd, text_type(book.id), "cover.jpg")).unlink_p() soup = None opff = os.path.join(tmpd, text_type(book.id), "content.opf") if os.path.exists(opff): opff_content, _ = read_file(opff) soup = BeautifulSoup(opff_content, "lxml-xml") for elem in soup.findAll(): if getattr(elem, "attrs", {}).get("href") == "cover.jpg": elem.decompose() save_bs_output(soup, opff, UTF8) # bundle epub as zip zip_epub(epub_fpath=dst, root_folder=tmpd, fpaths=zipped_files) path(tmpd).rmtree_p() def handle_companion_file( fname, dstfname=None, book=None, force=False, as_ext=None, html_file_list=None, s3_storage=None, ): ext = fname.suffix if as_ext is None else as_ext src = fname if dstfname is None: dstfname = fname.name dst = static_folder.joinpath(dstfname) if dst.exists() and not force: logger.debug("\t\tSkipping existing companion {}".format(dstfname)) return # optimization based on mime/extension if ext in (".png", ".jpg", ".jpeg", ".gif"): logger.info("\t\tCopying and optimizing image companion {}".format(fname)) optimize_image(src, dst) if dst.name == (f"{book.id}_cover_image.jpg") and s3_storage: upload_to_cache( asset=dst, book_format="cover", book_id=book.id, etag=book.cover_etag, s3_storage=s3_storage, optimizer_version=optimizer_version, ) update_download_cache(src, dst) elif html_file_list: html_file_list.append(dst) update_download_cache(src, dst) elif ext == ".epub": logger.info("\t\tCreating optimized EPUB file {}".format(fname)) tmp_epub = tempfile.NamedTemporaryFile(suffix=".epub", dir=TMP_FOLDER) tmp_epub.close() try: optimize_epub(src, tmp_epub.name) except zipfile.BadZipFile: logger.warn( "\t\tBad zip file. " "Copying as it might be working{}".format(fname) ) handle_companion_file(fname, dstfname, book, force, as_ext=".zip") else: path(tmp_epub.name).move(dst) if s3_storage: upload_to_cache( asset=dst, book_format="epub", book_id=book.id, etag=book.epub_etag, s3_storage=s3_storage, optimizer_version=optimizer_version, ) update_download_cache(src, dst) else: # excludes files created by Windows Explorer if src.name.endswith("_Thumbs.db"): return # copy otherwise (PDF mostly) logger.info("\t\tCopying companion file to {}".format(dst)) copy_file(src, dst) if ext != ".pdf" and ext != ".zip" and html_file_list: html_file_list.append(dst) update_download_cache(src, dst) # associated files (images, etc) for fpath in src_dir.iterdir(): if fpath.is_file() and fpath.name.startswith(f"{book.id}_"): if fpath.suffix in (".html", ".htm"): src = fpath dst = static_folder.joinpath(fpath.name) if dst.exists() and not force: logger.debug("\t\tSkipping existing HTML {}".format(dst)) continue logger.info("\t\tExporting HTML file to {}".format(dst)) html, _ = read_file(src) new_html = update_html_for_static(book=book, html_content=html) save_bs_output(new_html, dst, UTF8) html_book_optimized_files.append(dst) update_download_cache(src, dst) else: try: handle_companion_file( fpath, force=force, html_file_list=html_book_optimized_files, s3_storage=s3_storage, book=book, ) except Exception as e: logger.exception(e) logger.error( "\t\tException while handling companion file: {}".format(e) ) if s3_storage and html_book_optimized_files: upload_to_cache( asset=html_book_optimized_files, book_format="html", etag=book.html_etag, book_id=book.id, s3_storage=s3_storage, optimizer_version=optimizer_version, ) # other formats for format in formats: if format not in book.formats() or format == "html": continue book_file = src_dir.joinpath(fname_for(book, format)) if book_file.exists(): try: handle_companion_file( book_file, archive_name_for(book, format), force=force, book=book, s3_storage=s3_storage, ) except Exception as e: logger.exception(e) logger.error( "\t\tException while handling companion file: {}".format(e) )
def export_all_books( static_folder=None, download_cache=None, concurrency=None, languages=[], formats=[], only_books=[], force=False, title_search=False, add_bookshelves=False, s3_storage=None, optimizer_version=None, ): project_id = get_project_id( languages=languages, formats=formats, only_books=only_books ) # ensure dir exist path(static_folder).mkdir_p() books = get_list_of_filtered_books( languages=languages, formats=formats, only_books=only_books ) if not len(get_langs_with_count(books=books)): critical_error( "Unable to proceed. Combination of lamguages, " "books and formats has no result." ) # sz = len(list(books)) # logger.debug("\tFiltered book collection size: {}".format(sz)) def nb_by_fmt(fmt): return sum( [ 1 for book in books if BookFormat.select(BookFormat, Book, Format) .join(Book) .switch(BookFormat) .join(Format) .where(Book.id == book.id) .where(Format.mime == FORMAT_MATRIX.get(fmt)) .count() ] ) logger.debug("\tFiltered book collection, PDF: {}".format(nb_by_fmt("pdf"))) logger.debug("\tFiltered book collection, ePUB: {}".format(nb_by_fmt("epub"))) logger.debug("\tFiltered book collection, HTML: {}".format(nb_by_fmt("html"))) # export to JSON helpers export_to_json_helpers( books=books, static_folder=static_folder, languages=languages, formats=formats, project_id=project_id, title_search=title_search, add_bookshelves=add_bookshelves, ) # export HTML index and other static files export_skeleton( static_folder=static_folder, dev_mode=False, languages=languages, formats=formats, only_books=only_books, title_search=title_search, add_bookshelves=add_bookshelves, ) # Compute popularity popbooks = books.order_by(Book.downloads.desc()) popbooks_count = popbooks.count() stars_limits = [0] * NB_POPULARITY_STARS stars = NB_POPULARITY_STARS nb_downloads = popbooks[0].downloads for ibook in range(0, popbooks.count(), 1): if ( ibook > float(NB_POPULARITY_STARS - stars + 1) / NB_POPULARITY_STARS * popbooks_count and popbooks[ibook].downloads < nb_downloads ): stars_limits[stars - 1] = nb_downloads stars = stars - 1 nb_downloads = popbooks[ibook].downloads for book in books: book.popularity = sum( [int(book.downloads >= stars_limits[i]) for i in range(NB_POPULARITY_STARS)] ) def dlb(b): return export_book( b, static_folder=pathlib.Path(static_folder), book_dir=pathlib.Path(download_cache).joinpath(str(b.id)), languages=languages, formats=formats, books=books, project_id=project_id, force=force, title_search=title_search, add_bookshelves=add_bookshelves, s3_storage=s3_storage, optimizer_version=optimizer_version, ) Pool(concurrency).map(dlb, books)
def export_book_to(book, static_folder, download_cache, cached_files, languages, formats, books, project_id, force=False): logger.info("\tExporting Book #{id}.".format(id=book.id)) # actual book content, as HTML html, encoding = html_content_for(book=book, static_folder=static_folder, download_cache=download_cache) if html: article_fpath = os.path.join(static_folder, article_name_for(book)) if not path(article_fpath).exists() or force: logger.info("\t\tExporting to {}".format(article_fpath)) try: new_html = update_html_for_static(book=book, html_content=html) except Exception: raise new_html = html save_bs_output(new_html, article_fpath, UTF8) else: logger.info("\t\tSkipping HTML article {}".format(article_fpath)) def symlink_from_cache(fname, dstfname=None): src = os.path.join(path(download_cache).abspath(), fname) if dstfname is None: dstfname = fname dst = os.path.join(path(static_folder).abspath(), dstfname) logger.info("\t\tSymlinking {}".format(dst)) path(dst).unlink_p() try: path(src).link(dst) # hard link except IOError: logger.error("/!\\ Unable to symlink missing file {}".format(src)) return def copy_from_cache(fname, dstfname=None): src = os.path.join(path(download_cache).abspath(), fname) if dstfname is None: dstfname = fname dst = os.path.join(path(static_folder).abspath(), dstfname) logger.info("\t\tCopying {}".format(dst)) path(dst).unlink_p() try: path(src).copy(dst) except IOError: logger.error("/!\\ Unable to copy missing file {}".format(src)) return def optimize_image(src, dst, force=False): if path(dst).exists() and not force: logger.info("\tSkipping image optimization for {}".format(dst)) return dst logger.info("\tOptimizing image {}".format(dst)) if path(src).ext == '.png': return optimize_png(src, dst) if path(src).ext in ('.jpg', '.jpeg'): return optimize_jpeg(src, dst) if path(src).ext == '.gif': return optimize_gif(src, dst) return dst def optimize_gif(src, dst): exec_cmd(['gifsicle', '-O3', src, '-o', dst]) def optimize_png(src, dst): exec_cmd(['pngquant', '--nofs', '--force', '--output', dst, src]) exec_cmd(['advdef', '-z', '-4', '-i', '5', dst]) def optimize_jpeg(src, dst): copy_from_cache(src, dst) exec_cmd(['jpegoptim', '--strip-all', '-m50', dst]) def optimize_epub(src, dst): logger.info("\t\tCreating ePUB off {} at {}".format(src, dst)) zipped_files = [] # create temp directory to extract to tmpd = tempfile.mkdtemp(dir=TMP_FOLDER) with zipfile.ZipFile(src, 'r') as zf: zipped_files = zf.namelist() zf.extractall(tmpd) remove_cover = False for fname in zipped_files: fnp = os.path.join(tmpd, fname) if path(fname).ext in ('.png', '.jpeg', '.jpg', '.gif'): # special case to remove ugly cover if fname.endswith('cover.jpg') and is_bad_cover(fnp): zipped_files.remove(fname) remove_cover = True else: optimize_image(fnp, fnp) if path(fname).ext in ('.htm', '.html'): html_content, html_encoding = read_file(fnp) html = update_html_for_static(book=book, html_content=html_content, epub=True) save_bs_output(html, fnp, UTF8) if path(fname).ext == '.ncx': pattern = "*** START: FULL LICENSE ***" ncx, ncx_encoding = read_file(fnp) soup = BeautifulSoup(ncx, 'lxml-xml') for tag in soup.findAll('text'): if pattern in tag.text: s = tag.parent.parent s.decompose() for s in s.next_siblings: s.decompose() s.next_sibling save_bs_output(soup, fnp, UTF8) # delete {id}/cover.jpg if exist and update {id}/content.opf if remove_cover: # remove cover path( os.path.join(tmpd, text_type(book.id), 'cover.jpg')).unlink_p() soup = None opff = os.path.join(tmpd, text_type(book.id), 'content.opf') if os.path.exists(opff): opff_content, opff_encoding = read_file(opff) soup = BeautifulSoup(opff_content, 'lxml-xml') for elem in soup.findAll(): if getattr(elem, 'attrs', {}).get('href') == 'cover.jpg': elem.decompose() save_bs_output(soup, opff, UTF8) # bundle epub as zip zip_epub(epub_fpath=dst, root_folder=tmpd, fpaths=zipped_files) path(tmpd).rmtree_p() def handle_companion_file(fname, dstfname=None, book=None, force=False, as_ext=None): ext = path(fname).ext if as_ext is None else as_ext src = os.path.join(path(download_cache).abspath(), fname) if dstfname is None: dstfname = fname dst = os.path.join(path(static_folder).abspath(), dstfname) if path(dst).exists() and not force: logger.debug("\t\tSkipping existing companion {}".format(dstfname)) return # optimization based on mime/extension if ext in ('.png', '.jpg', '.jpeg', '.gif'): logger.info("\t\tCopying and optimizing image companion {}" .format(fname)) # copy_from_cache(src, dst) optimize_image(src, dst) elif ext == '.epub': logger.info("\t\tCreating optimized EPUB file {}".format(fname)) tmp_epub = tempfile.NamedTemporaryFile(suffix='.epub', dir=TMP_FOLDER) tmp_epub.close() try: optimize_epub(src, tmp_epub.name) except zipfile.BadZipFile: logger.warn("\t\tBad zip file. " "Copying as it might be working{}".format(fname)) handle_companion_file(fname, dstfname, book, force, as_ext='zip') else: path(tmp_epub.name).move(dst) else: # excludes files created by Windows Explorer if src.endswith('_Thumbs.db'): return # copy otherwise (PDF mostly) logger.debug("\t\tshitty ext: {}".format(dst)) logger.info("\t\tCopying companion file to {}".format(fname)) copy_from_cache(src, dst) # associated files (images, etc) for fname in [fn for fn in cached_files if fn.startswith("{}_".format(book.id))]: if path(fname).ext in ('.html', '.htm'): src = os.path.join(path(download_cache).abspath(), fname) dst = os.path.join(path(static_folder).abspath(), fname) if path(dst).exists() and not force: logger.debug("\t\tSkipping existing HTML {}".format(dst)) continue logger.info("\t\tExporting HTML file to {}".format(dst)) html, encoding = read_file(src) new_html = update_html_for_static(book=book, html_content=html) save_bs_output(new_html, dst, UTF8) else: try: handle_companion_file(fname, force=force) except Exception as e: logger.exception(e) logger.error("\t\tException while handling companion file: {}" .format(e)) # other formats for format in formats: if format not in book.formats() or format == 'html': continue try: handle_companion_file(fname_for(book, format), archive_name_for(book, format), force=force) except Exception as e: logger.exception(e) logger.error("\t\tException while handling companion file: {}" .format(e)) # book presentation article cover_fpath = os.path.join(static_folder, article_name_for(book=book, cover=True)) if not path(cover_fpath).exists() or force: logger.info("\t\tExporting to {}".format(cover_fpath)) html = cover_html_content_for(book=book, static_folder=static_folder, books=books, project_id=project_id) with open(cover_fpath, 'w') as f: if six.PY2: f.write(html.encode(UTF8)) else: f.write(html) else: logger.info("\t\tSkipping cover {}".format(cover_fpath))
def export_book_to(book, static_folder, download_cache, cached_files, languages, formats, books, project_id, force=False): logger.info("\tExporting Book #{id}.".format(id=book.id)) # actual book content, as HTML html, encoding = html_content_for(book=book, static_folder=static_folder, download_cache=download_cache) if html: article_fpath = os.path.join(static_folder, article_name_for(book)) if not path(article_fpath).exists() or force: logger.info("\t\tExporting to {}".format(article_fpath)) try: new_html = update_html_for_static(book=book, html_content=html) except: raise new_html = html save_bs_output(new_html, article_fpath, UTF8) else: logger.info("\t\tSkipping HTML article {}".format(article_fpath)) def symlink_from_cache(fname, dstfname=None): src = os.path.join(path(download_cache).abspath(), fname) if dstfname is None: dstfname = fname dst = os.path.join(path(static_folder).abspath(), dstfname) logger.info("\t\tSymlinking {}".format(dst)) path(dst).unlink_p() try: path(src).link(dst) # hard link except IOError: logger.error("/!\ Unable to symlink missing file {}".format(src)) return def copy_from_cache(fname, dstfname=None): src = os.path.join(path(download_cache).abspath(), fname) if dstfname is None: dstfname = fname dst = os.path.join(path(static_folder).abspath(), dstfname) logger.info("\t\tCopying {}".format(dst)) path(dst).unlink_p() try: path(src).copy(dst) except IOError: logger.error("/!\ Unable to copy missing file {}".format(src)) return def optimize_image(src, dst, force=False): if path(dst).exists() and not force: logger.info("\tSkipping image optimization for {}".format(dst)) return dst logger.info("\tOptimizing image {}".format(dst)) if path(src).ext == '.png': return optimize_png(src, dst) if path(src).ext in ('.jpg', '.jpeg'): return optimize_jpeg(src, dst) if path(src).ext == '.gif': return optimize_gif(src, dst) return dst def optimize_gif(src, dst): exec_cmd(['gifsicle', '-O3', src, '-o', dst]) def optimize_png(src, dst): exec_cmd(['pngquant', '--nofs', '--force', '--output', dst, src]) exec_cmd(['advdef', '-z', '-4', '-i', '5', dst]) def optimize_jpeg(src, dst): copy_from_cache(src, dst) exec_cmd(['jpegoptim', '--strip-all', '-m50', dst]) def optimize_epub(src, dst): logger.info("\t\tCreating ePUB off {} at {}".format(src, dst)) zipped_files = [] # create temp directory to extract to tmpd = tempfile.mkdtemp(dir=TMP_FOLDER) with zipfile.ZipFile(src, 'r') as zf: zipped_files = zf.namelist() zf.extractall(tmpd) remove_cover = False for fname in zipped_files: fnp = os.path.join(tmpd, fname) if path(fname).ext in ('.png', '.jpeg', '.jpg', '.gif'): # special case to remove ugly cover if fname.endswith('cover.jpg') and is_bad_cover(fnp): zipped_files.remove(fname) remove_cover = True else: optimize_image(fnp, fnp) if path(fname).ext in ('.htm', '.html'): html_content, html_encoding = read_file(fnp) html = update_html_for_static(book=book, html_content=html_content, epub=True) save_bs_output(html, fnp, UTF8) if path(fname).ext == '.ncx': pattern = "*** START: FULL LICENSE ***" ncx, ncx_encoding = read_file(fnp) soup = BeautifulSoup(ncx, 'lxml-xml') for tag in soup.findAll('text'): if pattern in tag.text: s = tag.parent.parent s.decompose() for s in s.next_siblings: s.decompose() s.next_sibling save_bs_output(soup, fnp, UTF8) # delete {id}/cover.jpg if exist and update {id}/content.opf if remove_cover: # remove cover path( os.path.join(tmpd, text_type(book.id), 'cover.jpg')).unlink_p() soup = None opff = os.path.join(tmpd, text_type(book.id), 'content.opf') if os.path.exists(opff): opff_content, opff_encoding = read_file(opff) soup = BeautifulSoup(opff_content, 'lxml-xml') for elem in soup.findAll(): if getattr(elem, 'attrs', {}).get('href') == 'cover.jpg': elem.decompose() save_bs_output(soup, opff, UTF8) # bundle epub as zip zip_epub(epub_fpath=dst, root_folder=tmpd, fpaths=zipped_files) path(tmpd).rmtree_p() def handle_companion_file(fname, dstfname=None, book=None, force=False, as_ext=None): ext = path(fname).ext if as_ext is None else as_ext src = os.path.join(path(download_cache).abspath(), fname) if dstfname is None: dstfname = fname dst = os.path.join(path(static_folder).abspath(), dstfname) if path(dst).exists() and not force: logger.debug("\t\tSkipping existing companion {}".format(dstfname)) return # optimization based on mime/extension if ext in ('.png', '.jpg', '.jpeg', '.gif'): logger.info("\t\tCopying and optimizing image companion {}" .format(fname)) # copy_from_cache(src, dst) optimize_image(src, dst) elif ext == '.epub': logger.info("\t\tCreating optimized EPUB file {}".format(fname)) tmp_epub = tempfile.NamedTemporaryFile(suffix='.epub', dir=TMP_FOLDER) tmp_epub.close() try: optimize_epub(src, tmp_epub.name) except zipfile.BadZipFile: logger.warn("\t\tBad zip file. " "Copying as it might be working{}".format(fname)) handle_companion_file(fname, dstfname, book, force, as_ext='zip') else: path(tmp_epub.name).move(dst) else: # excludes files created by Windows Explorer if src.endswith('_Thumbs.db'): return # copy otherwise (PDF mostly) logger.debug("\t\tshitty ext: {}".format(dst)) logger.info("\t\tCopying companion file to {}".format(fname)) copy_from_cache(src, dst) # associated files (images, etc) for fname in [fn for fn in cached_files if fn.startswith("{}_".format(book.id))]: if path(fname).ext in ('.html', '.htm'): src = os.path.join(path(download_cache).abspath(), fname) dst = os.path.join(path(static_folder).abspath(), fname) if path(dst).exists() and not force: logger.debug("\t\tSkipping existing HTML {}".format(dst)) continue logger.info("\t\tExporting HTML file to {}".format(dst)) html, encoding = read_file(src) new_html = update_html_for_static(book=book, html_content=html) save_bs_output(new_html, dst, UTF8) else: try: handle_companion_file(fname, force=force) except Exception as e: logger.exception(e) logger.error("\t\tException while handling companion file: {}" .format(e)) # other formats for format in formats: if format not in book.formats() or format == 'html': continue try: handle_companion_file(fname_for(book, format), archive_name_for(book, format), force=force) except Exception as e: logger.exception(e) logger.error("\t\tException while handling companion file: {}" .format(e)) # book presentation article cover_fpath = os.path.join(static_folder, article_name_for(book=book, cover=True)) if not path(cover_fpath).exists() or force: logger.info("\t\tExporting to {}".format(cover_fpath)) html = cover_html_content_for(book=book, static_folder=static_folder, books=books, project_id=project_id) with open(cover_fpath, 'w') as f: if six.PY2: f.write(html.encode(UTF8)) else: f.write(html) else: logger.info("\t\tSkipping cover {}".format(cover_fpath))
def download_book(book, download_cache, languages, formats, force, s3_storage, optimizer_version): logger.info( "\tDownloading content files for Book #{id}".format(id=book.id)) # apply filters if not formats: formats = FORMAT_MATRIX.keys() # HTML is our base for ZIM for add it if not present if "html" not in formats: formats.append("html") book_dir = pathlib.Path(download_cache).joinpath(str(book.id)) optimized_dir = book_dir.joinpath("optimized") unoptimized_dir = book_dir.joinpath("unoptimized") unsuccessful_formats = [] for book_format in formats: unoptimized_fpath = unoptimized_dir.joinpath( fname_for(book, book_format)) optimized_fpath = optimized_dir.joinpath( archive_name_for(book, book_format)) # check if already downloaded if (unoptimized_fpath.exists() or optimized_fpath.exists()) and not force: logger.debug( f"\t\t{book_format} already exists for book #{book.id}") continue if force: if book_format == "html": for fpath in book_dir.iterdir(): if fpath.is_file() and fpath.suffix not in [ ".pdf", ".epub" ]: fpath.unlink() else: if unoptimized_fpath.exists(): unoptimized_fpath.unlink() if optimized_fpath.exists(): optimized_fpath.unlink() # delete dirs which are empty for dir_name in [optimized_dir, unoptimized_dir]: if not dir_name.exists(): continue if not list(dir_name.iterdir()): dir_name.rmdir() # retrieve corresponding BookFormat bfs = BookFormat.filter(book=book) if book_format == "html": patterns = [ "mnsrb10h.htm", "8ledo10h.htm", "tycho10f.htm", "8ledo10h.zip", "salme10h.htm", "8nszr10h.htm", "{id}-h.html", "{id}.html.gen", "{id}-h.htm", "8regr10h.zip", "{id}.html.noimages", "8lgme10h.htm", "tycho10h.htm", "tycho10h.zip", "8lgme10h.zip", "8indn10h.zip", "8resp10h.zip", "20004-h.htm", "8indn10h.htm", "8memo10h.zip", "fondu10h.zip", "{id}-h.zip", "8mort10h.zip", ] bfso = bfs bfs = bfs.join(Format).filter(Format.pattern << patterns) if not bfs.count(): pp( list([(b.format.mime, b.format.images, b.format.pattern) for b in bfs])) pp( list([(b.format.mime, b.format.images, b.format.pattern) for b in bfso])) logger.error("html not found") unsuccessful_formats.append(book_format) continue else: bfs = bfs.filter(BookFormat.format << Format.filter( mime=FORMAT_MATRIX.get(book_format))) if not bfs.count(): logger.debug("[{}] not avail. for #{}# {}".format( book_format, book.id, book.title)) unsuccessful_formats.append(book_format) continue if bfs.count() > 1: try: bf = bfs.join(Format).filter(Format.images).get() except Exception: bf = bfs.get() else: bf = bfs.get() logger.debug("[{}] Requesting URLs for #{}# {}".format( book_format, book.id, book.title)) # retrieve list of URLs for format unless we have it in DB if bf.downloaded_from and not force: urls = [bf.downloaded_from] else: urld = get_urls(book) urls = list(reversed(urld.get(FORMAT_MATRIX.get(book_format)))) import copy allurls = copy.copy(urls) downloaded_from_cache = False while urls: url = urls.pop() # for development # if len(allurls) != 1: # if not resource_exists(url): # continue # HTML files are *sometime* available as ZIP files if url.endswith(".zip"): zpath = unoptimized_dir.joinpath( f"{fname_for(book, book_format)}.zip") etag = get_etag_from_url(url) if s3_storage: if download_from_cache( book=book, etag=etag, book_format=book_format, dest_dir=optimized_dir, s3_storage=s3_storage, optimizer_version=optimizer_version, ): downloaded_from_cache = True break if not download_file(url, zpath): logger.error("ZIP file donwload failed: {}".format(zpath)) continue # save etag book.html_etag = etag book.save() # extract zipfile handle_zipped_epub(zippath=zpath, book=book, dst_dir=unoptimized_dir) else: if (url.endswith(".htm") or url.endswith(".html") or url.endswith(".html.utf8") or url.endswith(".epub")): etag = get_etag_from_url(url) if s3_storage: logger.info( f"Trying to download {book.id} from optimization cache" ) if download_from_cache( book=book, etag=etag, book_format=book_format, dest_dir=optimized_dir, s3_storage=s3_storage, optimizer_version=optimizer_version, ): downloaded_from_cache = True break if not download_file(url, unoptimized_fpath): logger.error( "file donwload failed: {}".format(unoptimized_fpath)) continue # save etag if html or epub if download is successful if (url.endswith(".htm") or url.endswith(".html") or url.endswith(".html.utf8")): logger.debug(f"Saving html ETag for {book.id}") book.html_etag = etag book.save() elif url.endswith(".epub"): logger.debug(f"Saving epub ETag for {book.id}") book.epub_etag = etag book.save() # store working URL in DB bf.downloaded_from = url bf.save() # break as we got a working URL break if not bf.downloaded_from and not downloaded_from_cache: logger.error("NO FILE FOR #{}/{}".format(book.id, book_format)) # delete instance from DB if download failed logger.info("Deleting instance from DB") bf.delete_instance() unsuccessful_formats.append(book_format) pp(allurls) # delete book from DB if not downloaded in any format if len(unsuccessful_formats) == len(formats): logger.debug( f"Book #{book.id} could not be downloaded in any format. Deleting from DB ..." ) book.delete_instance() if book_dir.exists(): shutil.rmtree(book_dir, ignore_errors=True) return download_cover(book, book_dir, s3_storage, optimizer_version)
def export_all_books(static_folder, download_cache, concurrency, languages=[], formats=[], only_books=[], force=False): project_id = get_project_id(languages=languages, formats=formats, only_books=only_books) # ensure dir exist path(static_folder).mkdir_p() books = get_list_of_filtered_books(languages=languages, formats=formats, only_books=only_books) if not len(get_langs_with_count(books=books)): critical_error("Unable to proceed. Combination of lamguages, " "books and formats has no result.") # sz = len(list(books)) # logger.debug("\tFiltered book collection size: {}".format(sz)) def nb_by_fmt(fmt): return sum([ 1 for book in books if BookFormat.select(BookFormat, Book, Format).join(Book).switch( BookFormat).join(Format).where(Book.id == book.id).where( Format.mime == FORMAT_MATRIX.get(fmt)).count() ]) logger.debug("\tFiltered book collection, PDF: {}".format( nb_by_fmt('pdf'))) logger.debug("\tFiltered book collection, ePUB: {}".format( nb_by_fmt('epub'))) logger.debug("\tFiltered book collection, HTML: {}".format( nb_by_fmt('html'))) # export to JSON helpers export_to_json_helpers(books=books, static_folder=static_folder, languages=languages, formats=formats, project_id=project_id) # export HTML index and other static files export_skeleton(static_folder=static_folder, dev_mode=False, languages=languages, formats=formats, only_books=only_books) # Compute popularity popbooks = books.order_by(Book.downloads.desc()) popbooks_count = popbooks.count() stars_limits = [0] * NB_POPULARITY_STARS stars = NB_POPULARITY_STARS nb_downloads = popbooks[0].downloads for ibook in range(0, popbooks.count(), 1): if ibook > float(NB_POPULARITY_STARS-stars+1)/NB_POPULARITY_STARS*popbooks_count \ and popbooks[ibook].downloads < nb_downloads: stars_limits[stars - 1] = nb_downloads stars = stars - 1 nb_downloads = popbooks[ibook].downloads # export to HTML cached_files = os.listdir(download_cache) for book in books: book.popularity = sum([ int(book.downloads >= stars_limits[i]) for i in range(NB_POPULARITY_STARS) ]) dlb = lambda b: export_book_to(b, static_folder=static_folder, download_cache=download_cache, cached_files=cached_files, languages=languages, formats=formats, books=books, project_id=project_id, force=force) Pool(concurrency).map(dlb, books)
def handle_companion_file( fname, dstfname=None, book=None, force=False, as_ext=None, html_file_list=None, s3_storage=None, ): ext = fname.suffix if as_ext is None else as_ext src = fname if dstfname is None: dstfname = fname.name dst = static_folder.joinpath(dstfname) if dst.exists() and not force: logger.debug("\t\tSkipping existing companion {}".format(dstfname)) return # optimization based on mime/extension if ext in (".png", ".jpg", ".jpeg", ".gif"): logger.info("\t\tCopying and optimizing image companion {}".format(fname)) optimize_image(src, dst) if dst.name == (f"{book.id}_cover_image.jpg") and s3_storage: upload_to_cache( asset=dst, book_format="cover", book_id=book.id, etag=book.cover_etag, s3_storage=s3_storage, optimizer_version=optimizer_version, ) update_download_cache(src, dst) elif html_file_list: html_file_list.append(dst) update_download_cache(src, dst) elif ext == ".epub": logger.info("\t\tCreating optimized EPUB file {}".format(fname)) tmp_epub = tempfile.NamedTemporaryFile(suffix=".epub", dir=TMP_FOLDER) tmp_epub.close() try: optimize_epub(src, tmp_epub.name) except zipfile.BadZipFile: logger.warn( "\t\tBad zip file. " "Copying as it might be working{}".format(fname) ) handle_companion_file(fname, dstfname, book, force, as_ext=".zip") else: path(tmp_epub.name).move(dst) if s3_storage: upload_to_cache( asset=dst, book_format="epub", book_id=book.id, etag=book.epub_etag, s3_storage=s3_storage, optimizer_version=optimizer_version, ) update_download_cache(src, dst) else: # excludes files created by Windows Explorer if src.name.endswith("_Thumbs.db"): return # copy otherwise (PDF mostly) logger.info("\t\tCopying companion file to {}".format(dst)) copy_file(src, dst) if ext != ".pdf" and ext != ".zip" and html_file_list: html_file_list.append(dst) update_download_cache(src, dst)