def setup_database(wipe=False): logger.info("Setting up the database") for model in (License, Format, Author, Book, BookFormat): if wipe: model.drop_table(fail_silently=True) if not model.table_exists(): model.create_table() logger.debug("Created table for {}".format(model._meta.name)) load_fixtures(model) else: logger.debug("{} table already exists.".format(model._meta.name))
def handle_companion_file(fname, dstfname=None, book=None): src = os.path.join(path(download_cache).abspath(), fname) if dstfname is None: dstfname = fname dst = os.path.join(path(static_folder).abspath(), dstfname) # optimization based on mime/extension if path(fname).ext in ('.png', '.jpg', '.jpeg', '.gif'): copy_from_cache(src, dst) optimize_image(dst) elif path(fname).ext == '.epub': tmp_epub = tempfile.NamedTemporaryFile(suffix='.epub') tmp_epub.close() optimize_epub(src, tmp_epub.name) path(tmp_epub.name).move(dst) else: # PDF mostly logger.debug("\t\tshitty ext: {}".format(dst)) copy_from_cache(src, dst)
def handle_companion_file(fname, dstfname=None, book=None): src = os.path.join(path(download_cache).abspath(), fname) if dstfname is None: dstfname = fname dst = os.path.join(path(static_folder).abspath(), dstfname) # optimization based on mime/extension if path(fname).ext in ('.png', '.jpg', '.jpeg', '.gif'): copy_from_cache(src, dst) optimize_image(path_for_cmd(dst)) elif path(fname).ext == '.epub': tmp_epub = tempfile.NamedTemporaryFile(suffix='.epub', dir=TMP_FOLDER) tmp_epub.close() optimize_epub(src, tmp_epub.name) path(tmp_epub.name).move(dst) else: # excludes files created by Windows Explorer if src.endswith('_Thumbs.db'): return # copy otherwise (PDF mostly) logger.debug("\t\tshitty ext: {}".format(dst)) copy_from_cache(src, dst)
def handle_companion_file(fname, dstfname=None, book=None): src = os.path.join(path(download_cache).abspath(), fname) if dstfname is None: dstfname = fname dst = os.path.join(path(static_folder).abspath(), dstfname) # optimization based on mime/extension if path(fname).ext in ('.png', '.jpg', '.jpeg', '.gif'): copy_from_cache(src, dst) optimize_image(path_for_cmd(dst)) elif path(fname).ext == '.epub': tmp_epub = tempfile.NamedTemporaryFile(suffix='.epub', dir=TMP_FOLDER) tmp_epub.close() optimize_epub(src, tmp_epub.name) path(tmp_epub.name).move(dst) else: # excludes files created by Windows Explorer if src.endswith('_Thumbs.db'): return # copy otherwise (PDF mostly) logger.debug("\t\tshitty ext: {}".format(dst)) copy_from_cache(src, dst)
def download_all_books(url_mirror, download_cache, languages=[], formats=[], only_books=[], force=False): available_books = get_list_of_filtered_books( languages=languages, formats=formats, only_books=only_books) # ensure dir exist path(download_cache).mkdir_p() for book in available_books: logger.info("\tDownloading content files for Book #{id}" .format(id=book.id)) # apply filters if not formats: formats = FORMAT_MATRIX.keys() # HTML is our base for ZIM for add it if not present if not 'html' in formats: formats.append('html') for format in formats: fpath = os.path.join(download_cache, fname_for(book, format)) # check if already downloaded if path(fpath).exists() and not force: logger.debug("\t\t{fmt} already exists at {path}" .format(fmt=format, path=fpath)) continue # retrieve corresponding BookFormat bfs = BookFormat.filter(book=book) if format == 'html': patterns = ['mnsrb10h.htm', '8ledo10h.htm', 'tycho10f.htm', '8ledo10h.zip', 'salme10h.htm', '8nszr10h.htm', '{id}-h.html', '{id}.html.gen', '{id}-h.htm', '8regr10h.zip', '{id}.html.noimages', '8lgme10h.htm', 'tycho10h.htm', 'tycho10h.zip', '8lgme10h.zip', '8indn10h.zip', '8resp10h.zip', '20004-h.htm', '8indn10h.htm', '8memo10h.zip', 'fondu10h.zip', '{id}-h.zip', '8mort10h.zip'] bfso = bfs bfs = bfs.join(Format).filter(Format.pattern << patterns) if not bfs.count(): from pprint import pprint as pp ; pp(list([(b.format.mime, b.format.images, b.format.pattern) for b in bfs])) from pprint import pprint as pp ; pp(list([(b.format.mime, b.format.images, b.format.pattern) for b in bfso])) logger.error("html not found") continue else: bfs = bfs.filter(BookFormat.format << Format.filter(mime=FORMAT_MATRIX.get(format))) if not bfs.count(): logger.debug("[{}] not avail. for #{}# {}" .format(format, book.id, book.title)) continue if bfs.count() > 1: try: bf = bfs.join(Format).filter(Format.images == True).get() except: bf = bfs.get() else: bf = bfs.get() logger.debug("[{}] Requesting URLs for #{}# {}" .format(format, book.id, book.title)) # retrieve list of URLs for format unless we have it in DB if bf.downloaded_from and not force: urls = [bf.downloaded_from] else: urld = get_urls(book) urls = list(reversed(urld.get(FORMAT_MATRIX.get(format)))) import copy allurls = copy.copy(urls) while(urls): url = urls.pop() if not resource_exists(url): continue # HTML files are *sometime* available as ZIP files if url.endswith('.zip'): zpath = "{}.zip".format(fpath) if not download_file(url, zpath): logger.error("ZIP file donwload failed: {}".format(zpath)) continue # extract zipfile handle_zipped_epub(zippath=zpath, book=book, download_cache=download_cache) else: if not download_file(url, fpath): logger.error("file donwload failed: {}".format(fpath)) continue # store working URL in DB bf.downloaded_from = url bf.save() if not bf.downloaded_from: logger.error("NO FILE FOR #{}/{}".format(book.id, format)) from pprint import pprint as pp ; pp(allurls) continue
def export_all_books(static_folder, download_cache, languages=[], formats=[], only_books=[]): # ensure dir exist path(static_folder).mkdir_p() books = get_list_of_filtered_books(languages=languages, formats=formats, only_books=only_books) sz = len(list(books)) logger.debug("\tFiltered book collection size: {}".format(sz)) def nb_by_fmt(fmt): return sum([ 1 for book in books if BookFormat.select(BookFormat, Book, Format).join(Book).switch( BookFormat).join(Format).where(Book.id == book.id).where( Format.mime == FORMAT_MATRIX.get(fmt)).count() ]) logger.debug("\tFiltered book collection, PDF: {}".format( nb_by_fmt('pdf'))) logger.debug("\tFiltered book collection, ePUB: {}".format( nb_by_fmt('epub'))) logger.debug("\tFiltered book collection, HTML: {}".format( nb_by_fmt('html'))) # export to JSON helpers export_to_json_helpers(books=books, static_folder=static_folder, languages=languages, formats=formats) # copy CSS/JS/* to static_folder src_folder = tmpl_path() for fname in ('css', 'js', 'jquery', 'favicon.ico', 'favicon.png', 'jquery-ui', 'datatables', 'fonts', 'l10n'): src = os.path.join(src_folder, fname) dst = os.path.join(static_folder, fname) if not path(fname).ext: path(dst).rmtree_p() path(src).copytree(dst) else: path(src).copyfile(dst) # export homepage template = jinja_env.get_template('index.html') context = get_default_context(books=books) context.update({'show_books': True}) with open(os.path.join(static_folder, 'Home.html'), 'w') as f: f.write(template.render(**context).encode('utf-8')) # Compute popularity popbooks = books.order_by(Book.downloads.desc()) stars_limits = [0] * NB_POPULARITY_STARS stars = NB_POPULARITY_STARS nb_downloads = popbooks[0].downloads for ibook in range(0, popbooks.count(), 1): if ibook > float(NB_POPULARITY_STARS-stars+1)/NB_POPULARITY_STARS*popbooks.count() \ and popbooks[ibook].downloads < nb_downloads: stars_limits[stars - 1] = nb_downloads stars = stars - 1 nb_downloads = popbooks[ibook].downloads # export to HTML cached_files = os.listdir(download_cache) for book in books: book.popularity = sum([ int(book.downloads >= stars_limits[i]) for i in range(NB_POPULARITY_STARS) ]) export_book_to(book=book, static_folder=static_folder, download_cache=download_cache, cached_files=cached_files, languages=languages, formats=formats, books=books)
def build_zimfile(static_folder, zim_path=None, languages=[], formats=[], title=None, description=None, only_books=[]): if not languages: languages = ['mul'] languages.sort() formats.sort() if title is None: if len(languages) > 5: title = ("Project Gutenberg Library with {formats}" .format(formats=",".join(formats))) else: title = ("Project Gutenberg Library ({langs}) with {formats}" .format(langs=",".join(languages), formats=",".join(formats))) logger.info("\tWritting ZIM for {}".format(title)) if description is None: description = "The first producer of free ebooks" if zim_path is None: if len(languages) > 1: zim_path = "gutenberg_all_{date}.zim".format( date=datetime.datetime.now().strftime('%m_%Y')) else: zim_path = "gutenberg_{lang}_all_{date}.zim".format( lang=languages[0], date=datetime.datetime.now().strftime('%Y-%m')) languages = [ISO_MATRIX.get(lang, lang) for lang in languages] languages.sort() context = { 'languages': ','.join(languages), 'title': title, 'description': description, 'creator': 'gutenberg.org', 'publisher': 'Kiwix', 'home': 'Home.html', 'favicon': 'favicon.png', 'static': static_folder, 'zim': zim_path } cmd = ('zimwriterfs --welcome=\\"{home}\\" --favicon=\\"{favicon}\\" ' '--language=\\"{languages}\\" --title=\\"{title}\\" ' '--description=\\"{description}\\" ' '--creator=\\"{creator}\\" --publisher=\\"{publisher}\\" \\"{static}\\" \\"{zim}\\"' .format(**context)) logger.debug("\t\t{}".format(re.sub('\\\\"','"',cmd))) if exec_cmd(cmd): logger.info("Successfuly created ZIM file at {}".format(zim_path)) else: logger.error("Unable to create ZIM file :(")
def download_all_books(url_mirror, download_cache, languages=[], formats=[], only_books=[], force=False): available_books = get_list_of_filtered_books(languages=languages, formats=formats, only_books=only_books) # ensure dir exist path(download_cache).mkdir_p() for book in available_books: logger.info( "\tDownloading content files for Book #{id}".format(id=book.id)) # apply filters if not formats: formats = FORMAT_MATRIX.keys() # HTML is our base for ZIM for add it if not present if not 'html' in formats: formats.append('html') for format in formats: fpath = os.path.join(download_cache, fname_for(book, format)) # check if already downloaded if path(fpath).exists() and not force: logger.debug("\t\t{fmt} already exists at {path}".format( fmt=format, path=fpath)) continue # retrieve corresponding BookFormat bfs = BookFormat.filter(book=book) if format == 'html': patterns = [ 'mnsrb10h.htm', '8ledo10h.htm', 'tycho10f.htm', '8ledo10h.zip', 'salme10h.htm', '8nszr10h.htm', '{id}-h.html', '{id}.html.gen', '{id}-h.htm', '8regr10h.zip', '{id}.html.noimages', '8lgme10h.htm', 'tycho10h.htm', 'tycho10h.zip', '8lgme10h.zip', '8indn10h.zip', '8resp10h.zip', '20004-h.htm', '8indn10h.htm', '8memo10h.zip', 'fondu10h.zip', '{id}-h.zip', '8mort10h.zip' ] bfso = bfs bfs = bfs.join(Format).filter(Format.pattern << patterns) if not bfs.count(): from pprint import pprint as pp pp( list([(b.format.mime, b.format.images, b.format.pattern) for b in bfs])) from pprint import pprint as pp pp( list([(b.format.mime, b.format.images, b.format.pattern) for b in bfso])) logger.error("html not found") continue else: bfs = bfs.filter(BookFormat.format << Format.filter( mime=FORMAT_MATRIX.get(format))) if not bfs.count(): logger.debug("[{}] not avail. for #{}# {}".format( format, book.id, book.title)) continue if bfs.count() > 1: try: bf = bfs.join(Format).filter(Format.images == True).get() except: bf = bfs.get() else: bf = bfs.get() logger.debug("[{}] Requesting URLs for #{}# {}".format( format, book.id, book.title)) # retrieve list of URLs for format unless we have it in DB if bf.downloaded_from and not force: urls = [bf.downloaded_from] else: urld = get_urls(book) urls = list(reversed(urld.get(FORMAT_MATRIX.get(format)))) import copy allurls = copy.copy(urls) while (urls): url = urls.pop() if not resource_exists(url): continue # HTML files are *sometime* available as ZIP files if url.endswith('.zip'): zpath = "{}.zip".format(fpath) if not download_file(url, zpath): logger.error( "ZIP file donwload failed: {}".format(zpath)) continue # extract zipfile handle_zipped_epub(zippath=zpath, book=book, download_cache=download_cache) else: if not download_file(url, fpath): logger.error("file donwload failed: {}".format(fpath)) continue # store working URL in DB bf.downloaded_from = url bf.save() if not bf.downloaded_from: logger.error("NO FILE FOR #{}/{}".format(book.id, format)) from pprint import pprint as pp pp(allurls) continue
def load_fixtures(model): logger.info("Loading fixtures for {}".format(model._meta.name)) for fixture in getattr(model._meta, 'fixtures', []): f = model.create(**fixture) logger.debug("[fixtures] Created {}".format(f))
def export_all_books(static_folder, download_cache, languages=[], formats=[], only_books=[]): # ensure dir exist path(static_folder).mkdir_p() books = get_list_of_filtered_books(languages=languages, formats=formats, only_books=only_books) sz = len(list(books)) logger.debug("\tFiltered book collection size: {}".format(sz)) def nb_by_fmt(fmt): return sum([1 for book in books if BookFormat.select(BookFormat, Book, Format) .join(Book).switch(BookFormat) .join(Format) .where(Book.id == book.id) .where(Format.mime == FORMAT_MATRIX.get(fmt)) .count()]) logger.debug("\tFiltered book collection, PDF: {}" .format(nb_by_fmt('pdf'))) logger.debug("\tFiltered book collection, ePUB: {}" .format(nb_by_fmt('epub'))) logger.debug("\tFiltered book collection, HTML: {}" .format(nb_by_fmt('html'))) # export to JSON helpers export_to_json_helpers(books=books, static_folder=static_folder, languages=languages, formats=formats) # copy CSS/JS/* to static_folder src_folder = tmpl_path() for fname in ('css', 'js', 'jquery', 'favicon.ico', 'favicon.png', 'jquery-ui', 'datatables', 'fonts', 'l10n'): src = os.path.join(src_folder, fname) dst = os.path.join(static_folder, fname) if not path(fname).ext: path(dst).rmtree_p() path(src).copytree(dst) else: path(src).copyfile(dst) # export homepage template = jinja_env.get_template('index.html') context = get_default_context(books=books) context.update({'show_books': True}) with open(os.path.join(static_folder, 'Home.html'), 'w') as f: f.write(template.render(**context).encode('utf-8')) # Compute popularity popbooks = books.order_by(Book.downloads.desc()) stars_limits = [0] * NB_POPULARITY_STARS stars = NB_POPULARITY_STARS nb_downloads = popbooks[0].downloads for ibook in range(0, popbooks.count(), 1): if ibook > float(NB_POPULARITY_STARS-stars+1)/NB_POPULARITY_STARS*popbooks.count() \ and popbooks[ibook].downloads < nb_downloads: stars_limits[stars-1] = nb_downloads stars = stars - 1 nb_downloads = popbooks[ibook].downloads # export to HTML cached_files = os.listdir(download_cache) for book in books: book.popularity = sum( [int(book.downloads >= stars_limits[i]) for i in range(NB_POPULARITY_STARS)]) export_book_to(book=book, static_folder=static_folder, download_cache=download_cache, cached_files=cached_files, languages=languages, formats=formats, books=books)