async def load_guesses(filename, comics_data): comic_presets = comics_data.get('presets', {}) comic_mixins = comics_data.get('mixins', {}) guess_comics = comics_data['guess_comics'] if not guess_comics: return loaded_comic_data = {} failed_comics = {} comic_guessers = FutureList() comic_loaders = FutureList() for name, url in guess_comics.items(): comic_guessers.add(load_guess_for(name, url, comic_presets, comic_mixins)) async for name, data, comics in comic_guessers: print(name, data) if not data or not comics: print() print() failed_comics[name] = guess_comics[name] continue loaded_comic_data[name] = data parser = ComicParser.load_parser(data, comic_presets, comic_mixins) downloader = ComicDownloader(parser, data['meta']) await downloader.load_existing_comics() for comic_id, comic in comics.items(): if comic_id not in downloader.comic_site.comics: downloader.comic_site.set_comic(comic_id, comic) await downloader.comic_site.save() comic_loaders.add(downloader.load_comics()) comics_data['comics'] = dict(comics_data['comics']).update(loaded_comic_data) comics_data['failed_comics'] = failed_comics print(comics_data) await comic_loaders
async def find_base_class(self, url, content): for base_name in self._base_classes: parser = ComicParser.load_parser({'base': base_name}, self._base_classes, self._mixins) log.info("%s -> %r", base_name, parser) try: comic = parser.load_comic(url, content) if comic.next is not None: return base_name, parser, comic except (SkipComicError, MissingElementError) as e: # log.exception("%s", base_name) pass return None, None, None
async def load_comics(comics, comic_presets, comic_mixins): comic_parsers = FutureList() for name, comic in comics.items(): metadata = comic.get('meta', {}) for meta_keys in ['name', 'layout', 'folder', 'initialurl']: if meta_keys in comic and meta_keys not in metadata: metadata[meta_keys] = comic[meta_keys] metadata.setdefault('name', name) if metadata.get('layout') not in ('horizontal', 'vertical', 'pane'): metadata['layout'] = 'horizontal' if 'folder' not in comic: metadata['folder'] = to_folder_name(name) parser = ComicParser.load_parser(comic, comic_presets, comic_mixins) comic_parsers.add(ComicDownloader(parser, metadata).load_comics()) await comic_parsers