def _save_all_pages(self): """Iterate over the site pages, call _save_page for each.""" orm.create_tables('Page', 'Revision', 'Vote', 'ForumPost', 'PageTag', 'ForumThread', 'User', 'Tag') count = next(self.wiki.list_pages(body='total', limit=1))._body['total'] bar = utils.ProgressBar('SAVING PAGES'.ljust(20), int(count)) bar.start() for _ in self.pool.map(self._save_page, self.wiki.list_pages()): bar.value += 1 bar.stop()
def _save_all_pages(self): """Itera sobre las páginas del sitio, llama a _save_page para cada uno.""" orm.create_tables( 'Page', 'Revision', 'Vote', 'ForumPost', 'PageTag', 'ForumThread', 'User', 'Tag') count = next( self.wiki.list_pages(body='total', limit=1))._body['total'] bar = utils.ProgressBar('Guardando Página'.ljust(20), int(count)) bar.start() for _ in self.pool.map(self._save_page, self.wiki.list_pages()): bar.value += 1 bar.stop()
def _save_meta(self): orm.create_tables('Image', 'ImageStatus') licenses = { 'PERMISSION GRANTED', 'BY-NC-SA CC', 'BY-SA CC', 'PUBLIC DOMAIN' } images = [i for i in self.wiki.list_images() if i.status in licenses] self.ibar = utils.ProgressBar('SAVING IMAGES'.ljust(20), len(images)) self.ibar.start() data = list(self.pool.map(self._save_image, images)) self.ibar.stop() images = orm.ImageStatus.convert_to_id([i._asdict() for i in images], key='status') orm.Image.insert_many( dict(i, data=d) for i, d in zip(images, data) if d)
def _save_meta(self): orm.create_tables( 'Image', 'ImageStatus') licenses = { 'PERMISO GARANTIZÁDO', 'BY-NC-SA CC', 'BY-SA CC', 'DOMINIO PÚBLICO'} images = [i for i in self.wiki.list_images() if i.status in licenses] self.ibar = utils.ProgressBar( 'GUARDANDO IMÁGEN'.ljust(20), len(images)) self.ibar.start() data = list(self.pool.map(self._save_image, images)) self.ibar.stop() images = orm.ImageStatus.convert_to_id( [i._asdict() for i in images], key='status') orm.Image.insert_many( dict(i, data=d) for i, d in zip(images, data) if d)
def _save_forums(self): """Download and save standalone forum threads.""" orm.create_tables('ForumPost', 'ForumThread', 'ForumCategory', 'User') cats = self.wiki.list_categories() cats = [i for i in cats if i.title != 'Per page discussions'] orm.ForumCategory.insert_many( dict(id=c.id, title=c.title, description=c.description) for c in cats) total_size = sum(c.size for c in cats) bar = utils.ProgressBar('SAVING FORUM THREADS', total_size) bar.start() for cat in cats: threads = set(self.wiki.list_threads(cat.id)) c_id = itertools.repeat(cat.id) for _ in self.pool.map(self._save_thread, threads, c_id): bar.value += 1 bar.stop()
def _save_all_pages(self): """Iterate over the site pages, call _save_page for each.""" orm.create_tables( "Page", "Revision", "Vote", "ForumPost", "PageTag", "ForumThread", "User", "Tag", ) count = next(self.wiki.list_pages(body="total", limit=1))._body["total"] bar = utils.ProgressBar("SAVING PAGES".ljust(20), int(count)) bar.start() for _ in self.pool.map(self._save_page, self.wiki.list_pages()): bar.value += 1 bar.stop()
def _save_forums(self): """Descarga y salva los hilos autónomos del foro.""" orm.create_tables( 'ForumPost', 'ForumThread', 'ForumCategory', 'User') cats = self.wiki.list_categories() cats = [i for i in cats if i.title != 'Per page discussions'] orm.ForumCategory.insert_many(dict( id=c.id, title=c.title, description=c.description) for c in cats) total_size = sum(c.size for c in cats) bar = utils.ProgressBar('GUARDANDO HILO DEL FORO', total_size) bar.start() for cat in cats: threads = set(self.wiki.list_threads(cat.id)) c_id = itertools.repeat(cat.id) for _ in self.pool.map(self._save_thread, threads, c_id): bar.value += 1 bar.stop()