def show(): parser = optparse.OptionParser(usage="%prog [-e|--expand] --conf CONF ARTICLE [...]") parser.add_option("-c", "--conf", help="config file") parser.add_option("-e", "--expand", action="store_true", help="expand templates") parser.add_option("-t", "--template", action="store_true", help="show template") options, args = parser.parse_args() if not args: parser.error("missing ARTICLE argument") articles = [unicode(x, 'utf-8') for x in args] conf = options.conf if not conf: parser.error("missing --conf argument") from mwlib import wiki, expander db = wiki.makewiki(conf)['wiki'] for a in articles: if options.template: raw=db.getTemplate(a) else: raw=db.getRawArticle(a) if raw: if options.expand: te = expander.Expander(raw, pagename=a, wikidb=db) raw = te.expandTemplates() print raw.encode("utf-8")
def testserve(): parser = optparse.OptionParser(usage="%prog --config CONFIG ARTICLE [...]") parser.add_option("-c", "--config", help="configuration file/URL/shortcut") options, args = parser.parse_args() conf = options.config if not conf: parser.error("missing --config argument") from mwlib import wiki, web res = wiki.makewiki(conf) db = res['wiki'] images = res['images'] from wsgiref.simple_server import make_server, WSGIServer from SocketServer import ForkingMixIn class MyServer(ForkingMixIn, WSGIServer): pass iface, port = '0.0.0.0', 8080 print "serving on %s:%s" % (iface, port) http = make_server(iface, port, web.Serve(db, res['images']), server_class=MyServer) http.serve_forever()
def makewiki(self): kw = self.options.__dict__.copy() kw["metabook"] = self.metabook env = wiki.makewiki(**kw) if not env.metabook: self.metabook = env.metabook = metabook.collection() env.init_metabook() if self.options.noimages: env.images = None def setmb(name): n = getattr(self.options, name) if n: env.metabook[name] = n setmb("title") setmb("subtitle") setmb("editor") # add default licenses cfg = self.options.config or "" if cfg.startswith(":") and not env.metabook.licenses: mw_license_url = wiki.wpwikis.get(cfg[1:])['mw_license_url'] env.metabook.licenses.append(dict(mw_license_url=mw_license_url, type="license")) return env
def articles(): env = wiki.makewiki(wcb.paths["wikiconf"]) rm = nshandling.get_redirect_matcher(env.wiki.siteinfo, env.wiki.nshandler) if os.path.exists(os.path.join(wcb.paths["tmp"], 'articles.list')): logger.info("reading names from " + os.path.join(wcb.paths["tmp"], 'articles.list')) f = codecs.open(os.path.join(wcb.paths["tmp"], 'articles.list'), 'r', 'utf-8') names = [l.strip() for l in f] f.close() else: logger.warn("empty cache, this will take a while") names = [k for k in env.wiki.reader.keys() if env.wiki.nshandler.splitname(k)[0] == nshandling.NS_MAIN] logger.info("writing names to " + os.path.join(wcb.paths["tmp"], 'articles.list')) f = codecs.open(os.path.join(wcb.paths["tmp"], 'articles.list'), 'w', 'utf-8') names_nord = [] #names - redirects for n in names: raw = env.wiki.reader[n] if not rm(raw): f.write(n + '\n') names_nord.append(n) f.close() return names_nord return names
def serve(): parser = optparse.OptionParser(usage="%prog --conf CONF ARTICLE [...]") parser.add_option("-c", "--conf", help="config file") options, args = parser.parse_args() conf = options.conf if not options.conf: parser.error("missing --conf argument") from mwlib import wiki, web res = wiki.makewiki(conf) db = res['wiki'] images = res['images'] from wsgiref.simple_server import make_server, WSGIServer from SocketServer import ForkingMixIn class MyServer(ForkingMixIn, WSGIServer): pass iface, port = '0.0.0.0', 8080 print "serving on %s:%s" % (iface, port) http = make_server(iface, port, web.Serve(db, res['images']), server_class=MyServer) http.serve_forever()
def get_environment(self): from mwlib.status import Status from mwlib import nuwiki env = self.parser.makewiki() if (isinstance(env.wiki, (nuwiki.NuWiki, nuwiki.adapt)) or isinstance(env, wiki.MultiEnvironment)): self.status = Status(self.options.status_file, progress_range=(0, 100)) return env from mwlib.apps.buildzip import make_zip self.zip_filename = make_zip(output=self.options.keep_zip, options=self.options, metabook=env.metabook, status=self.status) if env.images: try: env.images.clear() except OSError as err: if err.errno != errno.ENOENT: raise env = wiki.makewiki(self.zip_filename) self.status = Status(self.options.status_file, progress_range=(34, 100)) return env
def parse(): parser = optparse.OptionParser( usage="%prog [-a|--all] --config CONFIG [ARTICLE1 ...]") parser.add_option("-a", "--all", action="store_true", help="parse all articles") parser.add_option("--tb", action="store_true", help="show traceback on error") parser.add_option("-c", "--config", help="configuration file/URL/shortcut") options, args = parser.parse_args() if not args and not options.all: parser.error("missing option.") if not options.config: parser.error("missing --config argument") articles = [unicode(x, 'utf-8') for x in args] conf = options.config import traceback from mwlib import wiki, uparser w = wiki.makewiki(conf) db = w.wiki if options.all: if not hasattr(db, "articles"): raise RuntimeError( "%s does not support iterating over all articles" % (db, )) articles = db.articles() import time for x in articles: try: page = db.normalize_and_get_page(x, 0) if page: raw = page.rawtext else: raw = None # yes, raw can be None, when we have a redirect to a non-existing article. if raw is None: continue stime = time.time() a = uparser.parseString(x, raw=raw, wikidb=db) except Exception as err: print "F", repr(x), err if options.tb: traceback.print_exc() else: print "G", time.time() - stime, repr(x)
def parse(): parser = optparse.OptionParser(usage="%prog [-a|--all] --config CONFIG [ARTICLE1 ...]") parser.add_option("-a", "--all", action="store_true", help="parse all articles") parser.add_option("--tb", action="store_true", help="show traceback on error") parser.add_option("-c", "--config", help="configuration file/URL/shortcut") options, args = parser.parse_args() if not args and not options.all: parser.error("missing option.") if not options.config: parser.error("missing --config argument") articles = [unicode(x, 'utf-8') for x in args] conf = options.config import traceback from mwlib import wiki, uparser w = wiki.makewiki(conf) db = w.wiki if options.all: if not hasattr(db, "articles"): raise RuntimeError("%s does not support iterating over all articles" % (db, )) articles = db.articles() import time for x in articles: try: page = db.normalize_and_get_page(x, 0) if page: raw = page.rawtext else: raw = None # yes, raw can be None, when we have a redirect to a non-existing article. if raw is None: continue stime=time.time() a=uparser.parseString(x, raw=raw, wikidb=db) except Exception, err: print "F", repr(x), err if options.tb: traceback.print_exc() else: print "G", time.time()-stime, repr(x)
def show(): parser = optparse.OptionParser() parser.add_option("-c", "--config", help="configuration file/URL/shortcut") parser.add_option("-e", "--expand", action="store_true", help="expand templates") parser.add_option("-t", "--template", action="store_true", help="show template") parser.add_option("-f", help='read input from file. implies -e') options, args = parser.parse_args() if not args and not options.f: parser.error("missing ARTICLE argument") articles = [unicode(x, 'utf-8') for x in args] conf = options.config if not conf: parser.error("missing --config argument") from mwlib import wiki, expander db = wiki.makewiki(conf).wiki for a in articles: if options.template: defaultns = 10 else: defaultns = 0 page = db.normalize_and_get_page(a, defaultns) if page: raw = page.rawtext else: raw = None if raw: if options.expand: te = expander.Expander(raw, pagename=a, wikidb=db) raw = te.expandTemplates() print raw.encode("utf-8") if options.f: raw = unicode(open(options.f).read(), 'utf-8') te = expander.Expander(raw, pagename='test', wikidb=db) raw = te.expandTemplates() print raw.encode("utf-8")
def html(): parser = optparse.OptionParser(usage="%prog --conf CONF ARTICLE [...]") parser.add_option("-c", "--conf", help="config file") options, args = parser.parse_args() if not args: parser.error("missing ARTICLE argument") articles = [unicode(x, 'utf-8') for x in args] conf = options.conf if not options.conf: parser.error("missing --conf argument") import StringIO import tempfile import os import webbrowser from mwlib import wiki, uparser, htmlwriter res = wiki.makewiki(conf) db = res['wiki'] images = res['images'] for a in articles: raw=db.getRawArticle(a) if not raw: continue out=StringIO.StringIO() out.write("""<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> <head> <meta http-equiv="content-type" content="text/html; charset="utf-8"></meta> <link rel="stylesheet" href="pedia.css" /> </head> <body> """) a=uparser.parseString(x, raw=raw, wikidb=db) w=htmlwriter.HTMLWriter(out, images) w.write(a) fd, htmlfile = tempfile.mkstemp(".html") os.close(fd) open(htmlfile, "wb").write(out.getvalue().encode('utf-8')) webbrowser.open("file://"+htmlfile)
def html(): parser = optparse.OptionParser(usage="%prog --conf CONF ARTICLE [...]") parser.add_option("-c", "--conf", help="config file") options, args = parser.parse_args() if not args: parser.error("missing ARTICLE argument") articles = [unicode(x, 'utf-8') for x in args] conf = options.conf if not options.conf: parser.error("missing --conf argument") import StringIO import tempfile import os import webbrowser from mwlib import wiki, uparser, htmlwriter res = wiki.makewiki(conf) db = res['wiki'] images = res['images'] for a in articles: raw = db.getRawArticle(a) if not raw: continue out = StringIO.StringIO() out.write("""<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> <head> <meta http-equiv="content-type" content="text/html; charset="utf-8"></meta> <link rel="stylesheet" href="pedia.css" /> </head> <body> """) a = uparser.parseString(x, raw=raw, wikidb=db) w = htmlwriter.HTMLWriter(out, images) w.write(a) fd, htmlfile = tempfile.mkstemp(".html") os.close(fd) open(htmlfile, "wb").write(out.getvalue().encode('utf-8')) webbrowser.open("file://" + htmlfile)
def show(): parser = optparse.OptionParser() parser.add_option("-c", "--config", help="configuration file/URL/shortcut") parser.add_option("-e", "--expand", action="store_true", help="expand templates") parser.add_option("-t", "--template", action="store_true", help="show template") parser.add_option("-f", help='read input from file. implies -e') options, args = parser.parse_args() if not args and not options.f: parser.error("missing ARTICLE argument") articles = [unicode(x, 'utf-8') for x in args] conf = options.config if not conf: parser.error("missing --config argument") from mwlib import wiki, expander db = wiki.makewiki(conf).wiki for a in articles: if options.template: defaultns=10 else: defaultns=0 page = db.normalize_and_get_page(a, defaultns) if page: raw = page.rawtext else: raw = None if raw: if options.expand: te = expander.Expander(raw, pagename=a, wikidb=db) raw = te.expandTemplates() print raw.encode("utf-8") if options.f: raw = unicode(open(options.f).read(), 'utf-8') te = expander.Expander(raw, pagename='test', wikidb=db) raw = te.expandTemplates() print raw.encode("utf-8")
def makewiki(self): username, password, domain = None, None, None if self.options.login: if self.options.login.count(':') == 1: username, password = self.options.login.split(':', 1) else: username, password, domain = self.options.login.split(':', 2) env = wiki.makewiki( self.options.config, metabook=self.metabook, username=username, password=password, domain=domain, script_extension=self.options.script_extension, ) if self.options.noimages: env.images = None if self.options.template_blacklist or self.options.template_exclusion_category: if hasattr(env.wiki, 'setTemplateExclusion'): env.wiki.setTemplateExclusion( blacklist=self.options.template_blacklist, category=self.options.template_exclusion_category, ) else: log.warn( 'WikiDB does not support setting a template blacklist') if self.options.collectionpage: wikitext = env.wiki.getRawArticle(self.options.collectionpage) if wikitext is None: raise RuntimeError('No such collection page: %r' % (self.options.collectionpage, )) self.metabook = metabook.parse_collection_page(wikitext) env.metabook = self.metabook if self.options.title: env.metabook['title'] = self.options.title if self.options.subtitle: env.metabook['subtitle'] = self.options.subtitle return env
def show(): parser = optparse.OptionParser( usage="%prog [-e|--expand] --conf CONF ARTICLE [...]") parser.add_option("-c", "--conf", help="config file") parser.add_option("-e", "--expand", action="store_true", help="expand templates") parser.add_option("-t", "--template", action="store_true", help="show template") options, args = parser.parse_args() if not args: parser.error("missing ARTICLE argument") articles = [unicode(x, 'utf-8') for x in args] conf = options.conf if not conf: parser.error("missing --conf argument") from mwlib import wiki, expander db = wiki.makewiki(conf)['wiki'] for a in articles: if options.template: raw = db.getTemplate(a) else: raw = db.getRawArticle(a) if raw: if options.expand: te = expander.Expander(raw, pagename=a, wikidb=db) raw = te.expandTemplates() print raw.encode("utf-8")
def makewiki(self): username, password, domain = None, None, None if self.options.login: if self.options.login.count(':') == 1: username, password = self.options.login.split(':', 1) else: username, password, domain = self.options.login.split(':', 2) env = wiki.makewiki(self.options.config, metabook=self.metabook, username=username, password=password, domain=domain, script_extension=self.options.script_extension, ) if self.options.noimages: env.images = None if self.options.template_blacklist or self.options.template_exclusion_category: if hasattr(env.wiki, 'setTemplateExclusion'): env.wiki.setTemplateExclusion( blacklist=self.options.template_blacklist, category=self.options.template_exclusion_category, ) else: log.warn('WikiDB does not support setting a template blacklist') if self.options.collectionpage: wikitext = env.wiki.getRawArticle(self.options.collectionpage) if wikitext is None: raise RuntimeError('No such collection page: %r' % ( self.options.collectionpage, )) self.metabook = metabook.parse_collection_page(wikitext) env.metabook = self.metabook if self.options.title: env.metabook['title'] = self.options.title if self.options.subtitle: env.metabook['subtitle'] = self.options.subtitle return env
def coll_from_zip(basedir, env, status_callback=None): def img_ext_correct(fn): from PIL import Image img = Image.open(fn) fmt = '.' + img.format.lower() name, ext = os.path.splitext(fn) ext = ext.lower() if ext == '.jpg': ext = '.jpeg' if fmt != ext: return (False, name + fmt) else: return (True, fn) if isinstance(env, basestring): from mwlib import wiki env = wiki.makewiki(env) coll = Collection(basedir=basedir, title=env.metabook.title or '', subtitle=env.metabook.subtitle or '', editor=env.metabook.editor or '', language=env.wiki.siteinfo.get('general', {}).get('lang', 'en')) missing_images = [] num_items = len(env.metabook.walk()) progress_inc = 100.0 / num_items license_checker = LicenseChecker(image_db=env.images, filter_type='blacklist') license_checker.readLicensesCSV() for n, item in enumerate(env.metabook.walk()): if item.type == 'chapter': chapter = Chapter(item.title) coll.append(chapter) continue elif item.type == 'custom': # a "custom" item currently can be the preface added at pediapress.com # FIXME: support custom item continue title = item.title if isinstance(title, str): title = unicode(title, 'utf-8') url = item.wiki.getURL(title, item.revision) if isinstance(url, str): url = unicode(url, 'utf-8') data = item.wiki.getHTML(title, item.revision) try: html = data['text']['*'] except KeyError: print 'WARNING: article missing, skipping %r' % item.title continue if isinstance(html, str): html = unicode(html, 'utf-8') html = '<div id="content"><h1>%s</h1>\n\n%s</div>' % ( title.encode('utf-8'), html.encode('utf-8')) wp = WebPage(coll, title, url, user_agent='Mozilla/5.0', contributors=env.wiki.getAuthors( title=item.title, revision=item.revision)) # images wp.canonical_url = urlparse.urljoin( item._env.wiki.siteinfo['general']['base'], urllib2.quote(title.replace(' ', '_').encode('utf-8')).decode('utf-8')) open(wp.get_path('content.orig'), 'wb').write(html) wp.tree = wp._get_parse_tree(html) for img in wp.tree.xpath('.//img'): src = img.attrib['src'] frags = src.split('/') if len(frags) > 1: fn = None for title in [frags[-2], frags[-1]]: title = urlparse.unquote( title.encode('utf-8')).decode('utf-8') fn = item.wiki.env.images.getDiskPath(title) if fn: correct, new_fn = img_ext_correct(fn) if not correct: os.rename(fn, new_fn) fn = new_fn fn = limit_size(img, fn) wp.images[src] = fn break if not fn and title not in missing_images: print 'image not found %r' % src missing_images.append(title) else: if not img.get('class') == 'tex': # skip math formulas _extract_license_info(coll, item.wiki.env.images, title, license_checker) if num_items > config.max_parsetree_num: del wp.tree coll.append(wp) if status_callback: status_callback(progress=n * progress_inc) return coll
except Exception, e: print 'ERROR posting progress %r to %r' % (progress, posturl) try: if options.logfile: utils.start_logging(options.logfile) output = options.output from mwlib import wiki, recorddb, metabook mb = metabook.MetaBook() if conf: from ConfigParser import ConfigParser w = wiki.makewiki(conf) cp = ConfigParser() cp.read(conf) license = {'name': cp.get('wiki', 'defaultarticlelicense')} if license['name'] is not None: license['wikitext'] = w['wiki'].getRawArticle(license['name']) mb.source = { 'name': cp.get('wiki', 'name'), 'url': cp.get('wiki', 'url'), 'defaultarticlelicense': license, } else: w = { 'wiki': wiki.wiki_mwapi(baseurl, options.license, options.template_blacklist),
print "name_count_avg length" for h in l: row = h.name.encode("utf-8", "ignore") + "_" row += str(h.count) + "_" row += str(h.length / h.count) print row if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--processes', '-p', type=int, default=1) parser.add_argument('--max', '-m', type=int, help="dont process more than this many pages") args = parser.parse_args() env = wiki.makewiki(wcb.paths["wikiconf"]) templr = template.create_actions(env, wcb.paths["templaterules"], wcb.paths["templatecache"]) noder = node.read_rules(wcb.paths["noderules"]) log.logger.info("Listing articles") names = multiprocessing.Queue() if not args.max: args.max = len(env.wiki.reader.keys()) i = 0 for n in util.articles(): if i < args.max: names.put(n) i += 1
def coll_from_zip(basedir, env, status_callback=None): def img_ext_correct(fn): from PIL import Image img = Image.open(fn) fmt = '.' + img.format.lower() name, ext = os.path.splitext(fn) ext = ext.lower() if ext == '.jpg': ext = '.jpeg' if fmt != ext: return (False, name + fmt) else: return (True, fn) if isinstance(env, basestring): from mwlib import wiki env = wiki.makewiki(env) coll = Collection(basedir=basedir, title=env.metabook.title or '', subtitle=env.metabook.subtitle or '', editor=env.metabook.editor or '', language=env.wiki.siteinfo.get('general',{}).get('lang', 'en') ) missing_images = [] num_items = len(env.metabook.walk()) progress_inc = 100.0/num_items license_checker = LicenseChecker(image_db=env.images, filter_type='blacklist') license_checker.readLicensesCSV() for n, item in enumerate(env.metabook.walk()): if item.type == 'chapter': chapter = Chapter(item.title) coll.append(chapter) continue elif item.type == 'custom': # a "custom" item currently can be the preface added at pediapress.com # FIXME: support custom item continue title = item.title if isinstance(title, str): title = unicode(title, 'utf-8') url = item.wiki.getURL(title, item.revision) if isinstance(url, str): url = unicode(url, 'utf-8') data = item.wiki.getHTML(title, item.revision) try: html = data['text']['*'] except KeyError: print 'WARNING: article missing, skipping %r' % item.title continue if isinstance(html, str): html = unicode(html, 'utf-8') html = '<div id="content"><h1>%s</h1>\n\n%s</div>' % (title.encode('utf-8'), html.encode('utf-8')) wp = WebPage(coll, title, url, user_agent='Mozilla/5.0', contributors=env.wiki.getAuthors(title=item.title, revision=item.revision) ) # images wp.canonical_url = urlparse.urljoin(item._env.wiki.siteinfo['general']['base'], urllib2.quote(title.replace(' ', '_').encode('utf-8')).decode('utf-8')) open(wp.get_path('content.orig'), 'wb').write(html) wp.tree = wp._get_parse_tree(html) for img in wp.tree.xpath('.//img'): src = img.attrib['src'] frags = src.split('/') if len(frags)>1: fn = None for title in [frags[-2], frags[-1]]: title = urlparse.unquote(title.encode('utf-8')).decode('utf-8') fn = item.wiki.env.images.getDiskPath(title) if fn: correct, new_fn = img_ext_correct(fn) if not correct: os.rename(fn, new_fn) fn = new_fn fn = limit_size(img, fn) wp.images[src] = fn break if not fn and title not in missing_images: print 'image not found %r' % src missing_images.append(title) else: if not img.get('class') == 'tex': # skip math formulas _extract_license_info(coll, item.wiki.env.images, title, license_checker) if num_items > config.max_parsetree_num: del wp.tree coll.append(wp) if status_callback: status_callback(progress=n*progress_inc) return coll
def setup_method(self, method): print "reading", self.zip_filename self.env = wiki.makewiki(self.zip_filename) self.wikidb = self.env.wiki self.imagedb = self.env.images
return ret if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--file', action='store_true') parser.add_argument('--extra-newlines', action='store_true') parser.add_argument('--keep-empty', action='store_true') parser.add_argument('article') args = parser.parse_args() env = wiki.makewiki(paths.paths["wikiconf"]) act = template.create_actions(env, paths.paths["templaterules"], paths.paths["templatecache"]) elementrules = node.read_rules(paths.paths["noderules"]) purifier = Purifier(env, act, elementrules) purifier.extra_newlines = args.extra_newlines purifier.keep_empty = args.keep_empty raw = """ This text is above all the headings == Template tests == Pagename: {{PAGENAME}} "Coor title dm" expands to "Coor dm" should be kept<br /> {{coor title dm|11|22|33|N|44|55|E|:city}}
# from mwlib import wiki, parser from mwlib.templ.scanner import symbols, tokenize import argparse import codecs parser = argparse.ArgumentParser(description='Prints the tokens generated in the template expansion phase') parser.add_argument('wikiconf') parser.add_argument('article') args = parser.parse_args() env = wiki.makewiki(args.wikiconf) try: args.article = unicode(args.article) except UnicodeError as ex: print "warning: ", print ex args.article = unicode(args.article,errors="ignore") if env: markup = env.wiki.nuwiki.get_page(args.article) if markup: for t in tokenize(markup.rawtext, included=False): print t else: print 'Could not find article "' + args.article + '"'
def makewiki(self): username, password, domain = None, None, None if self.options.login: if self.options.login.count(':') == 1: username, password = unicode(self.options.login, 'utf-8').split(':', 1) else: username, password, domain = unicode(self.options.login, 'utf-8').split(':', 2) if self.options.script_extension: script_extension = unicode(self.options.script_extension, 'utf-8') else: script_extension = None env = wiki.makewiki(self.options.config, metabook=self.metabook, username=username, password=password, domain=domain, script_extension=script_extension, ) if self.options.noimages: env.images = None if self.options.template_blacklist: template_blacklist = unicode(self.options.template_blacklist, 'utf-8') else: template_blacklist = None if self.options.template_exclusion_category: template_exclusion_category = unicode(self.options.template_exclusion_category, 'utf-8') else: template_exclusion_category = None if self.options.print_template_pattern: print_template_pattern = unicode(self.options.print_template_pattern, 'utf-8') else: print_template_pattern = None if self.options.print_template_prefix: if print_template_pattern is not None: log.warn('Both --print-template-pattern and --print-template-prefix (deprecated) specified. Using --print-template-pattern only.') else: print_template_pattern = '%s$1' % unicode(self.options.print_template_prefix, 'utf-8') if template_blacklist\ or template_exclusion_category\ or print_template_pattern: if hasattr(env.wiki, 'setTemplateExclusion'): env.wiki.setTemplateExclusion( blacklist=template_blacklist, category=template_exclusion_category, pattern=print_template_pattern, ) else: log.warn('WikiDB does not support setting a template blacklist') if self.options.collectionpage: wikitext = env.wiki.getRawArticle(unicode(self.options.collectionpage, 'utf-8')) if wikitext is None: raise RuntimeError('No such collection page: %r' % ( self.options.collectionpage, )) self.metabook = metabook.parse_collection_page(wikitext) env.metabook = self.metabook if self.options.title: env.metabook['title'] = unicode(self.options.title, 'utf-8') if self.options.subtitle: env.metabook['subtitle'] = unicode(self.options.subtitle, 'utf-8') if self.options.editor: env.metabook['editor'] = unicode(self.options.editor, 'utf-8') return env
except Exception, e: print 'ERROR posting progress %r to %r' % (progress, posturl) try: if options.logfile: utils.start_logging(options.logfile) output = options.output from mwlib import wiki, recorddb, metabook mb = metabook.MetaBook() if conf: from ConfigParser import ConfigParser w = wiki.makewiki(conf) cp = ConfigParser() cp.read(conf) license = { 'name': cp.get('wiki', 'defaultarticlelicense') } if license['name'] is not None: license['wikitext'] = w['wiki'].getRawArticle(license['name']) mb.source = { 'name': cp.get('wiki', 'name'), 'url': cp.get('wiki', 'url'), 'defaultarticlelicense': license, } else: w = { 'wiki': wiki.wiki_mwapi(baseurl, options.license, options.template_blacklist),