def get_siteinfo(self):
if self.siteinfo is not None:
return self.siteinfo
from mwlib.siteinfo import get_siteinfo
if self.sources:
self.siteinfo = get_siteinfo(self.sources.values()[0].language or "en")
else:
self.siteinfo = get_siteinfo('en')
return self.siteinfo
def get_nshandler_for_lang(lang):
if lang is None:
lang = "de" # FIXME: we currently need this to make the tests happy
# assert lang is not None, "expected some language"
from mwlib import siteinfo
si = siteinfo.get_siteinfo(lang)
if si is None:
si = siteinfo.get_siteinfo("en")
assert si, "siteinfo-en not found"
return nshandler(si)
def get_nshandler_for_lang(lang):
if lang is None:
lang = "de" # FIXME: we currently need this to make the tests happy
# assert lang is not None, "expected some language"
from mwlib import siteinfo
si = siteinfo.get_siteinfo(lang)
if si is None:
si = siteinfo.get_siteinfo("en")
assert si, "siteinfo-en not found"
return nshandler(si)
def __init__(self, txt, included=True, replace_tags=None, siteinfo=None):
if isinstance(txt, str):
txt = unicode(txt)
self.txt = txt
self.included = included
self.replace_tags = replace_tags
if siteinfo is None:
from mwlib.siteinfo import get_siteinfo
siteinfo = get_siteinfo("en")
self.siteinfo = siteinfo
self.name2rx = {"if": re.compile("^#if:"),
"switch": re.compile("^#switch:")}
magicwords = self.siteinfo.get("magicwords", [])
for d in magicwords:
name = d["name"]
if name in ("if", "switch"):
aliases = [re.escape(x) for x in d["aliases"]]
rx = "^#(%s):" % ("|".join(aliases),)
self.name2rx[name] = re.compile(rx)
# print name, rx
self.aliasmap = aliasmap(self.siteinfo)
def __init__(self, txt, included=True, replace_tags=None, siteinfo=None):
if isinstance(txt, str):
txt = unicode(txt)
self.txt = txt
self.included = included
self.replace_tags = replace_tags
if siteinfo is None:
from mwlib.siteinfo import get_siteinfo
siteinfo = get_siteinfo("en")
self.siteinfo = siteinfo
self.name2rx = {
"if": re.compile("^#if:"),
"switch": re.compile("^#switch:")
}
magicwords = self.siteinfo.get("magicwords", [])
for d in magicwords:
name = d["name"]
if name in ("if", "switch"):
aliases = [re.escape(x) for x in d["aliases"]]
rx = "^#(%s):" % ("|".join(aliases), )
self.name2rx[name] = re.compile(rx)
# print name, rx
self.aliasmap = aliasmap(self.siteinfo)
def fix_wikipedia_siteinfo(siteinfo):
# --- http://code.pediapress.com/wiki/ticket/754
if u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd' in [
x.get("prefix", u"")[2:] for x in siteinfo.get("interwikimap", [])
]:
print "WARNING: interwikimap contains garbage"
from mwlib import siteinfo as simod
en = simod.get_siteinfo("en")
siteinfo['interwikimap'] = list(en["interwikimap"])
prefixes = [x['prefix'] for x in siteinfo['interwikimap']]
for p in "pnb ckb mwl mhr ace krc pcd frr koi gag bjn pfl mrj bjn rue kbd ltg xmf".split(
):
if p in prefixes:
return
siteinfo['interwikimap'].append({
'prefix':
p,
'language':
p,
'url':
'http://%s.wikipedia.org/wiki/$1' % (p, ),
'local':
'',
})
class Expander(object):
magic_displaytitle = None # set via {{DISPLAYTITLE:...}}
def __init__(self, txt, pagename="", wikidb=None, recursion_limit=100):
assert wikidb is not None, "must supply wikidb argument in Expander.__init__"
self.pagename = pagename
self.db = wikidb
self.uniquifier = Uniquifier()
si = None
try:
si = self.db.get_siteinfo()
except Exception, err:
print 'Caught: %s' % err
if si is None:
print "WARNING: failed to get siteinfo from %r" % (self.db, )
si = siteinfo.get_siteinfo("de")
self.nshandler = nshandler = nshandling.nshandler(si)
self.siteinfo = si
if self.db and hasattr(self.db, "getSource"):
source = self.db.getSource(pagename) or metabook.source()
local_values = source.locals or u""
local_values = mwlocals.parse_locals(local_values)
else:
local_values = None
source = {}
# XXX we really should call Expander with a nuwiki.page object.
revisionid = 0
if self.db and hasattr(self.db, "nuwiki") and pagename:
page = self.db.nuwiki.get_page(self.pagename)
if page is not None:
revisionid = getattr(page, 'revid', 0) or 0
self.resolver = magics.MagicResolver(pagename=pagename,
revisionid=revisionid)
self.resolver.siteinfo = si
self.resolver.nshandler = nshandler
self.resolver.wikidb = wikidb
self.resolver.local_values = local_values
self.resolver.source = source
self.recursion_limit = recursion_limit
self.recursion_count = 0
self.aliasmap = parser.aliasmap(self.siteinfo)
self.parsed = parser.parse(txt,
included=False,
replace_tags=self.replace_tags,
siteinfo=self.siteinfo)
# show(self.parsed)
self.parsedTemplateCache = {}
def __init__(self, dir, prefix='wiki', lang="en"):
self.redirects = {}
self.dir = os.path.abspath(dir)
self.reader = ZCdbReader(os.path.join(self.dir, prefix))
# FIXME: guess language from xml namespace information???
self.siteinfo = siteinfo.get_siteinfo(lang)
if self.siteinfo is None:
raise RuntimeError("could not get siteinfo for language %r" % (lang,))
self.nshandler = nshandling.nshandler(self.siteinfo)
self.nfo = dict(base_url="http://%s.wikipedia.org/w/" % (lang, ), # FIXME
script_extension = ".php",
) # FIXME
self.redirect_matcher = self.nshandler.redirect_matcher
def __init__(self, dir, prefix='wiki', lang="en"):
self.redirects = {}
self.dir = os.path.abspath(dir)
self.reader = ZCdbReader(os.path.join(self.dir, prefix))
# FIXME: guess language from xml namespace information???
self.siteinfo = siteinfo.get_siteinfo(lang)
if self.siteinfo is None:
raise RuntimeError("could not get siteinfo for language %r" % (lang,))
self.nshandler = nshandling.nshandler(self.siteinfo)
self.nfo = dict(base_url="http://%s.wikipedia.org/w/" % (lang, ), # FIXME
script_extension = ".php",
) # FIXME
self.redirect_matcher = self.nshandler.redirect_matcher
def __init__(self, *args, **kw):
if args:
self.d, = args
else:
self.d = {}
self.d.update(kw)
normd = {}
for k, v in self.d.items():
normd[k.lower().replace(" ", "_")] = v
self.d = normd
self.siteinfo = get_siteinfo('de')
def __init__(self, *args, **kw):
if args:
self.d, = args
else:
self.d = {}
self.d.update(kw)
normd = {}
for k, v in self.d.items():
normd[k.lower().replace(" ", "_")] = v
self.d = normd
self.siteinfo = get_siteinfo('de')
def NS(self, args):
"""Returns the name of a given namespace number."""
try:
namespaces = self.siteinfo["namespaces"]
except (AttributeError, KeyError):
from mwlib import siteinfo
namespaces = siteinfo.get_siteinfo("en")["namespaces"]
ns = args[0]
try:
retval = namespaces[ns]['*']
except KeyError:
retval = ''
return retval
def NS(self, args):
"""Returns the name of a given namespace number."""
try:
namespaces = self.siteinfo["namespaces"]
except (AttributeError, KeyError):
from mwlib import siteinfo
namespaces = siteinfo.get_siteinfo("en")["namespaces"]
ns = args[0]
try:
retval = namespaces[ns]['*']
except KeyError:
retval = ''
return retval
def fix_wikipedia_siteinfo(siteinfo):
# --- http://code.pediapress.com/wiki/ticket/754
if u'\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd' in [
x.get("prefix", u"")[2:] for x in siteinfo.get("interwikimap", [])]:
print "WARNING: interwikimap contains garbage"
from mwlib import siteinfo as simod
en = simod.get_siteinfo("en")
siteinfo['interwikimap'] = list(en["interwikimap"])
prefixes = [x['prefix'] for x in siteinfo['interwikimap']]
for p in "pnb ckb mwl mhr ace krc pcd frr koi gag bjn pfl mrj bjn rue kbd ltg xmf".split():
if p in prefixes:
return
siteinfo['interwikimap'].append({
'prefix': p,
'language': p,
'url': 'http://%s.wikipedia.org/wiki/$1' % (p, ),
'local': '',
})
def fix_wikipedia_siteinfo(siteinfo):
# --- http://code.pediapress.com/wiki/ticket/754
if (
u"\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd"
in [x.get("prefix", u"")[2:] for x in siteinfo.get("interwikimap", [])]
):
print "WARNING: interwikimap contains garbage"
from mwlib import siteinfo as simod
en = simod.get_siteinfo("en")
siteinfo["interwikimap"] = list(en["interwikimap"])
prefixes = [x["prefix"] for x in siteinfo["interwikimap"]]
for p in "pnb ckb mwl mhr ace krc pcd frr koi gag bjn pfl mrj bjn rue".split():
if p in prefixes:
return
siteinfo["interwikimap"].append(
{"prefix": p, "language": p, "url": "http://%s.wikipedia.org/wiki/$1" % (p,), "local": ""}
)
def get_siteinfo(self):
from mwlib import siteinfo
return siteinfo.get_siteinfo("en")
#! /usr/bin/env py.test
from mwlib import nshandling, siteinfo
siteinfo_de = siteinfo.get_siteinfo("de")
assert siteinfo_de, "cannot find german siteinfo"
def test_fqname():
def get_fqname(name, expected):
fqname = nshandler.get_fqname(name)
print "%r -> %r" % (name, fqname)
assert fqname == expected
nshandler = nshandling.nshandler(siteinfo_de)
d = get_fqname
e = "Benutzer:Schmir"
yield d, "User:Schmir", e
yield d, "user:Schmir", e
yield d, "benutzer:schmir", e
yield d, " user: schmir ", e
yield d, "___user___:___schmir __", e
yield d, "User:SchmiR", "Benutzer:SchmiR"
def test_fqname_defaultns():
def get_fqname(name, expected):
fqname = nshandler.get_fqname(name, 10) # Vorlage
print "%r -> %r" % (name, fqname)
assert fqname == expected
def get_siteinfo(self):
from mwlib import siteinfo
return siteinfo.get_siteinfo("en")
#! /usr/bin/env py.test
import copy
from mwlib.templ import nodes, parser
from mwlib.siteinfo import get_siteinfo
si = copy.deepcopy(get_siteinfo("en"))
del si["magicwords"]
def test_aliasmap_no_magicwords():
parser.aliasmap(si)
def test_parser_no_magicwords():
p = parser.Parser(u"some text", siteinfo=si)
p.parse()
def __init__(self, lang="en"):
from mwlib import siteinfo
self.siteinfo = siteinfo.get_siteinfo(lang)
#! /usr/bin/env py.test
from mwlib import nshandling, siteinfo
siteinfo_de = siteinfo.get_siteinfo("de")
assert siteinfo_de, "cannot find german siteinfo"
def test_fqname():
def get_fqname(name, expected):
fqname = nshandler.get_fqname(name)
print "%r -> %r" % (name, fqname)
assert fqname == expected
nshandler = nshandling.nshandler(siteinfo_de)
d = get_fqname
e = "Benutzer:Schmir"
yield d, "User:Schmir", e
yield d, "user:Schmir", e
yield d, "benutzer:schmir", e
yield d, " user: schmir ", e
yield d, "___user___:___schmir __", e
yield d, "User:SchmiR", "Benutzer:SchmiR"
def test_fqname_defaultns():
def get_fqname(name, expected):
fqname = nshandler.get_fqname(name, 10) # Vorlage
print "%r -> %r" % (name, fqname)
assert fqname == expected
def __init__(self, lang="en"):
from mwlib import siteinfo
self.siteinfo = siteinfo.get_siteinfo(lang)
def get_siteinfo(self):
return siteinfo.get_siteinfo("en")
#! /usr/bin/env py.test
from mwlib.expander import parse
from mwlib.siteinfo import get_siteinfo
from mwlib.templ import nodes, magic_nodes
nl_siteinfo = get_siteinfo("nl")
def test_no_arguments():
t = parse(u"{{bla}}")
assert t[1] == (), "expected an empty tuple"
def test_one_empty_arguments():
t = parse(u"{{bla|}}")
assert len(t[1]) == 1, "expected exactly one argument"
def test_parse_if():
t = parse(u"{{#if: 1 | yes | no}}")
print t
assert isinstance(t, nodes.IfNode)
t = parse(u"{{#if: 1 | yes | no}}", siteinfo=nl_siteinfo)
print t
assert isinstance(t, nodes.IfNode)
def test_parse_if_localized():
t = parse(u"{{#als: 1 | yes | no}}", siteinfo=nl_siteinfo)
