def run_data(self, item, word="testpage", lang="English", field="related", ruby="", sense=None, senses=[], ctx=None, is_reconstruction=False): """Runs a test where we expect the parsing to return None. This function returns ``data``.""" assert isinstance(item, str) assert isinstance(word, str) assert isinstance(lang, str) assert isinstance(field, str) assert isinstance(ruby, str) assert sense is None or isinstance(sense, str) assert isinstance(senses, list) assert ctx is None or isinstance(ctx, Wtp) ctx1 = ctx if ctx is not None else Wtp() self.ctx = ctx1 self.config = WiktionaryConfig() self.ctx.start_page(word) self.ctx.start_section(lang) data = {} ret = parse_linkage_item_text(self.ctx, word, data, field, item, sense, ruby, senses, is_reconstruction) self.assertIs(ret, None) if ctx is None: self.assertEqual(self.ctx.errors, []) self.assertEqual(self.ctx.warnings, []) self.assertEqual(self.ctx.debugs, []) return data
def test_gender5(self): # Numeric inflection classes should only be interpreted for certain # languages (e.g., Bantu languages) ctx = Wtp() # To allow debug messages data = self.run_data("foo 1", lang="Swedish", ctx=ctx) self.assertEqual(data, {"related": [ {"word": "foo 1"}]})
def test_gender13(self): # They should not be interpreted for other languages ctx = Wtp() # To allow debug messages data = self.run_data("foo 1 or 2", lang="English", ctx=ctx) self.assertEqual(data, {"related": [ {"word": "foo 1"}, {"word": "2"}, ]})
def test_long(self): # Just parse through the data and make sure that we find some words # This takes about 0.5 minutes. langs = collections.defaultdict(int) words = collections.defaultdict(int) poses = collections.defaultdict(int) num_transl = 0 num_redirects = 0 def word_cb(data): nonlocal num_transl nonlocal num_redirects if "redirect" in data: assert isinstance(data["redirect"], str) word = data["title"] words[word] += 1 num_redirects += 1 return word = data["word"] assert word words[word] += 1 lang = data["lang"] pos = data["pos"] assert word and lang and pos langs[lang] += 1 poses[pos] += 1 if data.get("translations"): num_transl += 1 for sense in data.get("senses", ()): if sense.get("translations"): num_transl += 1 path = "tests/test-pages-articles.xml.bz2" print("Parsing test data") ctx = Wtp() config = WiktionaryConfig(capture_languages=[ "English", "Finnish", "Spanish", "German", "Chinese", "Japanese", "Italian", "Portuguese", "Translingual" ], capture_translations=True, capture_pronunciation=True, capture_linkages=True, capture_compounds=True, capture_redirects=True) parse_wiktionary(ctx, path, config, word_cb, None) print("Test data parsing complete") assert num_redirects > 0 assert len(words) > 100 assert all(x < 50 for x in words.values()) assert langs["English"] > 0 assert langs["Finnish"] > 0 assert langs["Translingual"] > 0 assert len(langs.keys()) == 9 assert len(poses.keys()) <= len(wiktextract.PARTS_OF_SPEECH) assert sum(poses.values()) == sum(langs.values()) assert sum(words.values()) == sum(poses.values()) + num_redirects assert num_transl > 0
def test_gender15(self): # inclusive or/English/Translations ctx = Wtp() # To allow debug messages data = self.run_data("μη αποκλειστικό or n (mi apokleistikó or)", lang="Greek", word="inclusive or", ctx=ctx) self.assertEqual(data, {"related": [ {"word": "μη αποκλειστικό or", "tags": ["neuter"], "roman": "mi apokleistikó or"}, ]})
def parse_with_ctx(title, text, **kwargs): assert isinstance(title, str) assert isinstance(text, str) ctx = Wtp() ctx.analyze_templates() ctx.start_page(title) root = ctx.parse(text, **kwargs) print("parse_with_ctx: root", type(root), root) return root, ctx
def test_prefix16(self): # Triggers an error due to invalid gloss reference ctx = Wtp() data = self.run_data("(4): foo", ctx=ctx, senses=[ {"glosses": ["sense1"]}, {"glosses": ["sense2", "sense2b"]}, {"glosses": ["sense3"]}]) self.assertEqual(data, {"related": [ {"word": "foo"}]}) self.assertNotEqual(ctx.debugs, [])
def setUp(self): self.maxDiff = 20000 self.ctx = Wtp() self.ctx.analyze_templates() self.ctx.start_page("testpage") self.config = WiktionaryConfig(capture_languages=None, capture_translations=True, capture_pronunciation=True, capture_linkages=True, capture_compounds=True, capture_redirects=True, capture_examples=True)
def runonce(self, num_threads): # Just parse through the data and make sure that we find some words path = "tests/test-pages-articles.xml.bz2" print("Parsing test data") ctx = Wtp(num_threads=num_threads) ret = ctx.process(path, page_cb) titles = collections.defaultdict(int) redirects = collections.defaultdict(int) for title, redirect_to in ret: titles[title] += 1 if redirect_to is not None: redirects[redirect_to] += 1 print("Test data parsing complete") assert sum(redirects.values()) > 0 assert len(titles) > 100 assert all(x == 1 for x in titles.values()) assert len(redirects) > 1
def setUp(self): self.maxDiff = 100000 self.ctx = Wtp() self.config = WiktionaryConfig() self.ctx.start_page("testpage") self.ctx.start_section("English")
def setUp(self): self.ctx = Wtp() self.config = WiktionaryConfig() self.ctx.start_page("testpage")
def test_prefix18(self): # Triggers error due to invalid prefix ctx = Wtp() data = self.run_data("dsafjdasfkldjas: foo", ctx=ctx) self.assertEqual(data, {"related": [{"word": "foo"}]}) self.assertNotEqual(ctx.debugs, [])
truncate_title = False if lst[-1] in ("A", "B", "C", "D", "E", "F", "I", "II", "III", "IV", "V"): truncate_title = False if not (len(lst) > 1 and any(x[0].isupper() for x in lst[1:]) and (len(lst[-1]) != 1 or lst[-1] not in ("virus",))): truncate_title = False if truncate_title: for i in range(1, len(lst)): if lst[i][0].isupper(): title = " ".join(lst[:i]) break return title ctx = Wtp(cache_file="species-cache") # Disable this for later runs to avoid recreating the cache. Makes developing # the code MUCH faster. Remove the cache file before reading the dump. # Read pages from the dump file into the cache file (Phase 1) #list(ctx.process(dumpfile, page_handler, phase1_only=True)) # Process the pages in the dump file. ret = list(ctx.reprocess(page_handler)) print("Count distinct titles:", len(set(ret))) firsts = set(x.split()[0] for x in ret if x.find("virus") < 0 and x.find("satellite") < 0 and x.find("viroid") < 0) print("Count distinct first words:", len(firsts)) words = set(x for title in ret for x in title.lower().split()) print("Count distinct latter:", len(words))
if node.kind not in (NodeKind.LEVEL2, NodeKind.LEVEL3, NodeKind.LEVEL4, NodeKind.LEVEL5, NodeKind.LEVEL6): continue if (len(node.args) != 1 or len(node.args[0]) != 1 or not isinstance(node.args[0][0], str)): print(" {} - {}: {}".format(title, node.kind, node.children)) continue t = node.args[0][0] assert isinstance(t, str) print(" {} - {}".format(title, t)) titles.append(t) sys.stdout.flush() return title, titles, ctx.errors ctx = Wtp() ret = ctx.process(path, page_handler) counts = collections.defaultdict(int) titles_ht = collections.defaultdict(list) for page_title, titles, errors in ret: for title in titles: titles_ht[title].append(page_title) for err in errors: msg = err["msg"] counts[msg] += 1 print("=== MOST COMMON ERRORS") errors = list(sorted(counts.items(), key=lambda x: x[1], reverse=True)) for err, cnt in errors[:40]: print(cnt, err)
def setUp(self): self.maxDiff = 20000 self.ctx = Wtp() self.config = WiktionaryConfig() self.ctx.start_page("abolitionism") # Note: some tests use last char self.ctx.start_section("English")