def main(): import argparse argparser = argparse.ArgumentParser(description="Find -ismo without -ista and vice versa") argparser.add_argument("file", help="Extract file to read") argparser.add_argument("--allforms", help="Allforms for checking lemmas") argparser.add_argument("--save", help="Save to wiktionary with specified commit message") argparser.add_argument("--date", help="Date of the database dump (used to generate page messages)") argparser.add_argument("--limit", type=int, help="Limit processing to first N articles") argparser.add_argument("--progress", help="Display progress", action='store_true') args = argparser.parse_args() count = 0 allforms = AllForms.from_file(args.allforms) if args.allforms else None all_lemmas = set(allforms.all_lemmas) for article in WikiExtract.iter_articles_from_bz2(args.file): if not count % 1000 and args.progress: print(count, end = '\r', file=sys.stderr) if args.limit and count >= args.limit: break count += 1 text = article.text path = article.title.split(":") page = path[0] pos = path[-1] if page.endswith("ismo"): error = "ismo_without_ista" search = page[:-4] + "ista" elif page.endswith("ista"): error = "ista_without_ismo" search = page[:-4] + "ismo" else: continue if page not in all_lemmas: continue if search in all_lemmas and search not in article.text: log(error, page) if args.save: base_url = "User:JeffDoozan/lists" logger.save(base_url, WikiSaver, commit_message=args.save) else: dest = "" logger.save(dest, FileSaver)
def main(): import argparse argparser = argparse.ArgumentParser( description="Find lemmas with only 'form of' senses") argparser.add_argument("--trans", help="Extract file to read") argparser.add_argument("--allforms", help="Allforms for resolving forms to lemmas") argparser.add_argument( "--save", help="Save to wiktionary with specified commit message") argparser.add_argument( "--date", help="Date of the database dump (used to generate page messages)") argparser.add_argument("--limit", type=int, help="Limit processing to first N articles") argparser.add_argument("--progress", help="Display progress", action='store_true') argparser.add_argument("--dump-aliases", help="Dump likely language aliases", action='store_true') argparser.add_argument("--dump-parents", help="Dump likely parent languages", action='store_true') args = argparser.parse_args() allforms = AllForms.from_file(args.allforms) if args.allforms else None if not os.path.isfile(args.trans): raise FileNotFoundError(f"Cannot open: {args.trans}") fixer = T9nFixer(allforms) logger = Logger() def log(error, page, pos, gloss, language, line="", highlight=""): if error is None: raise ValueError("error is none") if page is None: raise ValueError("page is none") if pos is None: raise ValueError("pos is none") if gloss is None: gloss = "" if language is None: language = "" if line is None: line = "" if highlight is None: highlight = "" logger.add(error, page, pos, gloss, language, line, highlight) # if language: # langlogger.add(error, page, pos, gloss, language, line, highlight) # if error != "text_outside_template": # logger.add(error, page, pos, gloss, line, highlight) count = 0 max_val = 0 pages_with_tables = set() for article in WikiExtract.iter_articles_from_bz2(args.trans): text = article.text path = article.title.split(":") page = path[0] pos = path[-1] if pos not in ALL_POS: log("outside_pos", page, pos, None, None, path) count += 1 if not count % 1000 and args.progress: print(count, end='\r', file=sys.stderr) if args.limit and count > args.limit: break # if page != "pie-eyed": # continue # if pathstr != "veggie:English:Adjective": # if pathstr != "I love you:English:Phrase": # continue # print("\n", count) # val = timeit.timeit(lambda: list(TranslationTable.find_tables(text, page, pos)), number=1) # if val > max_val: # max_val = val # max_page = pathstr # continue tables = list(TranslationTable.find_tables(text)) if not len(tables) and not re.search( "{{\s*(trans-see|checktrans|see translation)", text): log("no_tables", page, pos, None, None) # max_page = "X" pages_with_tables.add(page) stats["sections_with_tables"] += 1 for table_lines in tables: table_lines = table_lines.splitlines() # print(table_lines) # exit() # max_val += len(table_lines) # continue table = TranslationTable(page, pos, table_lines, log_function=log) stats["total_tables"] += 1 seen = set() for item in table.items: if isinstance(item, TranslationLine) and item.lang_id not in seen: stats["total_entries"] += len(item.entries) stats["lang_entries"][lang_ids[item.lang_id]] += 1 seen.add(item.lang_id ) # Don't count more than one entry per table if len(tables) > 1 and not table.gloss and table.template in [ "tran-top", "trans-top-see", "trans-top-also" ]: table.log("no_gloss") fixer.cleanup_table(table) # if "\n".join(map(str.strip,table_lines)) != str(table): # table.log("botfix_formatting") # print("OLD", page, pos, file=sys.stderr) # print("\n".join(table_lines), file=sys.stderr) # print("NEW", page, pos) # print(str(table)) #exit() stats["pages_with_tables"] = len(pages_with_tables) # print(max_val, max_page) # base_url = "User:JeffDoozan/lists/translations" if args.save else "Xtranslations" # langlogger.save(base_url, args.save) if args.save: base_url = "User:JeffDoozan/lists/translations" logger.save(base_url, WikiByLanguage, commit_message=args.save, page_limit=1000, data_date=args.date) logger.save(base_url + "/by_error", WikiByError, commit_message=args.save, data_date=args.date) else: dest = "Xtranslations" logger.save(dest, FileByLanguage, page_limit=1000, data_date=args.date) logger.save(dest + "/by_error", FileByError, data_date=args.date) # Dump nested language aliases if args.dump_aliases: print("language_aliases = {") #for lang,codes in sorted(UNKNOWN_LANGS.items(), key=lambda x: sum(x[1].values())*-1): for lang, codes in sorted(UNKNOWN_LANGS.items()): for code, count in sorted(codes.items(), key=lambda x: x[1] * -1): if count > 20: print( f" '{lang}': '{lang_ids[code]}', # {code} found in {count} entries" ) break print("}") if args.dump_parents: print("language_parents = {") for lang, count in sorted(LANG_PARENTS.items()): if count > 20: print(f" '{lang}', # used in {count} entries") print("}") colons = [x for x in lang_ids.values() if ":" in x] if colons: raise ValueError( "A language exists with a colon in the name, this may cause problems for nested languages that use : as a separator" ) print(f"Total pages with tables: {stats['pages_with_tables']}") print(f"Total sections with tables: {stats['sections_with_tables']}") total_lines = sum(stats["lang_entries"].values()) print(f"Total language lines in tables: {total_lines}") print(f"Total translation entries: {stats['total_entries']}")
if not args.data_dir: args.data_dir = os.environ.get("SPANISH_DATA_DIR", "spanish_data") if not args.custom_dir: args.custom_dir = os.environ.get("SPANISH_CUSTOM_DIR", "spanish_custom") with open(args.dictionary) as wordlist_data: cache_words = not args.low_mem wordlist = Wordlist(wordlist_data, cache_words=cache_words) print("wordlist", mem_use(), file=sys.stderr) ignore_data = open(args.ignore) if args.ignore else [] if args.allforms: allforms = AllForms.from_file(args.allforms) else: allforms = AllForms.from_wordlist(wordlist) print("all_forms", mem_use(), file=sys.stderr) sentences = spanish_sentences( sentences=args.sentences, data_dir=args.data_dir, custom_dir=args.custom_dir ) flist = FrequencyList(wordlist, allforms, sentences) with open(args.file) as infile: for line in flist.process(infile, ignore_data): print(line) ignore_data.close()
def allforms(self): if not self._allforms: self._allforms = AllForms.from_file(self.allforms_file) return self._allforms
def main(): global args global ngprobs parser = argparse.ArgumentParser(description="Summarize ngram usage") parser.add_argument( "--allforms", help="Exclude coordinate terms that have standalone entries") parser.add_argument("--min-count", help="Ignore forms with less than N uses", type=int) parser.add_argument( "--min-percent", help= "Ignore coordinate terms used less than N percent of the form's uses", type=int) parser.add_argument( "--save", help="Save to wiktionary with specified commit message") parser.add_argument( "--ignore2", help= "Ignore coords containing the specified word (can be used more than once)", action='append') parser.add_argument("--ngprobs", help="Ngram probability data file") parser.add_argument( "--coord2", help="File containing 2 word coordinate terms to check") parser.add_argument( "--coord3", help="File containing 3 word coordinate terms to check") parser.add_argument( "--coord4", help="File containing 4 word coordinate terms to check") args = parser.parse_args() allforms = AllForms.from_file(args.allforms) all_forms = set(allforms.all_forms) print("all_forms") lemma_forms = defaultdict(list) for form, pos, lemma in allforms.all: lemma_forms[(lemma, pos)].append(form) print("lemma_forms") alt_case = { form.lower(): form for form in all_forms if form != form.lower() } ngprobs = NgramPosProbability(args.ngprobs) if False: coord = "reharás tu vida" words = coord.split(" ") print([coord, get_coord_lemma(ngprobs, allforms, words)]) exit() form = "fijamente" form_pos = ngprobs.get_preferred_pos(form) form_lemma = get_form_lemma(ngprobs, allforms, form) count = get_lemma_count(ngprobs, lemma_forms, form_lemma, form_pos) print([form, form_pos, form_lemma, count]) exit() all_coords = {} if args.coord2: all_coords |= find_coords(allforms, all_forms, ngprobs, alt_case, args.coord2, args.ignore2) if args.coord3: all_coords |= find_coords(allforms, all_forms, ngprobs, alt_case, args.coord3) if args.coord4: all_coords |= find_coords(allforms, all_forms, ngprobs, alt_case, args.coord4) # seen2 = set() # all_coords = {} # if args.coord3: # coord_lemmas = find_coords(allforms, all_forms, ngprobs, alt_case, args.coord3) # for k in coord_lemmas.keys(): # coord_lemma, form, form_pos = k # words = coord_lemma.split(" ") # seen2.add(words[0:2]) # seen2.add(words[1:3]) # all_coords |= coord_lemmas # # if args.coord2: # coord_lemmas = find_coords(allforms, all_forms, ngprobs, alt_case, args.coord2, args.ignore2) # for k,count in coord_lemmas.items(): # coord_lemma, form, form_pos = k # if coord_lemma in seen2 and coord_lemma not in all_forms: # continue # all_coords[k] = count for k, coord_count in all_coords.items(): coord_lemma, form_lemma, form_pos = k form_count = get_lemma_count(ngprobs, lemma_forms, form_lemma, form_pos) # Skip uncommon forms if form_count < args.min_count: continue # Min ratio percent = int(coord_count / form_count * 100) if percent < args.min_percent: continue existing = coord_lemma if coord_lemma in all_forms else coord_lemma.lower( ) if coord_lemma.lower() in all_forms else None if existing: coord_lemma = f"[[{existing}]]" log(form_lemma, coord_lemma, form_count, coord_count, percent) if args.save: base_url = "User:JeffDoozan/lists" logger.save(base_url, WikiSaver, commit_message=args.save) else: dest = "" logger.save(dest, FileSaver)
def main(): global fixer global fixrunner import argparse parser = argparse.ArgumentParser( description="Generate list of missing forms") parser.add_argument("wordlist", help="wordlist") parser.add_argument("--allforms", required=True, help="all_forms file") parser.add_argument("--allpages", required=True, help="wiki.allpages") parser.add_argument( "--articles", required=True, help="Language extract with raw articles, used for checking autofixes") parser.add_argument("--save", help="wiktionary commit message") parser.add_argument("--limit", type=int, help="Limit processing to first N articles") parser.add_argument("--progress", help="Display progress", action='store_true') args = parser.parse_args() global ARTICLE_FILE ARTICLE_FILE = args.articles wordlist = Wordlist.from_file(args.wordlist) allforms = AllForms.from_file(args.allforms) fixer = FormFixer(wordlist) fixrunner = FixRunner("es", wordlist, allforms) with open(args.allpages) as infile: # Loading the entire contents of allpages takes 600M # To conserve memory, temporarily load allforms into a set # and then create a set of entries in allpages that are also in allforms allforms_set = set(allforms.all_forms) allpages = {x.strip() for x in infile if x in allforms_set} del allforms_set # form = "achaparrándolo" # declared_forms = fixer.get_declared_forms(form, wordlist, allforms) # existing_forms = get_existing_forms(form, wordlist) # missing_forms, unexpected_forms = fixer.compare_forms(declared_forms, existing_forms) # print("declared", declared_forms) # print("existing", existing_forms) # print("missing", missing_forms) # print("unexpected", unexpected_forms) # exit() count = 0 for form in allforms.all_forms: # Fix for conversion from <sup>x</sup> -> ^x if "^" in form: continue try: declared_forms = fixer.get_declared_forms(form, wordlist, allforms) except ValueError as e: print("ERROR", e) #error("form_errors", form, str(e)) continue if not count % 1000 and args.progress: print(count, end='\r', file=sys.stderr) if args.limit and count >= args.limit: break count += 1 existing_forms = get_existing_forms(form, wordlist) missing_forms, unexpected_forms = fixer.compare_forms( declared_forms, existing_forms) missing_pos = [] for item in missing_forms: if item.form != form: raise ValueError(form, item) if not FormFixer.can_handle_formtype(item.formtype): continue # TODO: for now skip multi word verbs if item.pos == "v" and " " in item.lemma: continue if item.pos == "n" and item.formtype == "m": error("should_be_lemma", form, item) continue words = list(wordlist.get_words(form, item.pos)) if not words: matches = list(wordlist.get_words(form)) if matches: if item.pos in missing_pos: continue ety = {w.etymology for w in matches} level = 4 if len(ety) > 1 else 3 # error("missing_pos_multi_ety", form, item) items = [i for i in missing_forms if i.pos == item.pos] if fixer.can_handle(item): pos_text = "\n".join( fixer.full_pos(item.form, level, items)) else: pos_text = "" error("missing_pos", form, item, pos_text) missing_pos.append(item.pos) else: if form in allpages: error("missing_entry", form, item) continue # if pos == "n" and formtype == "pl" and unexpected_forms: # masculines = get_masculines_from_fpl(words[0]) # masculine_links = [m for m in masculines if (pos, "fpl", m) in unexpected_forms] # if masculine_links: # for m in masculine_links: # unexpected_forms.remove((pos, "fpl", m)) # print(f"{form}:{pos} links to masculine {masculine_links} instead of feminine $is_doublet") # continue error("missing_sense", form, item) for item in sorted(unexpected_forms): words = list(wordlist.get_words(item.lemma, item.pos)) if words: error("unexpected_form", form, item) else: error("missing_lemma", form, item) if args.save: base_url = "User:JeffDoozan/lists/es/forms" logger.save(base_url, WikiSaver, commit_message=args.save) else: logger.save("forms", FileSaver)