def prepare_db(inputs, dbfile): """ Build a SQLite database that extracts some information from our parsed versions of Wiktionary. This is information that is needed by later reader steps, such as which words are known in which languages, and which words are forms of other words. """ # If the database already exists, delete it first try: os.unlink(dbfile) except FileNotFoundError: pass db = sqlite3.connect(dbfile) make_tables(db) try: for filename in inputs: filepath = pathlib.Path(filename) file_language = filepath.name.split('.')[0] for item in read_json_stream(filename): if 'rel' in item: tfrom = item['from'] tto = item['to'] # For all non-definition relations, record the fact that # the given entry name exists in the given language. We'll # use these to disambiguate definitions later. if item['rel'] != 'definition': if 'language' in tfrom and valid_language(tfrom['language']): add_title( db, file_language, tfrom['language'], tfrom['text'] ) if 'language' in tto and valid_language(tto['language']): add_title(db, file_language, tto['language'], tto['text']) # Record word forms so we can build a lemmatizer from them. if item['rel'].startswith('form/'): form_name = item['rel'][5:] # Look for the part of speech, first in the 'from' term, # then in the 'to' term. pos = tfrom.get('pos', tto.get('pos', '?')) # Use only Etymology 1 entries for learning word forms. if (tfrom.get('etym') or '1') == '1': language = tfrom.get('language', tto.get('language')) if ( valid_language(language) and tfrom['text'] != tto['text'] ): add_form( db, file_language, language, tfrom['text'], pos, tto['text'], form_name, ) db.commit() finally: db.close()
def prepare_db(inputs, dbfile): """ Build a SQLite database that extracts some information from our parsed versions of Wiktionary. This is information that is needed by later reader steps, such as which words are known in which languages, and which words are forms of other words. """ # If the database already exists, delete it first try: os.unlink(dbfile) except FileNotFoundError: pass db = sqlite3.connect(dbfile) make_tables(db) try: for filename in inputs: filepath = pathlib.Path(filename) file_language = filepath.name.split('.')[0] for item in read_json_stream(filename): if 'rel' in item: tfrom = item['from'] tto = item['to'] # For all non-definition relations, record the fact that # the given entry name exists in the given language. We'll # use these to disambiguate definitions later. if item['rel'] != 'definition': if 'language' in tfrom and valid_language(tfrom['language']): add_title( db, file_language, tfrom['language'], tfrom['text'] ) if 'language' in tto and valid_language(tto['language']): add_title(db, file_language, tto['language'], tto['text']) # Record word forms so we can build a lemmatizer from them. if item['rel'].startswith('form/'): form_name = item['rel'][5:] # Look for the part of speech, first in the 'from' term, # then in the 'to' term. pos = tfrom.get('pos', tto.get('pos', '?')) # Use only Etymology 1 entries for learning word forms. if (tfrom.get('etym') or '1') == '1': language = tfrom.get('language', tto.get('language')) if ( valid_language(language) and tfrom['text'] != tto['text'] ): add_form( db, file_language, language, tfrom['text'], pos, tto['text'], form_name, ) db.commit() finally: db.close()
def transform_term(data_language, termdata, assumed_languages, db, use_etyms=True): text = termdata['text'] # Sometimes - is used to fill a slot in a Wiktionary template where the # term would usually be. It typically means "don't show this part", with # the implication "the term in question is obvious from context". # # Context is hard, so let's just cope with a hyphen as the term by # discarding it. if text == '-': return None language = termdata.get('language') if language is None: language = disambiguate_language(text, assumed_languages, db) if not valid_language(language): return None # Remove unnecessary subtags from the Wiktionary language if '-' in language and language not in ALL_LANGUAGES: language = language.split('-')[0] if 'pos' not in termdata: return standardized_concept_uri(language, text) else: pos = termdata['pos'] etym_sense = None if use_etyms: etym_sense = etym_label(data_language, termdata) if etym_sense is not None: return standardized_concept_uri(language, text, pos, 'wikt', etym_sense) else: return standardized_concept_uri(language, text, pos)
def transform_term(data_language, termdata, assumed_languages, db, use_etyms=True): text = termdata['text'] # Sometimes - is used to fill a slot in a Wiktionary template where the # term would usually be. It typically means "don't show this part", with # the implication "the term in question is obvious from context". # # Context is hard, so let's just cope with a hyphen as the term by # discarding it. if text == '-': return None language = termdata.get('language') if language is None: language = disambiguate_language(text, assumed_languages, db) if not valid_language(language): return None # Remove unnecessary subtags from the Wiktionary language if '-' in language and language not in ALL_LANGUAGES: language = language.split('-')[0] if 'pos' not in termdata: return standardized_concept_uri(language, text) else: pos = termdata['pos'] etym_sense = None if use_etyms: etym_sense = etym_label(data_language, termdata) if etym_sense is not None: return standardized_concept_uri(language, text, pos, 'wikt', etym_sense) else: return standardized_concept_uri(language, text, pos)
def read_wiktionary(input_file, db_file, output_file): """ Convert a stream of parsed Wiktionary data into ConceptNet edges. A `db_file` containing all known words in all languages must have already been prepared from the same data. """ db = sqlite3.connect(db_file) out = MsgpackStreamWriter(output_file) for heading, items in segmented_stream(input_file): language = heading['language'] title = heading['title'] dataset = '/d/wiktionary/{}'.format(language) url_title = heading['title'].replace(' ', '_') web_url = 'http://{}.wiktionary.org/wiki/{}'.format( language, url_title) web_source = '/s/resource/wiktionary/{}'.format(language) source = {'contributor': web_source, 'process': PARSER_RULE} # Scan through the 'from' items, such as the start nodes of # translations, looking for distinct etymologies. If we get more than # one etymology for a language, we need to distinguish them as # different senses in that language. all_etyms = { (item['from']['language'], etym_label(language, item['from'])) for item in items if 'language' in item['from'] and item['from']['text'] == title and etym_label(language, item['from']) is not None } word_languages = {wlang for (wlang, _) in all_etyms} for wlang in sorted(word_languages): if valid_language(wlang): cpage = standardized_concept_uri(wlang, title) ld_edge = make_edge('/r/ExternalURL', cpage, web_url, dataset=dataset, weight=0.25, sources=[source], license=Licenses.cc_sharealike) out.write(ld_edge) etym_to_translation_sense = {} language_etym_counts = Counter(lang for (lang, etym) in all_etyms) polysemous_languages = { lang for lang in language_etym_counts if language_etym_counts[lang] > 1 } for item in items: tfrom = item['from'] tto = item['to'] assumed_languages = [language] lang1 = tfrom.get('language') lang2 = tto.get('language') if lang1 and (lang1 not in assumed_languages) and valid_language(lang1): assumed_languages.append(lang1) if lang2 and (lang2 not in assumed_languages) and valid_language(lang2): assumed_languages.append(lang2) cfrom = transform_term(language, tfrom, assumed_languages, db, use_etyms=(lang1 in polysemous_languages)) cpage = cfrom cto = transform_term(language, tto, assumed_languages, db, use_etyms=(lang2 in polysemous_languages)) if cfrom is None or cto is None: continue if uri_prefix(cfrom, 3) == uri_prefix(cto, 3): continue rel, switch = transform_relation(item['rel']) if rel is None: continue if switch: cfrom, cto = cto, cfrom # When translations are separated by sense, use only the first # sense we see for each etymology. That will have the most # representative translations. if item['rel'] == 'translation': etym_key = (tfrom['language'], etym_label(language, tfrom)) sense = tfrom.get('sense', '') if etym_key in etym_to_translation_sense: if etym_to_translation_sense[etym_key] != sense: continue else: etym_to_translation_sense[etym_key] = sense weight = 1. if rel == '/r/EtymologicallyRelatedTo': weight = 0.25 edge = make_edge(rel, cfrom, cto, dataset=dataset, weight=weight, sources=[source], surfaceStart=tfrom['text'], surfaceEnd=tto['text'], license=Licenses.cc_sharealike) out.write(edge) out.close()
def read_wiktionary(input_file, db_file, output_file): """ Convert a stream of parsed Wiktionary data into ConceptNet edges. A `db_file` containing all known words in all languages must have already been prepared from the same data. """ db = sqlite3.connect(db_file) out = MsgpackStreamWriter(output_file) for heading, items in segmented_stream(input_file): language = heading['language'] title = heading['title'] dataset = '/d/wiktionary/{}'.format(language) url_title = heading['title'].replace(' ', '_') web_url = 'http://{}.wiktionary.org/wiki/{}'.format(language, url_title) web_source = '/s/resource/wiktionary/{}'.format(language) source = {'contributor': web_source, 'process': PARSER_RULE} # Scan through the 'from' items, such as the start nodes of # translations, looking for distinct etymologies. If we get more than # one etymology for a language, we need to distinguish them as # different senses in that language. all_etyms = { (item['from']['language'], etym_label(language, item['from'])) for item in items if 'language' in item['from'] and item['from']['text'] == title and etym_label(language, item['from']) is not None } word_languages = {wlang for (wlang, _) in all_etyms} for wlang in sorted(word_languages): if valid_language(wlang): cpage = standardized_concept_uri(wlang, title) ld_edge = make_edge( '/r/ExternalURL', cpage, web_url, dataset=dataset, weight=0.25, sources=[source], license=Licenses.cc_sharealike, ) out.write(ld_edge) etym_to_translation_sense = {} language_etym_counts = Counter(lang for (lang, etym) in all_etyms) polysemous_languages = { lang for lang in language_etym_counts if language_etym_counts[lang] > 1 } for item in items: tfrom = item['from'] tto = item['to'] assumed_languages = [language] lang1 = tfrom.get('language') lang2 = tto.get('language') if lang1 and (lang1 not in assumed_languages) and valid_language(lang1): assumed_languages.append(lang1) if lang2 and (lang2 not in assumed_languages) and valid_language(lang2): assumed_languages.append(lang2) cfrom = transform_term( language, tfrom, assumed_languages, db, use_etyms=(lang1 in polysemous_languages), ) cpage = cfrom cto = transform_term( language, tto, assumed_languages, db, use_etyms=(lang2 in polysemous_languages), ) if cfrom is None or cto is None: continue if uri_prefix(cfrom, 3) == uri_prefix(cto, 3): continue rel, switch = transform_relation(item['rel']) if rel is None: continue if switch: cfrom, cto = cto, cfrom # When translations are separated by sense, use only the first # sense we see for each etymology. That will have the most # representative translations. if item['rel'] == 'translation': etym_key = (tfrom['language'], etym_label(language, tfrom)) sense = tfrom.get('sense', '') if etym_key in etym_to_translation_sense: if etym_to_translation_sense[etym_key] != sense: continue else: etym_to_translation_sense[etym_key] = sense weight = 1. if rel == '/r/EtymologicallyRelatedTo': weight = 0.25 edge = make_edge( rel, cfrom, cto, dataset=dataset, weight=weight, sources=[source], surfaceStart=tfrom['text'], surfaceEnd=tto['text'], license=Licenses.cc_sharealike, ) out.write(edge) out.close()