def run_single_process(): writer = MultiWriter('conceptnet4_nadya') raw_assertions = RawAssertion.objects.filter() for raw_assertion in raw_assertions: edges = handle_raw_assertion(raw_assertion) for edge in edges: writer.write(edge)
def sum_assertions(file_index): weights = defaultdict(float) assertions = {} ccby = defaultdict(bool) for line in codecs.open(CURRENT_DIR +'/data/temp/core_'+str(file_index)+'.txt', 'r','utf-8'): uri, rel, start, end, context, weight, sources, id, dataset = line.split('\t')[:9] if uri != 'uri' and context == '/ctx/all': weight = float(weight) weights[uri] += float(weight) assertions[uri] = (rel, start, end, context, weights[uri]) if not (dataset.startswith('/d/reverb') or dataset.startswith('/d/wiktionary') or dataset.startswith('/d/dbpedia')): ccby[uri] = True writer_core = MultiWriter('assertion_totals_core') #writer_sa = MultiWriter('assertion_totals_sa') for uri, values in assertions.iteritems(): relation, start, end, context, weight = values if ccby[uri]: license = '/l/CC/By' dataset = '/d/conceptnet/5/combined-core' else: license = '/l/CC/By-SA' dataset = '/d/conceptnet/5/combined-sa' edge = make_edge(relation, start, end, dataset, license, ['/s/rule/sum_edges'], '/ctx/all', weight=weight) if license == '/l/CC/By': writer_core.write(edge) #else: #writer_sa.write(edge) writer_core.close()
def run_single_process(): writer = MultiWriter('conceptnet4_nadya') raw_assertions = RawAssertion.objects.filter() for raw_assertion in raw_assertions: edges = handle_raw_assertion(raw_assertion) for edge in edges: writer.write(edge)
def run_single_process(): writer = MultiWriter("conceptnet4_nadya") path = "./raw_data/" for filename in os.listdir(path): for raw_assertion in codecs.open(path + filename, encoding="utf-8", errors="replace"): edges = handle_raw_flat_assertion(raw_assertion) for edge in edges: writer.write(edge)
def run_single_process(): writer = MultiWriter('conceptnet4') path = "./raw_data/" for filename in os.listdir(path): for raw_assertion in codecs.open(path + filename, encoding='utf-8', errors='replace'): edges = handle_raw_assertion(raw_assertion) for edge in edges: writer.write(edge)
def run_single_process(): writer = MultiWriter('conceptnet4_zh') path = "./raw_data/" for filename in os.listdir(path): for line in codecs.open(path + filename, encoding='utf-8', errors='replace'): aggregate_assertion(line) for assertion, users in assertion_map.items(): edges = handle_raw_assertion((assertion, users)) for edge in edges: writer.write(edge)
def run_single_process(): writer = MultiWriter('conceptnet4') path = "./raw_data/" for filename in os.listdir(path): for raw_assertion in codecs.open(path + filename, encoding='utf-8', errors='replace'): edges = handle_raw_assertion(raw_assertion) for edge in edges: writer.write(edge)
def run_single_process(): writer = MultiWriter('conceptnet4_zh') path = "./raw_data/" for filename in os.listdir(path): for line in codecs.open(path + filename, encoding='utf-8', errors='replace'): aggregate_assertion(line) for assertion, users in assertion_map.items(): edges = handle_raw_assertion((assertion, users)) for edge in edges: writer.write(edge)
def build_core_from_csvs(csv_files): weights = defaultdict(float) assertions = {} ccby = defaultdict(bool) for csv_file in csv_files: print "currently in file: " + str(csv_file) for line in codecs.open(csv_file, encoding='utf-8'): uri, rel, start, end, context, weight, sources, id, dataset = line.split( '\t')[:9] if uri != 'uri' and context == '/ctx/all': weight = float(weight) weights[uri] += float(weight) assertions[uri] = (rel, start, end, context, weights[uri]) if not (dataset.startswith('/d/reverb') or dataset.startswith('/d/wiktionary') or dataset.startswith('/d/dbpedia')): ccby[uri] = True print 'writing' writer_core = MultiWriter('assertion_totals_core') #writer_sa = MultiWriter('assertion_totals_sa') for uri, values in assertions.iteritems(): relation, start, end, context, weight = values if ccby[uri]: license = '/l/CC/By' dataset = '/d/conceptnet/5/combined-core' else: license = '/l/CC/By-SA' dataset = '/d/conceptnet/5/combined-sa' edge = make_edge(relation, start, end, dataset, license, ['/s/rule/sum_edges'], '/ctx/all', weight=weight) if license == '/l/CC/By': writer_core.write(edge) #else: #writer_sa.write(edge) writer_core.close()
def sum_assertions(file_index): weights = defaultdict(float) assertions = {} ccby = defaultdict(bool) for line in codecs.open( CURRENT_DIR + '/data/temp/core_' + str(file_index) + '.txt', 'r', 'utf-8'): uri, rel, start, end, context, weight, sources, id, dataset = line.split( '\t')[:9] if uri != 'uri' and context == '/ctx/all': weight = float(weight) weights[uri] += float(weight) assertions[uri] = (rel, start, end, context, weights[uri]) if not (dataset.startswith('/d/reverb') or dataset.startswith('/d/wiktionary') or dataset.startswith('/d/dbpedia')): ccby[uri] = True writer_core = MultiWriter('assertion_totals_core') #writer_sa = MultiWriter('assertion_totals_sa') for uri, values in assertions.iteritems(): relation, start, end, context, weight = values if ccby[uri]: license = '/l/CC/By' dataset = '/d/conceptnet/5/combined-core' else: license = '/l/CC/By-SA' dataset = '/d/conceptnet/5/combined-sa' edge = make_edge(relation, start, end, dataset, license, ['/s/rule/sum_edges'], '/ctx/all', weight=weight) if license == '/l/CC/By': writer_core.write(edge) #else: #writer_sa.write(edge) writer_core.close()
def build_core_from_csvs(csv_files): weights = defaultdict(float) assertions = {} ccby = defaultdict(bool) for csv_file in csv_files: print "currently in file: " + str(csv_file) for line in codecs.open(csv_file, encoding='utf-8'): uri, rel, start, end, context, weight, sources, id, dataset = line.split('\t')[:9] if uri != 'uri' and context == '/ctx/all': weight = float(weight) weights[uri] += float(weight) assertions[uri] = (rel, start, end, context, weights[uri]) if not (dataset.startswith('/d/reverb') or dataset.startswith('/d/wiktionary') or dataset.startswith('/d/dbpedia')): ccby[uri] = True print 'writing' writer_core = MultiWriter('assertion_totals_core') #writer_sa = MultiWriter('assertion_totals_sa') for uri, values in assertions.iteritems(): relation, start, end, context, weight = values if ccby[uri]: license = '/l/CC/By' dataset = '/d/conceptnet/5/combined-core' else: license = '/l/CC/By-SA' dataset = '/d/conceptnet/5/combined-sa' edge = make_edge(relation, start, end, dataset, license, ['/s/rule/sum_edges'], '/ctx/all', weight=weight) if license == '/l/CC/By': writer_core.write(edge) #else: #writer_sa.write(edge) writer_core.close()
class FindTranslations(ContentHandler): def __init__(self): self.lang = None self.langcode = None self.inArticle = False self.inTitle = False self.curSense = None self.curTitle = '' self.curText = '' self.locales = [] self.curRelation = None self.writer = MultiWriter('wiktionary') self.trans = False # in translation mode def startElement(self, name, attrs): if name == 'page': self.inArticle = True self.curText = [] elif name == 'title': self.inTitle = True self.curTitle = '' def endElement(self, name): if name == 'page': self.inArticle = False self.handleArticle(self.curTitle, ''.join(self.curText)) elif name == 'title': self.inTitle = False def characters(self, text): if self.inTitle: self.curTitle += text elif self.inArticle: self.curText.append(text) if len(self.curText) > 10000: # bail out self.inArticle = False def handleArticle(self, title, text): lines = text.split('\n') self.pos = None for line in lines: self.handleLine(title, line.strip()) def handleLine(self, title, line): language_match = LANGUAGE_HEADER.match(line) trans_top_match = TRANS_TOP.match(line) trans_bottom_match = TRANS_BOTTOM.match(line) trans_tag_match = TRANS_TAG.search(line) chinese_match = CHINESE_TAG.search(line) ### Get translation if trans_top_match: # start translation part self.trans = True if self.trans and trans_bottom_match: # end translation part self.trans = False if self.trans and line.startswith('*{{'): # get translation lang = line[3:5] # get language of translation # find all translations of that language translations = re.findall(u"\{\{Ü.*?\|.*?\|(.*?)\}\}", line) for translation in translations: # iterate over translations self.output_sense_translation(lang, translation, title, '') ### Get relation if line.startswith('{{Synonyme}}'): # synonym self.curRelation = 'synonym' elif line.startswith(u'{{Gegenwörter}}'): # antonym self.curRelation = 'antonym' elif line.startswith('{{Oberbegriffe}}'): # hypernym self.curRelation = 'hypernym' elif line.startswith('{{Unterbegriffe}}'): # hyponym self.curRelation = 'hyponym' elif line.startswith('{{Redewendungen}}'): # idiom self.curRelation = 'idiom' elif line.startswith('{{Charakteristische Wortkombinationen}}'): \ # word combination self.curRelation = 'word combination' elif line.startswith('{{Wortbildungen}}'): # morphology self.curRelation = 'morphology' if self.curRelation and line == '': # end relation self.curRelation = None if self.curRelation: related_words_or_phrases = re.findall(r"\[\[(.*?)\]\]", line) for related_word in related_words_or_phrases: self.output_monolingual('deu', self.curRelation, \ related_word, title) def output_monolingual(self, lang, relation, term1, term2): if 'Wik' in term1 or 'Wik' in term2: return source = make_concept_uri(term1, lang) if self.pos: target = make_concept_uri(term2, lang, self.pos) else: target = make_concept_uri(term2, lang) surfaceText = "[[%s]] %s [[%s]]" % (term1, relation, term2) #print surfaceText edge = make_edge('/r/'+relation, source, target, '/d/wiktionary/%s/%s' % (lang, lang), license='/l/CC/By-SA', sources=[SOURCE, MONOLINGUAL], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge) def output_sense_translation(self, lang, foreign, german, disambiguation): if 'Wik' in foreign or 'Wik' in german: return if lang == 'zh-cn': lang = 'zh_CN' elif lang == 'zh-tw': lang = 'zh_TW' source = make_concept_uri( unicodedata.normalize('NFKC', foreign), lang ) target = make_concept_uri( german, 'de', disambiguation ) relation = '/r/TranslationOf' try: surfaceRel = "is %s for" % (langs.english_name(lang)) except KeyError: surfaceRel = "is [language %s] for" % lang surfaceText = "[[%s]] %s [[%s (%s)]]" % (foreign, surfaceRel, english, disambiguation.split('/')[-1].replace('_', ' ')) #print surfaceText edge = make_edge(relation, source, target, '/d/wiktionary/en/%s' % lang, license='/l/CC/By-SA', sources=[SOURCE, TRANSLATE], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge) def output_translation(self, foreign, english, locale=''): source = make_concept_uri( unicodedata.normalize('NFKC', foreign), self.langcode+locale ) target = make_concept_uri( english, 'en' ) relation = '/r/TranslationOf' try: surfaceRel = "is %s for" % (langs.english_name(self.langcode)) except KeyError: surfaceRel = "is [language %s] for" % self.langcode surfaceText = "[[%s]] %s [[%s]]" % (foreign, surfaceRel, english) edge = make_edge(relation, source, target, '/d/wiktionary/en/%s' % self.langcode, license='/l/CC/By-SA', sources=[SOURCE, INTERLINGUAL], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge)
score = (freq*2-1) * (1000-orderscore) * (1-sls) / 1000 if score <= 0: counts['low score'] += 1 weak_out.write(line) continue count += 1 counts['success'] += 1 good_out.write(line) if make_json: edge = make_edge(rel, left, right, '/d/verbosity', '/l/CC/By', sources, surfaceText=text, weight = score/10.0) writer.write(edge) if make_json: writer.close() flag_out.close() good_out.close() weak_out.close() similar_out.close() simout = open('data/output/similarity-scores.txt', 'w') for sim in text_similarities: print >> simout, sim simout.close()
class FindTranslations(ContentHandler): def __init__(self): self.lang = None self.langcode = None self.inArticle = False self.inTitle = False self.curSense = None self.curTitle = '' self.curText = '' self.locales = [] self.curRelation = None self.writer = MultiWriter('wiktionary') def startElement(self, name, attrs): if name == 'page': self.inArticle = True self.curText = [] elif name == 'title': self.inTitle = True self.curTitle = '' def endElement(self, name): if name == 'page': self.inArticle = False self.handleArticle(self.curTitle, ''.join(self.curText)) elif name == 'title': self.inTitle = False def characters(self, text): if self.inTitle: self.curTitle += text elif self.inArticle: self.curText.append(text) if len(self.curText) > 10000: # bail out self.inArticle = False def handleArticle(self, title, text): lines = text.split('\n') self.pos = None for line in lines: self.handleLine(title, line.strip()) def handleLine(self, title, line): language_match = LANGUAGE_HEADER.match(line) trans_top_match = TRANS_TOP.match(line) trans_tag_match = TRANS_TAG.search(line) chinese_match = CHINESE_TAG.search(line) if line.startswith('===') and line.endswith('==='): pos = line.strip('= ') if pos == 'Synonyms': self.curRelation = 'Synonym' elif pos == 'Antonym': self.curRelation = 'Antonym' elif pos == 'Related terms': self.curRelation = 'ConceptuallyRelatedTo' elif pos == 'Derived terms': if not line.startswith('===='): # this is at the same level as the part of speech; # now we don't know what POS these apply to self.pos = None self.curRelation = 'DerivedFrom' else: self.curRelation = None if pos in PARTS_OF_SPEECH: self.pos = PARTS_OF_SPEECH[pos] elif language_match: self.lang = language_match.group(1) self.langcode = LANGUAGES.get(self.lang) elif chinese_match: scripttag = chinese_match.group(2) self.locales = [] if 's' in scripttag: self.locales.append('_CN') if 't' in scripttag: self.locales.append('_TW') elif line[0:1] == '#' and self.lang != 'English' and self.lang is not None: defn = line[1:].strip() if defn[0:1] not in ':*#': for defn2 in filter_line(defn): if not ascii_enough(defn2): continue if 'Index:' in title: continue if self.langcode == 'zh': for locale in self.locales: self.output_translation(title, defn2, locale) elif self.langcode: self.output_translation(title, defn2) elif line[0:4] == '----': self.pos = None self.lang = None self.langcode = None self.curRelation = None elif trans_top_match: pos = self.pos or 'n' sense = trans_top_match.group(1).split(';')[0].strip('.') if 'translations' in sense.lower(): self.curSense = None else: self.curSense = pos+'/'+sense elif trans_tag_match: lang = trans_tag_match.group(1) translation = trans_tag_match.group(2) if self.curSense is not None and self.lang == 'English': # handle Chinese separately if lang not in ('cmn', 'yue', 'zh-yue', 'zh'): self.output_sense_translation(lang, translation, title, self.curSense) elif '{{trans-bottom}}' in line: self.curSense = None elif line.startswith('* ') and self.curRelation and self.langcode: relatedmatch = WIKILINK.search(line) if relatedmatch: related = relatedmatch.group(1) self.output_monolingual(self.langcode, self.curRelation, related, title) def output_monolingual(self, lang, relation, term1, term2): if 'Wik' in term1 or 'Wik' in term2: return source = make_concept_uri(term1, lang) if self.pos: target = make_concept_uri(term2, lang, self.pos) else: target = make_concept_uri(term2, lang) surfaceText = "[[%s]] %s [[%s]]" % (term1, relation, term2) #print surfaceText edge = make_edge('/r/'+relation, source, target, '/d/wiktionary/%s/%s' % (lang, lang), license='/l/CC/By-SA', sources=[SOURCE, MONOLINGUAL], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge) def output_sense_translation(self, lang, foreign, english, disambiguation): if 'Wik' in foreign or 'Wik' in english: return if lang == 'zh-cn': lang = 'zh_CN' elif lang == 'zh-tw': lang = 'zh_TW' source = make_concept_uri( unicodedata.normalize('NFKC', foreign), lang ) target = make_concept_uri( english, 'en', disambiguation ) relation = '/r/TranslationOf' try: surfaceRel = "is %s for" % (langs.english_name(lang)) except KeyError: surfaceRel = "is [language %s] for" % lang surfaceText = "[[%s]] %s [[%s (%s)]]" % (foreign, surfaceRel, english, disambiguation.split('/')[-1].replace('_', ' ')) #print surfaceText edge = make_edge(relation, source, target, '/d/wiktionary/en/%s' % lang, license='/l/CC/By-SA', sources=[SOURCE, TRANSLATE], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge) def output_translation(self, foreign, english, locale=''): source = make_concept_uri( unicodedata.normalize('NFKC', foreign), self.langcode+locale ) target = make_concept_uri( english, 'en' ) relation = '/r/TranslationOf' try: surfaceRel = "is %s for" % (langs.english_name(self.langcode)) except KeyError: surfaceRel = "is [language %s] for" % self.langcode surfaceText = "[[%s]] %s [[%s]]" % (foreign, surfaceRel, english) edge = make_edge(relation, source, target, '/d/wiktionary/en/%s' % self.langcode, license='/l/CC/By-SA', sources=[SOURCE, INTERLINGUAL], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge)
class FindTranslations(ContentHandler): def __init__(self): self.lang = None self.langcode = None self.inArticle = False self.inTitle = False self.curSense = None self.curTitle = '' self.curText = '' self.locales = [] self.curRelation = None self.writer = MultiWriter('wiktionary_ja') self.nosensetrans = None # non-sense-specific translation def startElement(self, name, attrs): if name == 'page': self.inArticle = True self.curText = [] elif name == 'title': self.inTitle = True self.curTitle = '' def endElement(self, name): if name == 'page': self.inArticle = False self.handleArticle(self.curTitle, ''.join(self.curText)) elif name == 'title': self.inTitle = False def characters(self, text): if self.inTitle: self.curTitle += text elif self.inArticle: self.curText.append(text) if len(self.curText) > 10000: # bail out self.inArticle = False def handleArticle(self, title, text): lines = text.split('\n') self.pos = None for line in lines: self.handleLine(title, line.strip()) def handleLine(self, title, line): language_match = LANGUAGE_HEADER.match(line) trans_top_match = TRANS_TOP.match(line) trans_bottom_match = TRANS_BOTTOM.match(line) trans_match = TRANS.match(line) trans_tag_match = TRANS_TAG.search(line) chinese_match = CHINESE_TAG.search(line) if language_match: self.langcode = get_language_code(language_match.group(1)) ### Get sense-specific translation if trans_top_match: # start translation part pos = self.pos or 'n' # get translation sense if trans_top_match.group(1): sense = trans_top_match.group(1).lstrip('|') self.curSense = pos+'/'+sense return else: self.curSense = pos return if trans_bottom_match: # end translation part self.curSense = None return if self.curSense and line[0:5] == '*[[{{': # get translation lang = line[5:].split('}')[0] # get language of translation if lang in LANGUAGES_3_TO_2: # convert 3-letter code to 2-letter code lang = LANGUAGES_3_TO_2[lang] # find all translations of that language translations = re.findall(r"\[\[(.*?)\]\]", line)[1:] for translation in translations: # iterate over translations self.output_sense_translation(lang, translation, title, \ self.curSense) return ### Get relation if line.startswith('===={{rel}}===='): # start relation part self.curRelation = 'ConceptuallyRelatedTo' return if self.curRelation and self.langcode: # within relation part if line.startswith('*'): # get relation relations = re.findall(r"\{\{(.*?)\}\}", line) if len(relations) > 0: if relations[0] == 'syn': # synonym self.curRelation = 'Synonym' if relations[0] == 'drv': # derivative self.curRelation = 'Derivative' related_words = re.findall(r"\[\[(.*?)\]\]", line) for related_word in related_words: self.output_monolingual(self.langcode, self.curRelation, \ related_word, title) self.curRelation = 'ConceptuallyRelatedTo' # back to default else: self.curRelation = None ### Get non-sense-specific translation if trans_match: self.nosensetrans = 1 # *maybe* start non-sense-specific translation if self.nosensetrans == 1 and line.startswith('{{top}}'): self.nosensetrans = 2 # start non-sense-specific translation if self.nosensetrans == 2: if line.startswith('{{bottom}}'): self.nosensetrans = None return if line.startswith('*{{'): lang = line[3:].split('}')[0] if lang in LANGUAGES_3_TO_2: # convert 3-letter code to 2-letter code lang = LANGUAGES_3_TO_2[lang] translations = re.findall(r"\[\[(.*?)\]\]", line) for translation in translations: self.output_sense_translation(lang, translation, title, '') def output_monolingual(self, lang, relation, term1, term2): # skip Wiktionary: links and templates if u'ウィク' in term1 or u'ウィク' in term2: return if u'テンプレート' in term1 or u'テンプレート' in term2: return if lang in LANGUAGES_3_TO_2: # convert 3-letter code to 2-letter code lang = LANGUAGES_3_TO_2[lang] source = make_concept_uri_safe(term1, lang) if self.pos: target = make_concept_uri_safe(term2, lang, self.pos) else: target = make_concept_uri_safe(term2, lang) surfaceText = "[[%s]] %s [[%s]]" % (term1, relation, term2) #print surfaceText edge = make_edge('/r/'+relation, source, target, '/d/wiktionary/ja/%s' % (lang), license='/l/CC/By-SA', sources=[SOURCE, MONOLINGUAL], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge) def output_sense_translation(self, lang, foreign, translated, disambiguation): if u':' in foreign or u':' in translated: return if lang == 'zh-cn': lang = 'zh_CN' elif lang == 'zh-tw': lang = 'zh_TW' source = make_concept_uri_safe( unicodedata.normalize('NFKC', foreign), lang ) target = make_concept_uri_safe( translated, self.langcode, disambiguation ) relation = '/r/TranslationOf' try: surfaceRel = "is %s for" % (langs.english_name(lang)) except KeyError: surfaceRel = "is [language %s] for" % lang if disambiguation and '/' in disambiguation: surfaceText = "[[%s]] %s [[%s (%s)]]" % (foreign, surfaceRel, translated, disambiguation.split('/')[-1].replace('_', ' ')) else: surfaceText = "[[%s]] %s [[%s]]" % (foreign, surfaceRel, translated) #print surfaceText edge = make_edge(relation, source, target, '/d/wiktionary/ja/%s' % (self.langcode), license='/l/CC/By-SA', sources=[SOURCE, TRANSLATE], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge) def output_translation(self, foreign, japanese, locale=''): source = make_concept_uri_safe( unicodedata.normalize('NFKC', foreign), self.langcode+locale ) target = make_concept_uri_safe( japanese, 'ja' ) relation = '/r/TranslationOf' try: surfaceRel = "is %s for" % (langs.english_name(self.langcode)) except KeyError: surfaceRel = "is [language %s] for" % self.langcode surfaceText = "[[%s]] %s [[%s]]" % (foreign, surfaceRel, japanese) edge = make_edge(relation, source, target, '/d/wiktionary/ja/%s' % self.langcode, license='/l/CC/By-SA', sources=[SOURCE, INTERLINGUAL], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge)
class FindTranslations(ContentHandler): def __init__(self): self.lang = None self.langcode = None self.inArticle = False self.inTitle = False self.curSense = None self.curTitle = "" self.curText = "" self.locales = [] self.curRelation = None self.writer = MultiWriter("wiktionary") self.trans = False # in translation mode def startElement(self, name, attrs): if name == "page": self.inArticle = True self.curText = [] elif name == "title": self.inTitle = True self.curTitle = "" def endElement(self, name): if name == "page": self.inArticle = False self.handleArticle(self.curTitle, "".join(self.curText)) elif name == "title": self.inTitle = False def characters(self, text): if self.inTitle: self.curTitle += text elif self.inArticle: self.curText.append(text) if len(self.curText) > 10000: # bail out self.inArticle = False def handleArticle(self, title, text): lines = text.split("\n") self.pos = None for line in lines: self.handleLine(title, line.strip()) def handleLine(self, title, line): language_match = LANGUAGE_HEADER.match(line) trans_top_match = TRANS_TOP.match(line) trans_bottom_match = TRANS_BOTTOM.match(line) trans_tag_match = TRANS_TAG.search(line) chinese_match = CHINESE_TAG.search(line) ### Get translation if trans_top_match: # start translation part self.trans = True if self.trans and trans_bottom_match: # end translation part self.trans = False if self.trans and line.startswith("*{{"): # get translation lang = line[3:5] # get language of translation # find all translations of that language translations = re.findall(u"\{\{Ü.*?\|.*?\|(.*?)\}\}", line) for translation in translations: # iterate over translations self.output_sense_translation(lang, translation, title, "") ### Get relation if line.startswith("{{Synonyme}}"): # synonym self.curRelation = "synonym" elif line.startswith(u"{{Gegenwörter}}"): # antonym self.curRelation = "antonym" elif line.startswith("{{Oberbegriffe}}"): # hypernym self.curRelation = "hypernym" elif line.startswith("{{Unterbegriffe}}"): # hyponym self.curRelation = "hyponym" elif line.startswith("{{Redewendungen}}"): # idiom self.curRelation = "idiom" elif line.startswith("{{Charakteristische Wortkombinationen}}"): # word combination self.curRelation = "word combination" elif line.startswith("{{Wortbildungen}}"): # morphology self.curRelation = "morphology" if self.curRelation and line == "": # end relation self.curRelation = None if self.curRelation: related_words_or_phrases = re.findall(r"\[\[(.*?)\]\]", line) for related_word in related_words_or_phrases: self.output_monolingual("deu", self.curRelation, related_word, title) def output_monolingual(self, lang, relation, term1, term2): if "Wik" in term1 or "Wik" in term2: return source = make_concept_uri(term1, lang) if self.pos: target = make_concept_uri(term2, lang, self.pos) else: target = make_concept_uri(term2, lang) surfaceText = "[[%s]] %s [[%s]]" % (term1, relation, term2) # print surfaceText edge = make_edge( "/r/" + relation, source, target, "/d/wiktionary/%s/%s" % (lang, lang), license="/l/CC/By-SA", sources=[SOURCE, MONOLINGUAL], context="/ctx/all", weight=1.5, surfaceText=surfaceText, ) self.writer.write(edge) def output_sense_translation(self, lang, foreign, german, disambiguation): if "Wik" in foreign or "Wik" in german: return if lang == "zh-cn": lang = "zh_CN" elif lang == "zh-tw": lang = "zh_TW" source = make_concept_uri(unicodedata.normalize("NFKC", foreign), lang) target = make_concept_uri(german, "de", disambiguation) relation = "/r/TranslationOf" try: surfaceRel = "is %s for" % (langs.english_name(lang)) except KeyError: surfaceRel = "is [language %s] for" % lang surfaceText = "[[%s]] %s [[%s (%s)]]" % ( foreign, surfaceRel, english, disambiguation.split("/")[-1].replace("_", " "), ) # print surfaceText edge = make_edge( relation, source, target, "/d/wiktionary/en/%s" % lang, license="/l/CC/By-SA", sources=[SOURCE, TRANSLATE], context="/ctx/all", weight=1.5, surfaceText=surfaceText, ) self.writer.write(edge) def output_translation(self, foreign, english, locale=""): source = make_concept_uri(unicodedata.normalize("NFKC", foreign), self.langcode + locale) target = make_concept_uri(english, "en") relation = "/r/TranslationOf" try: surfaceRel = "is %s for" % (langs.english_name(self.langcode)) except KeyError: surfaceRel = "is [language %s] for" % self.langcode surfaceText = "[[%s]] %s [[%s]]" % (foreign, surfaceRel, english) edge = make_edge( relation, source, target, "/d/wiktionary/en/%s" % self.langcode, license="/l/CC/By-SA", sources=[SOURCE, INTERLINGUAL], context="/ctx/all", weight=1.5, surfaceText=surfaceText, ) self.writer.write(edge)
class FindTranslations(ContentHandler): def __init__(self): self.lang = None self.langcode = None self.inArticle = False self.inTitle = False self.curSense = None self.curTitle = '' self.curText = '' self.locales = [] self.curRelation = None self.writer = MultiWriter('wiktionary') def startElement(self, name, attrs): if name == 'page': self.inArticle = True self.curText = [] elif name == 'title': self.inTitle = True self.curTitle = '' def endElement(self, name): if name == 'page': self.inArticle = False self.handleArticle(self.curTitle, ''.join(self.curText)) elif name == 'title': self.inTitle = False def characters(self, text): if self.inTitle: self.curTitle += text elif self.inArticle: self.curText.append(text) if len(self.curText) > 10000: # bail out self.inArticle = False def handleArticle(self, title, text): lines = text.split('\n') self.pos = None for line in lines: self.handleLine(title, line.strip()) def handleLine(self, title, line): language_match = LANGUAGE_HEADER.match(line) trans_top_match = TRANS_TOP.match(line) trans_tag_match = TRANS_TAG.search(line) chinese_match = CHINESE_TAG.search(line) if line.startswith('===') and line.endswith('==='): pos = line.strip('= ') if pos == 'Synonyms': self.curRelation = 'Synonym' elif pos == 'Antonym': self.curRelation = 'Antonym' elif pos == 'Related terms': self.curRelation = 'ConceptuallyRelatedTo' elif pos == 'Derived terms': if not line.startswith('===='): # this is at the same level as the part of speech; # now we don't know what POS these apply to self.pos = None self.curRelation = 'DerivedFrom' else: self.curRelation = None if pos in PARTS_OF_SPEECH: self.pos = PARTS_OF_SPEECH[pos] elif language_match: self.lang = language_match.group(1) self.langcode = LANGUAGES.get(self.lang) elif chinese_match: scripttag = chinese_match.group(2) self.locales = [] if 's' in scripttag: self.locales.append('_CN') if 't' in scripttag: self.locales.append('_TW') elif line[ 0: 1] == '#' and self.lang != 'English' and self.lang is not None: defn = line[1:].strip() if defn[0:1] not in ':*#': for defn2 in filter_line(defn): if not ascii_enough(defn2): continue if 'Index:' in title: continue if self.langcode == 'zh': for locale in self.locales: self.output_translation(title, defn2, locale) elif self.langcode: self.output_translation(title, defn2) elif line[0:4] == '----': self.pos = None self.lang = None self.langcode = None self.curRelation = None elif trans_top_match: pos = self.pos or 'n' sense = trans_top_match.group(1).split(';')[0].strip('.') if 'translations' in sense.lower(): self.curSense = None else: self.curSense = pos + '/' + sense elif trans_tag_match: lang = trans_tag_match.group(1) translation = trans_tag_match.group(2) if self.curSense is not None and self.lang == 'English': # handle Chinese separately if lang not in ('cmn', 'yue', 'zh-yue', 'zh'): self.output_sense_translation(lang, translation, title, self.curSense) elif '{{trans-bottom}}' in line: self.curSense = None elif line.startswith('* ') and self.curRelation and self.langcode: relatedmatch = WIKILINK.search(line) if relatedmatch: related = relatedmatch.group(1) self.output_monolingual(self.langcode, self.curRelation, related, title) def output_monolingual(self, lang, relation, term1, term2): if 'Wik' in term1 or 'Wik' in term2: return source = make_concept_uri(term1, lang) if self.pos: target = make_concept_uri(term2, lang, self.pos) else: target = make_concept_uri(term2, lang) surfaceText = "[[%s]] %s [[%s]]" % (term1, relation, term2) print surfaceText edge = make_edge('/r/' + relation, source, target, '/d/wiktionary/%s/%s' % (lang, lang), license='/l/CC/By-SA', sources=[SOURCE, MONOLINGUAL], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge) def output_sense_translation(self, lang, foreign, english, disambiguation): if 'Wik' in foreign or 'Wik' in english: return if lang == 'zh-cn': lang = 'zh_CN' elif lang == 'zh-tw': lang = 'zh_TW' source = make_concept_uri(unicodedata.normalize('NFKC', foreign), lang) target = make_concept_uri(english, 'en', disambiguation) relation = '/r/TranslationOf' try: surfaceRel = "is %s for" % (langs.english_name(lang)) except KeyError: surfaceRel = "is [language %s] for" % lang surfaceText = "[[%s]] %s [[%s (%s)]]" % ( foreign, surfaceRel, english, disambiguation.split('/')[-1].replace('_', ' ')) print surfaceText edge = make_edge(relation, source, target, '/d/wiktionary/en/%s' % lang, license='/l/CC/By-SA', sources=[SOURCE, TRANSLATE], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge) def output_translation(self, foreign, english, locale=''): source = make_concept_uri(unicodedata.normalize('NFKC', foreign), self.langcode + locale) target = make_concept_uri(english, 'en') relation = '/r/TranslationOf' try: surfaceRel = "is %s for" % (langs.english_name(self.langcode)) except KeyError: surfaceRel = "is [language %s] for" % self.langcode surfaceText = "[[%s]] %s [[%s]]" % (foreign, surfaceRel, english) edge = make_edge(relation, source, target, '/d/wiktionary/en/%s' % self.langcode, license='/l/CC/By-SA', sources=[SOURCE, INTERLINGUAL], context='/ctx/all', weight=1.5, surfaceText=surfaceText) self.writer.write(edge)
for line in codecs.open('data/flat/CORE', encoding='utf-8'): uri, rel, start, end, context, weight, sources, id, dataset = line.split('\t')[:9] if uri != 'uri' and context == '/ctx/all': weight = float(weight) weights[uri] += float(weight) assertions[uri] = (rel, start, end, context, weight) if not (dataset.startswith('/d/reverb') or dataset.startswith('/d/wiktionary') or dataset.startswith('/d/dbpedia')): ccby[uri] = True print 'writing' writer_core = MultiWriter('assertion_totals_core') #writer_sa = MultiWriter('assertion_totals_sa') for uri, weight in assertions.iteritems(): if ccby[uri]: license = '/l/CC/By' dataset = '/d/conceptnet/5/combined-core' else: license = '/l/CC/By-SA' dataset = '/d/conceptnet/5/combined-sa' relation, start, end, context, weight = assertions[uri] edge = make_edge(relation, start, end, dataset, license, ['/s/rule/sum_edges'], '/ctx/all', weight=weight) if license == '/l/CC/By': writer_core.write(edge) #else: # writer_sa.write(edge) writer_core.close() #writer_sa.close()
if score <= 0: counts['low score'] += 1 weak_out.write(line) continue count += 1 counts['success'] += 1 good_out.write(line) if make_json: left = make_concept_uri(unicode(left), 'en') right = make_concept_uri(unicode(right), 'en') edge = make_edge(rel, left, right, '/d/verbosity', '/l/CC/By', sources, surfaceText=text, weight = score/10.0) writer.write(edge) if make_json: writer.close() flag_out.close() good_out.close() weak_out.close() similar_out.close() simout = open('data/output/similarity-scores.txt', 'w') for sim in text_similarities: print >> simout, sim simout.close()
if not (dataset.startswith('/d/reverb') or dataset.startswith('/d/wiktionary') or dataset.startswith('/d/dbpedia')): ccby[uri] = True print 'writing' writer_core = MultiWriter('assertion_totals_core') #writer_sa = MultiWriter('assertion_totals_sa') for uri, weight in assertions.iteritems(): if ccby[uri]: license = '/l/CC/By' dataset = '/d/conceptnet/5/combined-core' else: license = '/l/CC/By-SA' dataset = '/d/conceptnet/5/combined-sa' relation, start, end, context, weight = assertions[uri] edge = make_edge(relation, start, end, dataset, license, ['/s/rule/sum_edges'], '/ctx/all', weight=weight) if license == '/l/CC/By': writer_core.write(edge) #else: # writer_sa.write(edge) writer_core.close() #writer_sa.close()