def test_as_dict_key(self): s = SynsetID.from_string('12345678n') s2 = SynsetID.from_string('112345678') d = {} d['12345678-n'] = 'abc' self.assertEqual(d[s], 'abc') self.assertIn(s2, d)
def test_unusual_sid(self): s = SynsetID.from_string('80000683-x') s = SynsetID.from_string(s) self.assertEqual(s.pos, 'x') self.assertEqual(s.offset, '80000683') # ? s = SynsetID.from_string('02315002-a')
def test_pos(self): self.assertEqual(POS.num2pos(1), 'n') self.assertEqual(POS.num2pos("1"), 'n') self.assertRaises(WordnetException, lambda: POS.num2pos("7")) self.assertRaises(WordnetException, lambda: POS.num2pos(None)) self.assertRaises(WordnetException, lambda: POS.num2pos(0)) self.assertEqual(POS.pos2num("n"), '1') self.assertEqual(POS.pos2num("v"), '2') self.assertEqual(POS.pos2num("a"), '3') self.assertEqual(POS.pos2num("r"), '4') self.assertEqual(POS.pos2num("s"), '5') self.assertEqual(POS.pos2num("x"), '6') self.assertRaises(WordnetException, lambda: POS.pos2num(None)) self.assertRaises(WordnetException, lambda: POS.pos2num("g")) self.assertRaises(WordnetException, lambda: POS.pos2num("")) self.assertRaises(WordnetException, lambda: POS.pos2num(1)) self.assertRaises(WordnetException, lambda: POS.pos2num("n ")) self.assertEqual(SynsetID.from_string('112345678').pos, 'n') self.assertEqual(SynsetID.from_string('212345678').pos, 'v') self.assertEqual(SynsetID.from_string('312345678').pos, 'a') self.assertEqual(SynsetID.from_string('412345678').pos, 'r') self.assertEqual(SynsetID.from_string('512345678').pos, 's') self.assertEqual(SynsetID.from_string('n12345678').pos, 'n') self.assertEqual(SynsetID.from_string('v12345678').pos, 'v') self.assertEqual(SynsetID.from_string('a12345678').pos, 'a') self.assertEqual(SynsetID.from_string('r12345678').pos, 'r') self.assertEqual(SynsetID.from_string('s12345678').pos, 's')
def test_synset_wrong_format(self): print("Test invalid synset formats") self.assertRaises(Exception, lambda: SynsetID.from_string(None)) # wrong POS (canonical) self.assertRaises(Exception, lambda: SynsetID.from_string('12345678g')) # wrong POS (WNSQL) self.assertRaises(Exception, lambda: SynsetID.from_string('712345678')) # wrong POS (WNSQL) #2 self.assertRaises(Exception, lambda: SynsetID.from_string('k12345678')) # no POS self.assertRaises(Exception, lambda: SynsetID.from_string('12345678'))
def test_pos(self): self.assertEqual(SynsetID.from_string('112345678').pos, 'n') self.assertEqual(SynsetID.from_string('212345678').pos, 'v') self.assertEqual(SynsetID.from_string('312345678').pos, 'a') self.assertEqual(SynsetID.from_string('412345678').pos, 'r') self.assertEqual(SynsetID.from_string('512345678').pos, 's') self.assertEqual(SynsetID.from_string('n12345678').pos, 'n') self.assertEqual(SynsetID.from_string('v12345678').pos, 'v') self.assertEqual(SynsetID.from_string('a12345678').pos, 'a') self.assertEqual(SynsetID.from_string('r12345678').pos, 'r') self.assertEqual(SynsetID.from_string('s12345678').pos, 's')
def export_wnsql_synsets(args): print( "Exporting synsets' info (lemmas/defs/examples) from WordnetSQL (Princeton Wordnet 3.0) to text file" ) show_info(args) output_with_sid_file = os.path.abspath('./data/wn30_lemmas.txt') output_without_sid_file = os.path.abspath('./data/wn30_lemmas_noss.txt') output_defs = os.path.abspath('./data/wn30_defs.txt') output_exes = os.path.abspath('./data/wn30_exes.txt') wn = get_wn(args) # Extract lemmas records = wn.get_all_synsets() synsets_lemmas = [] for r in records: synsets_lemmas.append( (SynsetID.from_string(str(r.synsetid)).to_canonical(), r.lemma)) synsets_lemmas.sort(key=itemgetter(0, 1)) with open(output_with_sid_file, 'w') as with_sid, open(output_without_sid_file, 'w') as without_sid: for row in synsets_lemmas: with_sid.write('%s\t%s\n' % row) without_sid.write('%s\n' % (row[1], )) # just the lemma # Extract synset definitions records = wn.schema.ss.select(orderby='synsetid') synsets_defs = [] for r in records: synsets_defs.append( (SynsetID.from_string(r.synsetid).to_canonical(), r.definition)) synsets_defs.sort(key=itemgetter(0)) with open(output_defs, 'w') as def_file: for row in synsets_defs: def_file.write('%s\t%s\n' % row) # Extract examples records = wn.schema.ex.select(orderby='synsetid') synsets_examples = [] for r in records: synsets_examples.append( (SynsetID.from_string(r.synsetid).to_canonical(), r.sample)) synsets_examples.sort(key=itemgetter(0)) with open(output_exes, 'w') as ex_file: for row in synsets_examples: ex_file.write('%s\t%s\n' % row) # summary print("Data has been extracted to:") print(" + {}".format(output_with_sid_file)) print(" + {}".format(output_without_sid_file)) print(" + {}".format(output_defs)) print(" + {}".format(output_exes)) print("Done!")
def list_unksense(args): header("List unknown sensekeys in Semcor") semxml = SemcorXML(SEMCOR_TTL) unk = Counter() sids = Counter() c = Counter() out = TextReport() if not args.out else TextReport(args.out) for f in semxml.files[:args.limit] if args.limit else ttl.files: doc = ttl.Document.from_json_file(ttl.files.abspath(f)) for s in doc: for concept in s.concepts: try: sid = SynsetID.from_string(concept.tag) sids.count((sid, concept.clemma)) c.count("Known instances") except: sid = None unk.count((concept.tag, concept.clemma)) c.count("Unknown instances") out.header("Known concepts") out.writeline("\t".join(("synsetID", "lemma", "count"))) for k, v in sids.sorted_by_count(): sid, lemma = k out.writeline("\t".join((str(sid), lemma, str(v)))) out.header("Unknown concepts") out.writeline("\t".join(("sensekey", "lemma", "count"))) for k, v in unk.sorted_by_count(): sk, lemma = k out.writeline("\t".join((sk, lemma, str(v)))) out.header("Total") out.writeline("Known: {}".format(len(sids))) out.writeline("Unknown: {}".format(len(unk))) c.summarise(out)
def omw_fix_dup(cli, args): rp = TextReport(args.output) omw = get_omw() c = Counter() with omw.ctx() as ctx: senses = ctx.sense.select(limit=args.topk, columns=('synset', )) synsetids = {s.synset for s in senses} rp.print("-- OMW synsets: {}\n".format(len(synsetids))) for sid in synsetids: try: sid = SynsetID.from_string(sid) except: cli.logger.warning("Ignored synset ID: {}".format(sid)) continue ss = omw.get_synset(sid, ctx=ctx) fixed_def, dup_defs = join_definitions(ss) if dup_defs: c.count("Duplicated") rp.print("-- Original {}: {}".format(ss.ID, ss.definition)) rp.print("-- Fixed {}: {}".format(ss.ID, fixed_def)) for dup in dup_defs: rp.print( "DELETE FROM synset_def WHERE synset='{}' and def='{}';" .format(ss.ID, to_sqlite_string(dup))) rp.print() c.summarise() pass
def list_unksense(args): header("List unknown sensekeys in Semcor") semxml = SemcorXML(SEMCOR_TTL) unk = Counter() sids = Counter() c = Counter() out = TextReport() if not args.out else TextReport(args.out) for f in semxml.files[:args.limit] if args.limit else ttl.files: doc = ttl.Document.from_json_file(ttl.files.abspath(f)) for s in doc: for concept in s.concepts: try: sid = SynsetID.from_string(concept.tag) sids.count((sid, concept.clemma)) c.count("Known instances") except: sid = None unk.count((concept.tag, concept.clemma)) c.count("Unknown instances") out.header("Known concepts") out.writeline("\t".join(("synsetID", "lemma", "count"))) for k, v in sids.sorted_by_count(): sid, lemma = k out.writeline("\t".join((str(sid), lemma, str(v)))) out.header("Unknown concepts") out.writeline("\t".join(("sensekey", "lemma", "count"))) for k, v in unk.sorted_by_count(): sk, lemma = k out.writeline("\t".join((sk, lemma, str(v)))) out.header("Total") out.writeline("Known: {}".format(len(sids))) out.writeline("Unknown: {}".format(len(unk))) c.summarise(out)
def test_assign_synsetid(self): s1 = SynsetID.from_string('12345678-n') ss = Synset(s1) ssid = ss.ID self.assertIs(ss.ID, ssid) self.assertEqual(ss.ID, s1) self.assertIsNot(ss.ID, s1) # must not be the same instance SynsetID
def get_omw_synsets(): omw_ss = omw.sense.select() omw_ssids = set() for x in omw_ss: try: omw_ssids.add(SynsetID.from_string(x.synset)) except: print(x) return list(omw_ssids)
def bake_doc(doc): ''' Convert concepts to tags ''' for sent in doc: for concept in sent.concepts: cfrom = min(t.cfrom for t in concept.tokens) cto = min(t.cto for t in concept.tokens) sid = SynsetID.from_string(concept.tag, default=None) # must be a valid synsetID if cfrom >= 0 and cto >= 0 and sid is not None: sent.new_tag(concept.tag, cfrom, cto, tagtype='WN') return doc
def filter_bad_synsetids(concepts): good_concepts = [] bad_concepts = [] for c in concepts: valid_sid = SynsetID.from_string(c.tag, default=None) if valid_sid is None: getLogger().warning("{} is not a valid synset ID".format(c.tag)) bad_concepts.append(c) else: good_concepts.append(c) return good_concepts, bad_concepts
def test_synset_id(self): print("Test synset ID") sid = SynsetID('12345678', 'n') expected_canonical = '12345678-n' expected_wnsql = '112345678' # validate formats self.assertEqual(str(sid), expected_canonical) self.assertEqual(sid.to_canonical(), expected_canonical) self.assertEqual(sid.to_wnsql(), expected_wnsql) # comparing synset sid2 = SynsetID.from_string('112345678') sid3 = SynsetID.from_string('12345678-n') sid4 = SynsetID.from_string('12345678n') sid5 = SynsetID.from_string('n12345678') sid6 = SynsetID.from_string(112345678) self.assertEqual(sid, sid2) self.assertEqual(sid, sid3) self.assertEqual(sid, sid4) self.assertEqual(sid, sid5) self.assertEqual(sid, sid6)
def to_ttl(sent, with_nonsense=True, sk_map=None, wnctx=None): tokens = sent['tokens'] text = detokenize(tokens) s = ttl.Sentence(text=text) s.new_tag(sent['sid'], tagtype='origid') s.import_tokens((t.text for t in tokens)) for tinfo, tk in zip(tokens, s): for k, v in tinfo.data: if (k, v) == ('tag', 'wf') or k == 'sk': continue if k == 'lemma': tk.lemma = v elif k == 'pos': tk.pos = v else: tk.new_tag(label=v, tagtype=k) # if sensekey exists, add it as a concept lemma = tinfo.lemma sk = fix_sensekey(tinfo.get('sk')) rdf = tinfo.get('rdf') comment = None if sk and (with_nonsense or not is_nonsense(lemma, sk, rdf)): sensetag = sk if sk_map is not None and sk in sk_map: sensetag = sk_map[sk] elif wnctx is not None: # try to determine synsetID ss = wnctx.senses.select_single('sensekey=?', (sk, )) if ss is not None: sid = str(SynsetID.from_string(ss.synsetid)) if sk_map is not None: sk_map[sk] = sid sensetag = sid else: # sensekey not found getLogger().warning( "There is no synsetID with sensekey={} | rdf={}". format(sk, rdf)) comment = 'sensekey' s.new_concept(clemma=lemma, tag=sensetag, tokens=(tk, ), comment=comment) return s
def test_get_mfs(self): words = 'we this sing full cat tongue name dry die horn sun with mountain eye belly old big red woman live head animal because cloud louse sleep ear wet know salt walk eat seed green bite say person all child count thin stand father laugh night give stone heavy if bone sister other yellow small work snake smoke kill white swim short grease worm narrow flower neck path drink flesh good sharp ash snow hot fire mouth see dirty hand egg skin cold fly wood mother come I warm where one play foot sea year new earth smooth two water what burn fish vomit bird how long hunt sit rope feather nose dust round wind tooth correct bark root ice not blood tail dull brother man heart lie liver many pig rain claw who day grass knee when leaf wide hair meat black dog star dance breasts wife sand husband You bad hear moon river tree that'.split( ) with omw.ctx() as ctx, TextReport( 'data/mfs1500.txt') as rp, TextReport( "data/wndef.txt") as deffile: query = 'wordid in (SELECT wordid FROM word WHERE lemma in {})'.format( repr(tuple(words))) rows = ctx.sense.select(query) ssids = [SynsetID.from_string(r.synset) for r in rows] for ssid in ssids: ss = omw.get_synset(ssid, ctx=ctx) if ss.lemmas and ss.definition: rp.print("{id} ({lm}): {df}".format( id=ss.ID, lm=", ".join(ss.lemmas), df=ss.definition.strip())) deffile.print(ss.definition.strip()) print("Done!")
def test_def_dup(self): header("Check if a definition is not unique") sid = '11937102-n' omwss = omw.get_synset(sid) gwnss = gwn.get_synset(sid) self.assertEqual(omwss.definition, 'a variety of aster (Symphyotrichum lateriflorum)') self.assertEqual(gwnss.definition, 'a variety of aster;') glosses = gwn.gloss.select('surface = ?', (gwnss.definition, )) ssids = {str(SynsetID.from_string(g.sid)) for g in glosses} self.assertEqual( ssids, { '11935627-n', '11935715-n', '11935794-n', '11935877-n', '11935953-n', '11936027-n', '11936113-n', '11936199-n', '11936287-n', '11936369-n', '11936448-n', '11936539-n', '11936624-n', '11936707-n', '11936782-n', '11936864-n', '11936946-n', '11937023-n', '11937102-n', '11937195-n', '11937278-n', '11937360-n', '11937446-n' })
def search(query): # assume that query is a synset? try: sid = SynsetID.from_string(query) ss = wsql.get_synset_by_id(sid) if ss is not None: return SynsetCollection().add(ss).to_json_str() except Exception as e: # not synsetid logger.exception(e, "Invalid synset ID") pass # try search by lemma synsets = wsql.get_synsets_by_lemma(query) if synsets: return synsets.to_json_str() else: # search by sensekey ss = wsql.get_synset_by_sk(query) if ss: return SynsetCollection().add(ss).to_json_str() # invalid query abort(404)
def to_ttl(sent, with_nonsense=True, sk_map=None, wnctx=None): tokens = sent['tokens'] text = detokenize(tokens) s = ttl.Sentence(text=text) s.new_tag(sent['sid'], tagtype='origid') s.import_tokens((t.text for t in tokens)) for tinfo, tk in zip(tokens, s): for k, v in tinfo.data: if (k, v) == ('tag', 'wf') or k == 'sk': continue if k == 'lemma': tk.lemma = v elif k == 'pos': tk.pos = v else: tk.new_tag(label=v, tagtype=k) # if sensekey exists, add it as a concept lemma = tinfo.lemma sk = fix_sensekey(tinfo.get('sk')) rdf = tinfo.get('rdf') comment = None if sk and (with_nonsense or not is_nonsense(lemma, sk, rdf)): sensetag = sk if sk_map is not None and sk in sk_map: sensetag = sk_map[sk] elif wnctx is not None: # try to determine synsetID ss = wnctx.senses.select_single('sensekey=?', (sk,)) if ss is not None: sid = str(SynsetID.from_string(ss.synsetid)) if sk_map is not None: sk_map[sk] = sid sensetag = sid else: # sensekey not found getLogger().warning("There is no synsetID with sensekey={} | rdf={}".format(sk, rdf)) comment = 'sensekey' s.new_concept(clemma=lemma, tag=sensetag, tokens=(tk,), comment=comment) return s
def search(request, query): """ Search by lemma, sensekey or synsetID Mapping: /yawol/search/<query> """ # assume that query is a synset? sid = SynsetID.from_string(query.strip(), default=None) if sid: ss = wsql.get_synset(sid) if ss is None and omwsql is not None: # try to search by OMW print("Searching in OMW") ss = omwsql.get_synset(sid) print("OMW SS", ss) if ss is not None: return SynsetCollection().add(ss).to_json() # try to search by lemma synsets = wsql.search(lemma=query) if synsets: logger.info("Query: {} - Results: {}".format(query, synsets)) return synsets.to_json() else: if not synsets and omwsql is not None: print("Try to search {} in OMW".format(query)) synsets = omwsql.search(lemma=query) if synsets: logger.info("Query: {} - Results: {}".format(query, synsets)) return synsets.to_json() else: logger.warning("Not found {} in OMW".format(query)) else: # try to search by sensekey try: ss = wsql.get_by_key(query) except: ss = None if ss: return SynsetCollection().add(ss).to_json() # invalid query raise Http404('Invalid query')
def search(request, query): ''' Search by lemma, sensekey or synsetID Mapping: /yawol/search/<query> ''' # assume that query is a synset? try: sid = SynsetID.from_string(query) ss = wsql.get_synset_by_id(sid) if ss is not None: return SynsetCollection().add(ss).to_json() except: pass # try to search by lemma synsets = wsql.get_synsets_by_lemma(query) if synsets: return synsets.to_json() else: # try to search by sensekey ss = wsql.get_synset_by_sk(query) if ss: return SynsetCollection().add(ss).to_json() # invalid query raise Http404('Invalid query')
def get_gwn_synsets(): gwn_ss = gwn.synset.select(columns=('ID',)) return [SynsetID.from_string(x.ID) for x in gwn_ss]
def generate_lelesk_test_profile(): '''Generate test profile for lelesk (new format 31st Mar 2015) ''' # Read all tags tagmap = dd(list) # sentence ID > tags list # a sample line: 10000 4 13 00796315-n adventure NN TagInfo = namedtuple('TagInfo', 'sentid cfrom cto synsetid word pos'.split()) c = Counter() with open(TAG_FILE, 'r') as tags: for tag in tags: # br-k22-1 8 11 not%4:02:00:: not parts = [x.strip() for x in tag.split('\t')] if len(parts) == 6: if parts[3][0] in '=!': parts[3] = parts[3][1:] parts[3] = SynsetID.from_string(parts[3]) tag = TagInfo(*parts) tagmap[tag.sentid].append(tag) else: print("WARNING: Invalid line | ``{}''".format(parts)) # read in words wordmap = dd(list) with open(TOKEN_FILE, 'r') as wordfile: for line in wordfile: if line.startswith('#') or len(line.strip()) == 0: continue # sid word parts = [x.strip() for x in line.split('\t')] if len(parts) == 2: (sid, word) = parts wordmap[sid].append(word) # build test profile sentences = [] line_count = 0 with open(RAW_FILE, 'r') as lines: for line in lines: line_count += 1 if line.startswith('#'): continue else: parts = [x.strip() for x in line.split('\t')] if len(parts) == 2: sid, sent = parts tokens = '' if sid in wordmap: tokens = '|'.join(wordmap[sid]) if sid in tagmap: print(sent) # found tags pos = pos2wnpos(tag.pos, tag.word) c.count(pos) for tag in tagmap[sid]: sentences.append((tag.word, tag.synsetid, pos, sent, tokens)) # write to file with open(TEST_PROFILE_OUTPUT, 'w') as outputfile: for k, v in c.sorted_by_count(): outputfile.write("# %s: %s\n" % (k, v)) for sentence in sentences: outputfile.write("%s\t%s\t%s\t%s\t%s\n" % sentence) # write dev profile random.seed(31032015) itemids = sorted(random.sample(range(line_count), 500)) with open(TEST_PROFILE_OUTPUT_DEV, 'w') as outputfile: for itemid in itemids: outputfile.write("%s\t%s\t%s\t%s\t%s\n" % sentences[itemid]) pass
def get_wn30_synsets(): wn_ss = wn30.schema.ss.select() wn_ssids = [SynsetID.from_string(x.synsetid) for x in wn_ss] return wn_ssids
def omw_vs_gwn_def(): rp = TextReport("data/omw_gwn_report.txt") rpdiff = TextReport("data/omw_gwn_diff.txt") rptypo = TextReport("data/omw_gwn_typo.txt") rpsn = TextReport("data/omw_gwn_sciname.txt") c = Counter(TAGS.ORDER) # ssids to compare diff_ssids = [] ssids = read_diff_ssids() if not ssids: print("Generating synset ID list") omw_ssids = set(get_omw_synsets()) gwn_ssids = set(get_gwn_synsets()) # only care about old GWN synsets ssids = omw_ssids.intersection(gwn_ssids) else: print("Comparing {} synsets loaded from {}".format(len(ssids), ssid_filepath)) lang = 'eng' with omw.ctx() as omw_ctx, gwn.ctx() as gwn_ctx: print("Comparing {} synsets".format(len(ssids))) for ss in list(ssids): ss = str(ss) c.count("total") tags, odef, gdef = compare_synset(omw, gwn, ss, omw_ctx, gwn_ctx) omwss = omw.get_synset(ss, ctx=omw_ctx) tags_str = ' '.join('[{}]'.format(t.upper()) for t in tags) if TAGS.DIFF in tags: diff_ssids.append(ss) # [FCB] why did we change? gwnss = gwn.get_synset(ss, ctx=gwn_ctx) glosses = gwn_ctx.gloss.select('surface = ?', (gwnss.definition,)) if glosses and len(glosses) > 1: tags.add(TAGS.DUP) ssids = [str(SynsetID.from_string(g.sid)) for g in glosses] reason = "Not unique (Shared among {}) so OMW team changed it".format(' '.join(ssids)) else: tags.add(TAGS.OMW) defs = omw.sdef.select('synset=? and lang=?', (ss, lang)) usrs = {d.usr for d in defs if d.usr} usrs_str = ', '.join(usrs) if usrs else "someone in NTU" reason = "{} made this change.".format(usrs_str) tags_str = ' '.join('[{}]'.format(t.upper()) for t in tags) rpdiff.header("{} {}".format(tags_str, ss)) rpdiff.print("OMW: {}".format(omwss.definition)) rpdiff.print("GWN: {}".format(gdef)) rpdiff.print("Reason: {}".format(reason)) if TAGS.SCINAME in tags: rpsn.header("{} {}".format(tags_str, ss)) rpsn.print("OMW: {}".format(omwss.definition)) rpsn.print("GWN: {}".format(gdef)) if TAGS.REP in tags or TAGS.TYPO in tags: rptypo.header("{} {}".format(tags_str, ss)) rptypo.print("OMW: {}".format(omwss.definition)) rptypo.print("GWN: {}".format(gdef)) # master report for tag in tags: c.count(tag) if not tags: c.count(TAGS.IDENT) rp.header("{} {}".format(tags_str, ss)) rp.print("OMW: {}".format(omwss.definition)) rp.print("GWN: {}".format(gdef)) # done c.summarise(report=rp) with open('data/omw_gwn_diff_ssids.txt', 'wt') as diff_ssid_file: for ss in diff_ssids: diff_ssid_file.write('{}\n'.format(ss))