def mark_adj_concepts(self):
with self.db.ctx() as ctx:
lus = self.db.find_lexunits(flag=LexUnit.PROCESSED, pos='a', ctx=ctx)
for lu in lus[:50]:
self.db.get_lexunit(lu, ctx=ctx)
header(lu)
mark_adj_concepts(lu, ctx)
def main():
header("Main method")
c = Counter()
t = Timer()
t.start("Doing some time-consuming tasks ...")
logging.info("Count even & odd numbers ...")
for i in range(10000):
if i % 2 == 0:
c.count("even")
else:
c.count("odd")
c.summarise()
logging.info("Creating report dir ...")
FileHelper.create_dir(DATA_DIR)
report = TextReport(REPORT_LOC)
logging.info("Now try to create a text report (Located at: %s)" %
(report.get_path()))
generate_report(report)
# try to report to stdout
logging.info("The same report to stdout ...")
generate_report(TextReport())
t.end("Done")
def list_unksense(args):
header("List unknown sensekeys in Semcor")
semxml = SemcorXML(SEMCOR_TTL)
unk = Counter()
sids = Counter()
c = Counter()
out = TextReport() if not args.out else TextReport(args.out)
for f in semxml.files[:args.limit] if args.limit else ttl.files:
doc = ttl.Document.from_json_file(ttl.files.abspath(f))
for s in doc:
for concept in s.concepts:
try:
sid = SynsetID.from_string(concept.tag)
sids.count((sid, concept.clemma))
c.count("Known instances")
except:
sid = None
unk.count((concept.tag, concept.clemma))
c.count("Unknown instances")
out.header("Known concepts")
out.writeline("\t".join(("synsetID", "lemma", "count")))
for k, v in sids.sorted_by_count():
sid, lemma = k
out.writeline("\t".join((str(sid), lemma, str(v))))
out.header("Unknown concepts")
out.writeline("\t".join(("sensekey", "lemma", "count")))
for k, v in unk.sorted_by_count():
sk, lemma = k
out.writeline("\t".join((sk, lemma, str(v))))
out.header("Total")
out.writeline("Known: {}".format(len(sids)))
out.writeline("Unknown: {}".format(len(unk)))
c.summarise(out)
def test_parsing(self):
ghub = GrammarHub()
# noun
sent = ghub.ERG.parse('dog', extra_args=['-r', 'root_wn_n'])
self.assertGreater(len(sent), 0)
dmrs = sent[0].dmrs().obj()
eps = dmrs.eps()
self.assertEqual(len(eps), 1)
self.assertEqual(eps[0].pred.pos, 'n')
# verb
sent = ghub.ERG.parse('love', extra_args=['-r', 'root_wn_v'])
self.assertGreater(len(sent), 0)
dmrs = sent[0].dmrs().obj()
eps = dmrs.eps()
self.assertEqual(len(eps), 1)
self.assertEqual(eps[0].pred.pos, 'v')
# adjective
sent = ghub.ERG.parse('nice', extra_args=['-r', 'root_wn_adj'])
self.assertGreater(len(sent), 0)
dmrs = sent[0].dmrs().obj()
eps = dmrs.eps()
print(sent[0].mrs())
self.assertEqual(len(eps), 1)
self.assertEqual(eps[0].pred.pos, 'a')
# test iterative parsing
words = ['drink', 'eat', 'eat', 'drink']
for parses in ghub.ERG.parse_many_iterative(words, extra_args=['-r', 'root_frag'], ignore_cache=True):
header(parses)
for p in parses:
print(p.mrs())
def list_unksense(args):
header("List unknown sensekeys in Semcor")
semxml = SemcorXML(SEMCOR_TTL)
unk = Counter()
sids = Counter()
c = Counter()
out = TextReport() if not args.out else TextReport(args.out)
for f in semxml.files[:args.limit] if args.limit else ttl.files:
doc = ttl.Document.from_json_file(ttl.files.abspath(f))
for s in doc:
for concept in s.concepts:
try:
sid = SynsetID.from_string(concept.tag)
sids.count((sid, concept.clemma))
c.count("Known instances")
except:
sid = None
unk.count((concept.tag, concept.clemma))
c.count("Unknown instances")
out.header("Known concepts")
out.writeline("\t".join(("synsetID", "lemma", "count")))
for k, v in sids.sorted_by_count():
sid, lemma = k
out.writeline("\t".join((str(sid), lemma, str(v))))
out.header("Unknown concepts")
out.writeline("\t".join(("sensekey", "lemma", "count")))
for k, v in unk.sorted_by_count():
sk, lemma = k
out.writeline("\t".join((sk, lemma, str(v))))
out.header("Total")
out.writeline("Known: {}".format(len(sids)))
out.writeline("Unknown: {}".format(len(unk)))
c.summarise(out)
def main():
header("Main method")
c = Counter()
t = Timer()
t.start("Doing some time-consuming tasks ...")
logging.info("Count even & odd numbers ...")
for i in range(10000):
if i % 2 == 0:
c.count("even")
else:
c.count("odd")
c.summarise()
logging.info("Creating report dir ...")
FileHelper.create_dir(DATA_DIR)
report = TextReport(REPORT_LOC)
logging.info("Now try to create a text report (Located at: %s)" % (report.get_path()))
generate_report(report)
# try to report to stdout
logging.info("The same report to stdout ...")
generate_report(TextReport())
t.end("Done")
def test_get_usr(self):
header("Test get information")
sid = '02386612-a'
lang = 'eng'
defs = omw.sdef.select('synset=? and lang=?', (sid, lang))
usrs = {d.usr for d in defs if d.usr}
self.assertTrue(usrs)
def test_remove_sciname(self):
header("Test removing scientific name")
d = 'triggerfishes ❲Balistidae❳'
self.assertTrue(has_sciname(d))
self.assertTrue(SCIENTIFIC_NAME.search(d))
d_nosn = remove_sciname(d)
expected = 'triggerfishes'
self.assertEqual(d_nosn, expected)
def test_compare_synset(self):
header("Test compare synset")
with omw.ctx() as omw_ctx, gwn.ctx() as gwn_ctx:
ss = '01850676-n'
tags, odef, gdef = compare_synset(omw, gwn, ss, omw_ctx, gwn_ctx)
self.assertEqual(tags, {TAGS.SAME, TAGS.SCINAME, TAGS.REP})
ss = '00445467-v'
tags, odef, gdef = compare_synset(omw, gwn, ss, omw_ctx, gwn_ctx)
self.assertEqual(tags, set())
def test_merge_compound(self):
header("Test merge compound")
n = self.data.get_named()
ne = n.edit(0)
head = ne.top['ARG']
comps = Integral.get_comps(head)
self.assertEqual(len(comps), 1)
Integral.merge_compound(comps[0], True)
self.assertEqual(head.predstr, "named")
self.assertEqual(head.carg, "Charles Bond")
def test_detect_dup(self):
header("Detect duplication in OMW definition")
ss = omw.get_synset('01850676-n')
fixed_def, dup_defs = join_definitions(ss)
self.assertEqual(
ss.definition,
'canvasback; redhead; pochard; canvasback; redhead; pochard; etc. ❲Aythya❳'
)
self.assertEqual(fixed_def,
'canvasback; redhead; pochard; etc. ❲Aythya❳')
self.assertEqual(set(dup_defs), {'canvasback', 'redhead', 'pochard'})
def wn2ttl(args):
inpath = FileHelper.abspath(args.inpath)
header("WN to TTL format")
wn = GWordnetXML()
wn.read(inpath)
print("Found senses: {}".format(len(wn.synsets)))
outpath = FileHelper.abspath(args.outpath) if args.outpath else None
with TextReport(outpath, 'w') as outfile:
if args.format == 'json':
convert_json(wn.synsets, outfile)
elif args.format == 'xml':
convert_xml(wn.synsets, outfile)
print("Done!")
def test_tag_one_sent(self):
getLogger().debug("Test tagging one sentence")
sid = '10081'
sents = self.gold()
smap = {str(s.ident): s for s in sents}
sent = smap[sid]
doc = ttl.Document('gold', TEST_GOLD_DIR).read()
sent.shallow = doc.get(sid)
if sent.text != sent.shallow.text:
getLogger().debug("WARNING: Inconsistent")
getLogger().debug(sent.text)
getLogger().debug(sent.shallow.text)
dmrs = sent[0].dmrs()
# print(sent.to_xml_str())
m, n, ignored = tag_gold(dmrs, sent.shallow, sent.text)
header("#{}: {}".format(sid, sent.text), 'h0')
getLogger().debug(sent[0].dmrs())
header('Available concepts')
for c in sent.shallow.concepts:
getLogger().debug("{} {}".format(c, c.tokens))
header('Matched')
for con, nid, pred in m:
getLogger().debug("{}::{} => #{}::{}".format(con.tag, con.clemma, nid, pred))
header("Not matched")
if n:
for c in n:
getLogger().debug(c)
else:
getLogger().debug("All was matched.")
sent.tag(method=ttl.Tag.MFS)
xml_str = sent.tag_xml().to_xml_str()
self.assertTrue(xml_str)
self.assertIn("<sensegold", xml_str)
self.assertIn("<sense", xml_str)
getLogger().debug(sent.to_xml_str())
def test_generate_from_gold(self):
header("Test generating from gold (THIS CAN BE VERY SLOW)")
sents = self.gold()
c = Counter()
for sent in sents[:2]:
if len(sent) > 0:
try:
gsents = ERG.generate(sent[0])
c.count("OK" if gsents else "ERR")
except:
c.count("BROKEN")
else:
c.count("SKIP")
c.summarise()
def test_find_surface(self):
sid = '00445467-v'
omwss = omw.get_synset(sid)
gwnss = gwn.get_synset(sid)
odef_orig = '; '.join(omwss.definitions)
odef, is_duplicated = join_definitions(omwss)
gdef = gwnss.get_def().text()
if gdef.endswith(";"):
gdef = gdef[:-1].strip()
if odef != gdef:
header(sid)
print("OMW-orig: {} | is_dup: {}".format(odef_orig, is_duplicated))
print("OMW:", odef)
print("GWN:", gdef)
def test_sent(self):
header("Test model")
doc = Document("test")
s = doc.new("It rains.")
s.add('''[ TOP: h0
INDEX: e2 [ e SF: prop TENSE: pres MOOD: indicative PROG: - PERF: - ]
RELS: < [ _rain_v_1<3:9> LBL: h1 ARG0: e2 ] >
HCONS: < h0 qeq h1 > ]''')
# test full work flow:
# mrs_str > dmrs() > xml > layout > dmrs > mrs > mrs_str
expected = '''[ TOP: h0 RELS: < [ _rain_v_1<3:9> LBL: h1 ARG0: e2 [ e SF: prop TENSE: pres MOOD: indicative PROG: - PERF: - ] ] > HCONS: < h0 qeq h1 > ]'''
actual = DMRSLayout.from_xml(s[0].edit().to_dmrs().xml()).to_dmrs().to_mrs().tostring(False)
self.assertEqual(actual, expected)
# Test different formats
xstr = s.to_xml_str()
self.assertTrue(xstr)
lts = s.to_latex() # LaTeX scripts
self.assertTrue(lts)
def test_def_dup(self):
header("Check if a definition is not unique")
sid = '11937102-n'
omwss = omw.get_synset(sid)
gwnss = gwn.get_synset(sid)
self.assertEqual(omwss.definition,
'a variety of aster (Symphyotrichum lateriflorum)')
self.assertEqual(gwnss.definition, 'a variety of aster;')
glosses = gwn.gloss.select('surface = ?', (gwnss.definition, ))
ssids = {str(SynsetID.from_string(g.sid)) for g in glosses}
self.assertEqual(
ssids, {
'11935627-n', '11935715-n', '11935794-n', '11935877-n',
'11935953-n', '11936027-n', '11936113-n', '11936199-n',
'11936287-n', '11936369-n', '11936448-n', '11936539-n',
'11936624-n', '11936707-n', '11936782-n', '11936864-n',
'11936946-n', '11937023-n', '11937102-n', '11937195-n',
'11937278-n', '11937360-n', '11937446-n'
})
def test_dmrs(self):
header("Test building a DMRS from scratch")
corpus = Corpus(name="manual")
doc = corpus.new(name="testdoc")
sent = doc.new("It rains.")
self.assertIsInstance(sent, Sentence)
reading = sent.add('[]')
dmrs = reading.dmrs()
l = dmrs.layout
n = Node(10000, "_rain_v_1", 3, 9)
n.sortinfo.update({'sf': 'prop', 'tense': 'pres', 'mood': 'indicative',
'prog': '-', 'perf': '-', 'sarcasm': '-'})
l.add_node(n)
l.add_link(Link(0, 10000, '', 'H')) # this is top
l.save()
# sense tag the DMRS
sent.tag(ttl.Tag.MFS)
self.assertGreaterEqual(len(dmrs.tags), 1)
self.assertTrue(sent.to_xml_str())
def mine_rdf_values(sc, limit=None):
ot_values = set()
rdf_counter = Counter()
rdf_with_key_counter = Counter()
for f in sc.files if not limit else sc.files[:limit]:
for sj in sc.iterparse(f):
for t in sj['tokens']:
if 'rdf' in t:
l = t['lemma'] if 'lemma' in t else t.text
r = t['rdf']
sk = t['sk'] if 'sk' in t else ''
item = (l, r, sk)
rdf_counter.count(item)
if sk:
rdf_with_key_counter.count(item)
if 'ot' in t:
ot_values.add(t['ot'])
header("RDF values")
for k, v in rdf_counter.most_common():
print("{}: {}".format(k, v))
header("RDF values (with valid keys)")
for k, v in rdf_with_key_counter.most_common():
print("{}: {}".format(k, v))
header("OT values")
for o in ot_values:
print(o)
def dump(lu):
header(lu)
for r in lu:
print(r.mode, r.dmrs())
def compare_gwn_wn30(self):
header("GWN and WN30 are equal")
# compare synset IDso
gwn_ssids = set(get_gwn_synsets())
wn_ssids = set(get_wn30_synsets())
self.assertEqual(gwn_ssids, wn_ssids)
def archive_corpus(bib, corpus, ctx):
header("Archiving corpus {}".format(corpus.name))
docs = bib.sqldao.get_docs(corpus.ID, ctx=ctx)
for doc in docs:
archive_doc(bib, corpus, doc.name, ctx=ctx)
def archive_collection(bib, ctx):
header("Archiving collection {}".format(bib.name), level="h0")
corpuses = ctx.corpus.select()
for corpus in corpuses:
archive_corpus(bib, corpus, ctx)
def test_all_grammars(self):
header("Verify available grammars (JACY/VRG/ERG)")
for n in ('JACY', 'VRG', 'ERG'):
self.assertIn(n, self.ghub.names)
def test_omw(self):
header("Ensure that OMW is working")
omw_ssids = get_omw_synsets()
print("OMW synsets: {}".format(len(omw_ssids)))
print(omw_ssids[:5])
def test_xml_to_ttl(self):
header("Test fixed 3rada to TTL format")
sc = SemcorXML(SEMCOR_FIXED)
scttl = FileSet(SEMCOR_TTL)
sc.convert_to_ttl(scttl, limit=1, with_nonsense=False)
def test_xml2json(self):
header("Test fixed 3rada to JSON")
sc = SemcorXML(SEMCOR_FIXED)
sc_json = FileSet(SEMCOR_JSON)
for f in sc.files:
xml2json(f, sc, sc_json)
def test_fix_3rada(self):
header("Test fix original 3rada dataset")
fix_3rada(SEMCOR_ORIG, SEMCOR_FIXED)
def test_gwn(self):
header("Ensure that GWN is working")
gwn_ssids = get_gwn_synsets()
print("GWN synsets: {}".format(len(gwn_ssids)))
print(gwn_ssids[:5])
def test_wn30(self):
header("Ensure that PWN-3.0 is working")
wn_ssids = get_wn30_synsets()
print("WN synsets: {}".format(len(wn_ssids)))
print(wn_ssids[:5])
