def gen_stats(corpus_file, report_path=None):
''' Generate statistics for a text corpus (word count, most frequent words, etc.)
'''
report = TextReport(report_path)
report.header("Stat for %s" % corpus_file)
line_count = -1
word_count = 0
c = Counter()
with open(corpus_file, 'r', encoding='utf8') as infile:
lines = infile.readlines()
line_count = len(lines)
for line in lines:
tokens = line.split()
for token in tokens:
parts = token.split("/")
if len(parts) == 2:
word = parts[0]
POS = parts[1]
else:
word = parts[0]
POS = None
for spechar in SPECIAL_CHARS:
word = word.replace(spechar, '')
word = word.lower().replace("_", " ") # original word form
if word == '':
print(token)
c.count(word)
word_count += 1
report.writeline("Line count: %s" % line_count)
report.writeline("Word count: %s" % word_count)
report.writeline("Word class: %s" % len(c.sorted_by_count()))
report.writeline("Top %d :" % TOP_K)
for item in c.sorted_by_count()[:TOP_K]:
report.writeline("%s: %s" % (item[0], item[1]), level=1)
report.writeline("Bottom %d :" % TOP_K)
for item in c.sorted_by_count()[-TOP_K:]:
report.writeline("%s: %s" % (item[0], item[1]), level=1)
report.writeline("-" * 80)
for item in c.group_by_count():
report.writeline("%s: %s" % (item[0], ', '.join(DaoPhay.vn_sorted(item[1]))))
def gen_stats(corpus_file, report_path=None):
''' Generate statistics for a text corpus (word count, most frequent words, etc.)
'''
report = TextReport(report_path)
report.header("Stat for %s" % corpus_file)
line_count = -1
word_count = 0
c = Counter()
with open(corpus_file, 'r', encoding='utf8') as infile:
lines = infile.readlines()
line_count = len(lines)
for line in lines:
tokens = line.split()
for token in tokens:
parts = token.split("/")
if len(parts) == 2:
word = parts[0]
POS = parts[1]
else:
word = parts[0]
POS = None
for spechar in SPECIAL_CHARS:
word = word.replace(spechar, '')
word = word.lower().replace("_", " ") # original word form
if word == '':
print(token)
c.count(word)
word_count += 1
report.writeline("Line count: %s" % line_count)
report.writeline("Word count: %s" % word_count)
report.writeline("Word class: %s" % len(c.sorted_by_count()))
report.writeline("Top %d :" % TOP_K)
for item in c.sorted_by_count()[:TOP_K]:
report.writeline("%s: %s" % (item[0], item[1]), level=1)
report.writeline("Bottom %d :" % TOP_K)
for item in c.sorted_by_count()[-TOP_K:]:
report.writeline("%s: %s" % (item[0], item[1]), level=1)
report.writeline("-" * 80)
for item in c.group_by_count():
report.writeline("%s: %s" %
(item[0], ', '.join(DaoPhay.vn_sorted(item[1]))))
def dump_synset(ss, compact_gloss=False, compact_tags=False, more_compact=True, report_file=None):
''' Print synset details for debugging purpose
Arguments:
ss -- Synset object to dump
compact_gloss -- Don't dump gloss tokens' details
compact_tags -- Don't dump tagged senses' details
more_compact -- Don't dump full details of synset
report_file -- Report file to write to
'''
if report_file == None:
report_file = TextReport() # Default to stdout
if more_compact:
report_file.header("Synset: %s (terms=%s | keys=%s)" % (ss.get_synsetid(), ss.terms, ss.keys), 'h0')
else:
report_file.header("Synset: %s" % ss, 'h0')
for rgloss in ss.raw_glosses:
if more_compact:
if rgloss.cat != 'orig':
continue
report_file.print(rgloss)
gloss_count = itertools.count(1)
for gloss in ss.glosses:
print()
report_file.header("Gloss #%s: %s" % (next(gloss_count), gloss), 'h2')
# Dump gloss items
if compact_gloss:
report_file.print("Tokens => %s" % gloss.get_gramwords(), level=2)
else:
for item in gloss.items:
# print("\t%s - { %s }" % (uniquify(item.get_gramwords()), item))
report_file.print("%s - { %s }" % (set(item.get_gramwords()), item), level=2)
report_file.print(("-" * 10), level=1)
# Dump tags
if compact_tags:
report_file.print("Tags => %s" % gloss.get_tagged_sensekey(), level=2)
else:
for tag in gloss.tags:
report_file.print("%s" % tag, level=1)
report_file.print('')
def dump_synset(ss,
compact_gloss=False,
compact_tags=False,
more_compact=True,
report_file=None,
compact=True):
''' Print synset details for debugging purpose
Arguments:
ss -- Synset object to dump
compact_gloss -- Don't dump gloss tokens' details
compact_tags -- Don't dump tagged senses' details
more_compact -- Don't dump full details of synset
report_file -- Report file to write to
'''
if report_file is None:
report_file = TextReport() # Default to stdout
if more_compact:
report_file.header(
"Synset: %s (lemmas=%s | keys=%s)" %
(ss.sid.to_canonical(), ss.lemmas, ss.keys), 'h0')
else:
report_file.header("Synset: %s" % ss, 'h0')
for rgloss in ss.raw_glosses:
if more_compact:
if rgloss.cat != 'orig':
continue
report_file.print(rgloss)
gloss_count = itertools.count(1)
for gloss in ss.glosses:
if compact:
report_file.print("({cat}) {txt}".format(cat=gloss.cat,
txt=gloss.text()))
else:
report_file.print('')
report_file.header("Gloss #%s: %s" % (next(gloss_count), gloss),
'h2')
# Dump gloss items
if compact_gloss:
report_file.print("Tokens => %s" % gloss.get_gramwords(),
level=2)
else:
for item in gloss.items:
# print("\t%s - { %s }" % (uniquify(item.get_gramwords()), item))
report_file.print("%s - { %s }" %
(set(item.get_gramwords()), item),
level=2)
report_file.print(("-" * 10), level=1)
# Dump tags
if compact_tags:
report_file.print("Tags => %s" % gloss.get_tagged_sensekey(),
level=2)
else:
for tag in gloss.tags:
report_file.print("%s" % tag, level=1)
report_file.print('')
def omw_vs_gwn_def():
rp = TextReport("data/omw_gwn_report.txt")
rpdiff = TextReport("data/omw_gwn_diff.txt")
rptypo = TextReport("data/omw_gwn_typo.txt")
rpsn = TextReport("data/omw_gwn_sciname.txt")
c = Counter(TAGS.ORDER)
# ssids to compare
diff_ssids = []
ssids = read_diff_ssids()
if not ssids:
print("Generating synset ID list")
omw_ssids = set(get_omw_synsets())
gwn_ssids = set(get_gwn_synsets())
# only care about old GWN synsets
ssids = omw_ssids.intersection(gwn_ssids)
else:
print("Comparing {} synsets loaded from {}".format(len(ssids), ssid_filepath))
lang = 'eng'
with omw.ctx() as omw_ctx, gwn.ctx() as gwn_ctx:
print("Comparing {} synsets".format(len(ssids)))
for ss in list(ssids):
ss = str(ss)
c.count("total")
tags, odef, gdef = compare_synset(omw, gwn, ss, omw_ctx, gwn_ctx)
omwss = omw.get_synset(ss, ctx=omw_ctx)
tags_str = ' '.join('[{}]'.format(t.upper()) for t in tags)
if TAGS.DIFF in tags:
diff_ssids.append(ss)
# [FCB] why did we change?
gwnss = gwn.get_synset(ss, ctx=gwn_ctx)
glosses = gwn_ctx.gloss.select('surface = ?', (gwnss.definition,))
if glosses and len(glosses) > 1:
tags.add(TAGS.DUP)
ssids = [str(SynsetID.from_string(g.sid)) for g in glosses]
reason = "Not unique (Shared among {}) so OMW team changed it".format(' '.join(ssids))
else:
tags.add(TAGS.OMW)
defs = omw.sdef.select('synset=? and lang=?', (ss, lang))
usrs = {d.usr for d in defs if d.usr}
usrs_str = ', '.join(usrs) if usrs else "someone in NTU"
reason = "{} made this change.".format(usrs_str)
tags_str = ' '.join('[{}]'.format(t.upper()) for t in tags)
rpdiff.header("{} {}".format(tags_str, ss))
rpdiff.print("OMW: {}".format(omwss.definition))
rpdiff.print("GWN: {}".format(gdef))
rpdiff.print("Reason: {}".format(reason))
if TAGS.SCINAME in tags:
rpsn.header("{} {}".format(tags_str, ss))
rpsn.print("OMW: {}".format(omwss.definition))
rpsn.print("GWN: {}".format(gdef))
if TAGS.REP in tags or TAGS.TYPO in tags:
rptypo.header("{} {}".format(tags_str, ss))
rptypo.print("OMW: {}".format(omwss.definition))
rptypo.print("GWN: {}".format(gdef))
# master report
for tag in tags:
c.count(tag)
if not tags:
c.count(TAGS.IDENT)
rp.header("{} {}".format(tags_str, ss))
rp.print("OMW: {}".format(omwss.definition))
rp.print("GWN: {}".format(gdef))
# done
c.summarise(report=rp)
with open('data/omw_gwn_diff_ssids.txt', 'wt') as diff_ssid_file:
for ss in diff_ssids:
diff_ssid_file.write('{}\n'.format(ss))
