def transcripts_to_vocabulary(target, source, env):
word_counts = FrequencyList()
for fname in source:
with meta_open(fname.rstr()) as ifd:
for line in [x for x in ifd if not re.match(r"^\[.*\]\s*", x)]:
for tok in line.split():
word_counts[tok] = word_counts.get(tok, 0) + 1
with meta_open(target[0].rstr(), "w") as ofd:
ofd.write(word_counts.format())
return None
def plot_reduction(target, source, env):
args = source[-1].read()
bins = args["bins"]
with meta_open(source[-3].rstr()) as in_voc_fd, meta_open(source[-2].rstr()) as all_voc_fd:
in_vocabulary = FrequencyList(in_voc_fd).make_conservative()
other_vocabulary = FrequencyList(all_voc_fd).make_conservative()
all_vocabulary = other_vocabulary.join(in_vocabulary)
out_of_vocabulary = set([x for x in all_vocabulary.keys() if x not in in_vocabulary])
num_iv_types = len(in_vocabulary)
num_iv_tokens = sum([all_vocabulary.get(x, 0) for x in in_vocabulary])
num_types = len(all_vocabulary)
num_tokens = sum(all_vocabulary.values())
num_oov_types = num_types - num_iv_types
num_oov_tokens = num_tokens - num_iv_tokens
logging.info("%d/%d in-vocabulary types", num_iv_types, num_types)
logging.info("%d/%d in-vocabulary tokens", num_iv_tokens, num_tokens)
#pyplot.figure(figsize=(8 * 2, 7))
pyplot.figure(figsize=(8, 7))
for expansion_fname in source[0:-3]:
good_tokens = 0
good_types = 0
token_based = numpy.empty(shape=(bins + 1))
type_based = numpy.empty(shape=(bins + 1))
token_based[0] = float(0.0)
type_based[0] = float(0.0)
name = {"morph" : "just Morfessor",
"lm" : "reranking by ngrams",
"lm_avg" : "reranking by ngram average",
"lm_morph" : "reranking by boundary-ngrams",
}[os.path.splitext(os.path.basename(expansion_fname.rstr()))[0]]
method = os.path.dirname(expansion_fname.rstr()).split("/")[-1]
name = "%s - %s" % (method, name)
with meta_open(expansion_fname.rstr()) as expansion_fd:
expansions = [(w, p) for w, p in [x.strip().split() for x in expansion_fd]]
bin_size = len(expansions) / bins
for i in range(bins):
correct = [x for x in expansions[i*bin_size:(i+1)*bin_size] if x[0] in all_vocabulary]
good_types += len(correct)
good_tokens += sum([all_vocabulary.get(x[0], 0) for x in correct])
type_based[i + 1] = good_types
token_based[i + 1] = good_tokens
logging.info("%d recovered types", good_types)
logging.info("%d recovered tokens", good_tokens)
#pyplot.subplot(1, 2, 1)
logging.info("%s at %d, %d/%d recovered types", name, (type_based.shape[0] / 2) * bin_size, type_based[type_based.shape[0] / 2], num_oov_types)
pyplot.plot(100 * type_based / float(num_oov_types), label=name)
#pyplot.subplot(1, 2, 2)
#pyplot.plot(token_based, label=name)
#pyplot.subplot(1, 2, 1)
#pyplot.title("Type-based")
#pyplot.xlabel("Expansion threshold (in 1000s of words)")
pyplot.ylabel("% OOV reduction")
pyplot.legend(loc="lower right", fontsize=10)
pyplot.xticks([x * bin_size for x in range(11)], [(x * bin_size) / 10 for x in range(11)])
#print type_based.max()
yinc = float(type_based.max()) / 9
#yinc = 2409.0 / 9
#pyplot.yticks([x * yinc for x in range(10)], ["%d" % (int(100 * x * yinc / float(num_oov_types))) for x in range(10)])
yinc = 35.0 / 9
pyplot.yticks([x * yinc for x in range(10)], ["%d" % (x * yinc) for x in range(10)])
#pyplot.yticks([x * yinc for x in range(10)], ["%d/%d" % (int(100 * x * yinc / float(num_oov_types)), int(100 * x * yinc / float(num_iv_types))) for x in range(10)], fontsize=8)
pyplot.grid()
#pyplot.subplot(1, 2, 2)
#pyplot.title("Token-based")
pyplot.xlabel("1000s of words")
#pyplot.ylabel("%% OOV reduction/IV increase (%d initially OOV tokens)" % (num_oov_tokens))
#pyplot.legend(loc="lower right", fontsize=10)
#pyplot.xticks([x * bin_size for x in range(11)], [(x * bin_size) / 10 for x in range(11)])
#yinc = float(token_based.max()) / 9
#pyplot.yticks([x * yinc for x in range(10)], ["%d" % (int(100 * x * yinc / float(num_oov_tokens))) for x in range(10)])
#pyplot.yticks([x * yinc for x in range(10)], ["%d/%d" % (int(100 * x * yinc / float(num_oov_tokens)), int(100 * x * yinc / float(num_iv_tokens))) for x in range(10)], fontsize=8)
#pyplot.grid()
pyplot.savefig(target[0].rstr())
pyplot.cla()
pyplot.clf()
return None
