args = parser.parse_args()
categories = args.categories
if categories == ["all"]:
categories = category.ALL_CATEGORIES
elif categories == ["function"]:
categories = category.FUNCTION_CATEGORIES
elif categories == ["content"]:
categories = category.CONTENT_CATEGORIES
texts = PredictionTexts(args.text_numbers, filter_by=categories)
ngram_predictor = NgramPredictor(max(args.orders))
human_predictor = HumanPredictor()
ngram_probs = ngram_predictor.batch_predict(texts)
cloze_probs = human_predictor.batch_predict(texts)
# freqs = [target.frequency() for target in texts.target_words()]
def plot_hist(title, data, bins=50):
plt.hist(data, bins)
# plt.show()
plt.title("Histograma de %s" % title)
plt.savefig("plots/histogram_%s.png" % title)
plt.close()
plot_hist("ngram_prob", ngram_probs)
plot_hist("logit_ngram_prob", logit(ngram_probs))
plot_hist("cloze_prob", cloze_probs)
plot_hist("logit_cloze_prob", logit(cloze_probs))
