def choose_jm(lm, params, qb_location, num_globals):
qdb = QuestionDatabase(qb_location)
pages = qdb.questions_with_pages()
scores = defaultdict(float)
for ll in params:
for pp in sorted(pages, key=lambda k: len(pages[k]), reverse=True):
compare = (hash(pp) + 1) % num_globals
for qq in [x for x in pages[pp] if x.fold == "dev"]:
for ss in qq.text_lines():
lm[compare].set_jm_interp(ll)
text = list(lm[compare].tokenize_and_censor(ss["text"]))
try:
val = lm[compare].ll(text)
except OverflowError:
val = float("nan")
if isnan(val):
continue
else:
scores[ll] += val
print(scores, max(scores.values()))
print(scores)
return [x for x in scores if scores[x] == max(scores.values())][0]
def verbose(self, qb_location):
qdb = QuestionDatabase(qb_location)
pages = qdb.questions_with_pages()
import time
for pp in sorted(pages, key=lambda k: len(pages[k]), reverse=True):
need_title = True
compare = (hash(pp) + 1) % self._globals
for corpus in self._lm:
if not pp in self._lm[corpus]:
continue
for qq in [x for x in pages[pp] if x.fold == "dev"]:
if need_title:
print("--------------\t%s\t--------------" % pp)
need_title = False
for ss in qq.text_lines():
self.set_metadata(qq.page, qq.category, qq.qnum,
ss["sent"], 0, None, qq.fold)
start = time.time()
print("===============\t%s\t===============" % corpus)
print(self.vw_from_title(pp, ss["text"]))
text = list(self._lm[corpus][0].tokenize_and_censor(
ss["text"]))
sent = self._lm[corpus][pp].mean_ll(text)
background = \
self._lm[corpus][compare].mean_ll(text)
score = self.text_score(corpus, pp, text)
print("sent: ([%f - %f] - %f) / %f = %f" %
(sent, background, self._sent_mean[corpus],
self._sent_var[corpus], score))
for cc in self._lm[corpus][pp].\
ngram_chains(text):
ngram_score = self.ngram_score(corpus, pp, cc)
vv = self._lm[corpus][pp].mean_ll(cc)
background = \
self._lm[corpus][compare].mean_ll(cc)
print("ngram, %s: ([%f - %f] - %f) / %f = %f" %
(display_ngram(cc), vv, background,
self._ngram_mean[corpus][len(cc)],
self._ngram_var[corpus][len(cc)],
ngram_score))
print(
list(x if x in
self._lm[corpus][compare]._vocab else None
for x in cc))
print("TIME: %f" % (time.time() - start))
def _set_stats(self, corpus, lm, qb_location, max_pages):
sents = []
ngrams = defaultdict(list)
qdb = QuestionDatabase(qb_location)
pages = qdb.questions_with_pages()
print("Computing stats for %s from %i pages ..." % (corpus, max_pages))
page_count = 0
for pp in sorted(pages, key=lambda k: len(pages[k]), reverse=True):
compare = (hash(pp) + 1) % self._globals
page_count += 1
for qq in [x for x in pages[pp] if x.fold == "dev"]:
if max_pages > 0 and page_count > max_pages:
break
if page_count % 34 == 0:
print("%i\t%s" % (page_count, pp))
for ss in qq.text_lines():
if pp in lm:
text = list(lm[pp].tokenize_and_censor(ss["text"]))
sents.append(lm[pp].mean_ll(text) -
lm[compare].mean_ll(text))
for cc in lm[pp].ngram_chains(text):
ngrams[len(cc)].\
append(lm[pp].mean_ll(cc) -
lm[compare].mean_ll(cc))
print("done")
print("Sents", sents[:10])
self._sent_mean[corpus] = mean(sents)
self._sent_var[corpus] = var(sents)
print("Ngrams", ngrams[2][:10])
for ii in ngrams:
self._ngram_mean[corpus][ii] = mean(
list(x for x in ngrams[ii] if x > self._threshold))
self._ngram_var[corpus][ii] = var(
list(x for x in ngrams[ii] if x > self._threshold))
print("Stats for %s: SM: %f, SV: %f, NM: %f, NV: %f" %
(corpus, self._sent_mean[corpus], self._sent_var[corpus],
self._ngram_mean[corpus][2], self._ngram_var[corpus][2]))
parser.add_argument("--params",
default="data/deep/params.pkl",
help="Location of parameter pickle")
parser.add_argument("--vocab",
default="data/deep/deep_vocab.pkl",
help="Location of vocab pickle")
parser.add_argument("--ners",
default="data/common/ners.pkl",
help="Location of NER pickle")
flags = parser.parse_args()
import time
start = time.time()
questions = questions = QuestionDatabase("data/questions.db")
page_dict = {}
for page in questions.get_all_pages():
page_dict[page.lower().replace(' ', '_')] = page
ws = DeepExtractor(flags.classifier, flags.params, flags.vocab, flags.ners,
page_dict)
print("Startup: %f sec" % (time.time() - start))
tests = {}
tests[u"Tannhäuser (opera)"] = u"""He sought out the pope to
seek forgiveness of his sins, only to be told that just as the pope's staff
would never (*) blossom, his sins are never be forgiven. Three days later,
the pope's staff miraculously bore flowers. For 10 points--identify this
German folk hero, the subject of an opera by Wagner [VAHG-ner]."""
if __name__ == "__main__":
from util import flags
flags.define_string("title_index", None, "Pickle of all titles")
flags.define_string("label_path", None, "Where we write page associations")
flags.define_string("database", None, "Question database")
flags.define_string("performance_output", None,
"Where we write user performance")
flags.define_string("user", None, "User identifier")
flags.InitFlags()
seen = already_answered(flags.performance_output, flags.user)
al = ActiveLearner(None, flags.label_path)
print("Loading question db %s" % flags.database)
db = QuestionDatabase(flags.database)
pw = PerformanceWriter(flags.performance_output, flags.user)
tf = TitleFinder(open(flags.title_index))
questions = db.questions_by_tournament("High School Championship")
for qid in questions:
question = questions[qid]
if question.fold == "train" or qid in seen:
continue
choices = list(tf.query(question.answer))
# Get what and when the human answered
wp, idx, ans = get_answer(
[question.text[x] for x in sorted(question.text)], question.answer,
question.page)
c.execute(query)
answer_count = defaultdict(int)
for pp, in c:
answer_count[pp] += 1
query = 'select page, id, naqt, fold from questions where page != ""'
c = question_database.cursor()
c.execute(query)
print(list(x for x in answer_count if answer_count[x] >= kCOUNT_CUTOFF))
print(
len(list(x for x in answer_count if answer_count[x] >= kCOUNT_CUTOFF)))
# Load the DAN to generate guesses if they're missing from the database
deep = instantiate_feature("deep", QuestionDatabase(flags.question_db))
questions = {}
question_num = 0
for pp, ii, nn, ff in c:
if nn >= 0 or answer_count[pp] < kCOUNT_CUTOFF:
continue
question_num += 1
question = McScience(pp, ii, ff)
question.add_text(question_first_sentence(question_database, ii))
choices = question_top_guesses(question.text, deep, guess_database, ii,
pp, flags.num_choices)
question.add_choices(choices)
questions[ii] = question
if question_num % 100 == 0:
print(pp, ii, question_num)
parser.add_argument('--neg_weight',
type=float,
default=0.0,
help="Negative example weight")
parser.add_argument('--question_out',
type=str,
default='',
help="Where we write out questions for buzzer")
parser.add_argument('--finals',
type=str,
default='',
help="Where we write out answer after entire question")
parser.add_argument('--expo', type=str, default='', help="The expo file")
flags = parser.parse_args()
qdb = QuestionDatabase(flags.qbdb)
buzz = DictWriter(open(flags.buzzes, 'w'), fieldnames=kBUZZ_OUT)
buzz.writeheader()
final_out = DictWriter(open(flags.finals, 'w'),
fieldnames=["question", "answer"])
final_out.writeheader()
# Check file length
with open(flags.meta) as infile:
meta_lines = sum(1 for line in infile)
with open(flags.pred) as infile:
pred_lines = sum(1 for line in infile)
assert meta_lines == pred_lines, "Prediction and meta files mismatch" + \
"(%s: %i vs %s: %i)" % (flags.meta, meta_lines, flags.pred, pred_lines)
args.add_argument('--database',
type=str,
default='data/questions.db',
help='sqlite3 database of questions')
args.add_argument('--titles',
type=str,
default='data/wiki_index.pkl',
help='page title candiates')
args.add_argument('--labels',
type=str,
default='data/map/ans_to_wiki',
help='write page assignment answers')
args = args.parse_args()
# Open up the database
d = QuestionDatabase(args.database)
page_diversity = d.answer_map(normalize)
# Set up the active learner for writing assignments
al = ActiveLearner(None, args.labels)
existing_labels = set(x[0] for x in al.human_labeled())
# get the candidates we want to assign to pages
answers = d.unmatched_answers(existing_labels)
print(answers.keys()[:10])
# Open up the title finder
tf = TitleFinder(open(args.titles))
for ans, count in sorted(answers.items(),
key=lambda x: sum(x[1].values()),
print 'top@', top, 'accuracy: ', corr / len(probs)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='')
parser.add_argument('--question_db', type=str, default='data/questions.db')
parser.add_argument('--attribute', type=str, default='category')
parser.add_argument('--bigram_thresh', type=int, default=1000)
parser.add_argument("--output",
type=str,
default="data/classifier/",
help="Where we write output file")
flags = parser.parse_args()
questions = QuestionDatabase(flags.question_db)
bigram_filename = "%s/bigrams.pkl" % flags.output
if os.path.exists(bigram_filename):
bgset = pickle.load(open(bigram_filename, 'rb'))
print("Using previous bigrams")
else:
print("computing bigrams...")
bgset = compute_frequent_bigrams(flags.bigram_thresh, questions)
write_bigrams(bgset, bigram_filename)
train_classifier("%s/%s.pkl" % (flags.output, flags.attribute), bgset,
questions, flags.attribute)
evaluate("%s/%s.pkl" % (flags.output, flags.attribute), bgset, questions,
flags.attribute)
