def main():
train = read_data(BUZZER_TRAIN_FOLD)
valid = read_data(BUZZER_DEV_FOLD)
print('# train data: {}'.format(len(train)))
print('# valid data: {}'.format(len(valid)))
train_iter = chainer.iterators.SerialIterator(train, args.batch_size)
valid_iter = chainer.iterators.SerialIterator(valid,
args.batch_size,
repeat=False,
shuffle=False)
args.n_input = train[0][1][0].shape[0]
model = RNNBuzzer(args.n_input, args.n_layers, args.n_hidden,
args.n_output, args.dropout)
if args.gpu >= 0:
chainer.backends.cuda.get_device_from_id(args.gpu).use()
model.to_gpu()
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(1e-4))
updater = training.updaters.StandardUpdater(train_iter,
optimizer,
converter=convert_seq,
device=args.gpu)
trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=output_dir)
trainer.extend(
extensions.Evaluator(valid_iter,
model,
converter=convert_seq,
device=args.gpu))
record_trigger = training.triggers.MaxValueTrigger(
'validation/main/accuracy', (1, 'epoch'))
trainer.extend(extensions.snapshot_object(model, args.model_name),
trigger=record_trigger)
trainer.extend(extensions.LogReport())
trainer.extend(
extensions.PrintReport([
'epoch', 'main/loss', 'validation/main/loss', 'main/accuracy',
'validation/main/accuracy', 'elapsed_time'
]))
trainer.extend(extensions.ProgressBar())
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
# Run the training
trainer.run()
def main(model):
train = read_data(fold=BUZZER_TRAIN_FOLD)
valid = read_data(fold=BUZZER_DEV_FOLD)
print("# train data: {}".format(len(train)))
print("# valid data: {}".format(len(valid)))
train_iter = chainer.iterators.SerialIterator(train, 64)
valid_iter = chainer.iterators.SerialIterator(valid,
64,
repeat=False,
shuffle=False)
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(1e-4))
updater = training.updaters.StandardUpdater(train_iter,
optimizer,
converter=convert_seq,
device=0)
trainer = training.Trainer(updater, (20, "epoch"), out=model.model_dir)
trainer.extend(
extensions.Evaluator(valid_iter,
model,
converter=convert_seq,
device=0))
record_trigger = training.triggers.MaxValueTrigger(
"validation/main/accuracy", (1, "epoch"))
trainer.extend(extensions.snapshot_object(model, "buzzer.npz"),
trigger=record_trigger)
trainer.extend(extensions.LogReport())
trainer.extend(extensions.ProgressBar())
trainer.extend(
extensions.PrintReport([
"epoch",
"main/loss",
"validation/main/loss",
"main/accuracy",
"validation/main/accuracy",
"elapsed_time",
]))
if not os.path.isdir(model.model_dir):
os.mkdir(model.model_dir)
trainer.run()
def get_buzzes(model, fold=BUZZER_DEV_FOLD):
valid = read_data(fold)
print('# {} data: {}'.format(fold, len(valid)))
valid_iter = chainer.iterators.SerialIterator(valid,
64,
repeat=False,
shuffle=False)
predictions = []
buzzes = dict()
for batch in tqdm(valid_iter):
qids, vectors, labels, positions = list(map(list, zip(*batch)))
batch = convert_seq(batch, device=0)
preds = model.predict(batch['xs'], softmax=True)
preds = [p.tolist() for p in preds]
predictions.extend(preds)
for i, qid in enumerate(qids):
buzzes[qid] = []
for pos, pred in zip(positions[i], preds[i]):
buzzes[qid].append((pos, pred))
buzzes[qid] = list(map(list, zip(*buzzes[qid])))
buzz_dir = os.path.join(model.model_dir, '{}_buzzes.pkl'.format(fold))
with open(buzz_dir, 'wb') as f:
pickle.dump(buzzes, f)
return buzzes
def get_buzzes(model, fold=BUZZER_DEV_FOLD):
valid = read_data(fold)
print('# {} data: {}'.format(fold, len(valid)))
valid_iter = chainer.iterators.SerialIterator(
valid, 64, repeat=False, shuffle=False)
predictions = []
buzzes = dict()
for batch in tqdm(valid_iter):
qids, vectors, labels, positions = list(map(list, zip(*batch)))
batch = convert_seq(batch, device=0)
preds = model.predict(batch['xs'], softmax=True)
preds = [p.tolist() for p in preds]
predictions.extend(preds)
for i, qid in enumerate(qids):
buzzes[qid] = []
for pos, pred in zip(positions[i], preds[i]):
buzzes[qid].append((pos, pred))
buzzes[qid] = list(map(list, zip(*buzzes[qid])))
buzz_dir = os.path.join(model.model_dir,
'{}_buzzes.pkl'.format(fold))
with open(buzz_dir, 'wb') as f:
pickle.dump(buzzes, f)
return buzzes
def main(model):
train = read_data(BUZZER_TRAIN_FOLD)
valid = read_data(BUZZER_DEV_FOLD)
print('# train data: {}'.format(len(train)))
print('# valid data: {}'.format(len(valid)))
train_iter = chainer.iterators.SerialIterator(train, 64)
valid_iter = chainer.iterators.SerialIterator(
valid, 64, repeat=False, shuffle=False)
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(1e-4))
updater = training.updaters.StandardUpdater(
train_iter, optimizer,
converter=convert_seq, device=0)
trainer = training.Trainer(updater, (20, 'epoch'),
out=model.model_dir)
trainer.extend(extensions.Evaluator(
valid_iter, model,
converter=convert_seq, device=0))
record_trigger = training.triggers.MaxValueTrigger(
'validation/main/accuracy', (1, 'epoch'))
trainer.extend(extensions.snapshot_object(
model, 'buzzer.npz'),
trigger=record_trigger)
trainer.extend(extensions.LogReport())
trainer.extend(extensions.PrintReport(
['epoch', 'main/loss', 'validation/main/loss',
'main/accuracy', 'validation/main/accuracy', 'elapsed_time']))
trainer.extend(extensions.ProgressBar())
if not os.path.isdir(model.model_dir):
os.mkdir(model.model_dir)
# Run the training
trainer.run()
def main():
fold = BUZZER_DEV_FOLD
# load questions
print('loading questions')
questions = QuizBowlDataset(buzzer_train=True).questions_by_fold()
questions = questions[fold]
# load guesser outputs
print('loading guesser outputs')
guesses = read_data(fold)
guesses = {x[0]: x for x in guesses}
# load buzzer outputs
print('loading buzzer outputs')
buzz_dir = os.path.join(buzzes_dir.format(fold))
with open(buzz_dir, 'rb') as f:
buzzes = pickle.load(f)
# load protobowl records
print('loading protobowl records')
df, _ = load_protobowl()
record_groups = df.groupby('qid')
metrics = [_protobowl_scores, _curve_scores]
pool = Pool(8)
worker = partial(run_all_metrics, guesses, buzzes, record_groups, metrics)
scores = pool.map(worker, questions)
all_scores = list(map(list, zip(*scores)))
protobowl_scores = all_scores[0]
protobowl_scores = list(map(list, zip(*protobowl_scores)))
protobowl_scores = [[x for x in s if x is not None]
for s in protobowl_scores]
print([np.mean(s) for s in protobowl_scores])
curve_scores = all_scores[1]
curve_scores = list(map(list, zip(*curve_scores)))
curve_scores = [[x for x in s if x is not None] for s in curve_scores]
print([np.mean(s) for s in curve_scores])
def main():
fold = BUZZER_DEV_FOLD
# load questions
print('loading questions')
questions = QuizBowlDataset(buzzer_train=True).questions_by_fold()
questions = questions[fold]
# load guesser outputs
print('loading guesser outputs')
guesses = read_data(fold)
guesses = {x[0]: x for x in guesses}
# load buzzer outputs
print('loading buzzer outputs')
buzz_dir = os.path.join(buzzes_dir.format(fold))
with open(buzz_dir, 'rb') as f:
buzzes = pickle.load(f)
# load protobowl records
print('loading protobowl records')
df, _ = load_protobowl()
record_groups = df.groupby('qid')
metrics = [_protobowl_scores, _curve_scores]
pool = Pool(8)
worker = partial(run_all_metrics, guesses, buzzes, record_groups, metrics)
scores = pool.map(worker, questions)
all_scores = list(map(list, zip(*scores)))
protobowl_scores = all_scores[0]
protobowl_scores = list(map(list, zip(*protobowl_scores)))
protobowl_scores = [[x for x in s if x is not None] for s in protobowl_scores]
print([np.mean(s) for s in protobowl_scores])
curve_scores = all_scores[1]
curve_scores = list(map(list, zip(*curve_scores)))
curve_scores = [[x for x in s if x is not None] for s in curve_scores]
print([np.mean(s) for s in curve_scores])
def eval(fold=BUZZER_DEV_FOLD):
if not os.path.isdir(report_dir):
os.mkdir(report_dir)
valid = read_data(fold)
print('# {} data: {}'.format(fold, len(valid)))
valid_iter = chainer.iterators.SerialIterator(valid,
args.batch_size,
repeat=False,
shuffle=False)
args.n_input = valid[0][1][0].shape[0]
model = RNNBuzzer(args.n_input, args.n_layers, args.n_hidden,
args.n_output, args.dropout)
chainer.serializers.load_npz(args.model_path, model)
if args.gpu >= 0:
chainer.backends.cuda.get_device_from_id(args.gpu).use()
model.to_gpu()
predictions = []
buzzes = dict()
for batch in tqdm(valid_iter):
qids, vectors, labels, positions = list(map(list, zip(*batch)))
batch = convert_seq(batch, device=args.gpu)
preds = model.predict(batch['xs'], softmax=True)
preds = [p.tolist() for p in preds]
predictions.extend(preds)
for i in range(len(qids)):
buzzes[qids[i]] = []
for pos, pred in zip(positions[i], preds[i]):
buzzes[qids[i]].append((pos, pred))
buzzes[qids[i]] = list(map(list, zip(*buzzes[qids[i]])))
buzz_dir = os.path.join(buzzes_dir.format(fold))
with open(buzz_dir, 'wb') as f:
pickle.dump(buzzes, f)
results = dict()
for example_idx in range(len(valid)):
qid, vectors, labels, positions = valid[example_idx]
preds = predictions[example_idx]
q_len = positions[-1]
for i, pos in enumerate(positions):
rel_pos = int(100 * pos / q_len)
if rel_pos not in results:
results[rel_pos] = []
results[rel_pos].append((labels[i], preds[i][1]))
freq = {'x': [], 'y': [], 'type': []}
for k, rs in results.items():
rs, scores = list(map(list, zip(*rs)))
freq['x'].append(k / 100)
freq['y'].append(sum(rs) / len(rs))
freq['type'].append('acc')
freq['x'].append(k / 100)
freq['y'].append(sum(x > 0.5 for x in scores) / len(scores))
freq['type'].append('0.5')
freq['x'].append(k / 100)
freq['y'].append(sum(x > 0.3 for x in scores) / len(scores))
freq['type'].append('0.3')
freq['x'].append(k / 100)
freq['y'].append(sum(x > 0.7 for x in scores) / len(scores))
freq['type'].append('0.7')
freq_df = pd.DataFrame(freq)
p0 = ggplot(freq_df) + geom_smooth(aes(x='x', y='y', color='type'))
p0.save(os.path.join(report_dir, '{}_acc_buzz.pdf'.format(fold)))
stack_freq = {'x': [], 'y': [], 'type': []}
threshold = 0.5
for k, rs in results.items():
num = len(rs)
only_oracle = sum((c == 1 and b <= threshold) for c, b in rs)
only_buzzer = sum((c == 0 and b > threshold) for c, b in rs)
both = sum((c == 1 and b > threshold) for c, b in rs)
neither = sum((c == 0 and b <= threshold) for c, b in rs)
stack_freq['x'].append(k / 100)
stack_freq['y'].append(only_oracle / num)
stack_freq['type'].append('only_oracle')
stack_freq['x'].append(k / 100)
stack_freq['y'].append(only_buzzer / num)
stack_freq['type'].append('only_buzzer')
stack_freq['x'].append(k / 100)
stack_freq['y'].append(both / num)
stack_freq['type'].append('both')
stack_freq['x'].append(k / 100)
stack_freq['y'].append(neither / num)
stack_freq['type'].append('neither')
stack_freq_df = pd.DataFrame(stack_freq)
p1 = ggplot(stack_freq_df) + geom_area(aes(x='x', y='y', fill='type'))
p1.save(os.path.join(report_dir, '{}_stack_area.pdf'.format(fold)))