def visualize(batch_data, model):
i_datum = 0
# mod_layout_choice = model.module_layout_choices[i_datum]
# print model.apollo_net.blobs.keys()
# att_blob_name = "Find_%d_softmax" % (mod_layout_choice * 100 + 1)
#
datum = batch_data[i_datum]
question = (" ".join([QUESTION_INDEX.get(w) for w in datum.question[1:-1]]),)
preds = model.prediction_data[i_datum, :]
top = np.argsort(preds)[-5:]
top_answers = reversed([ANSWER_INDEX.get(p) for p in top])
# att_data = model.apollo_net.blobs[att_blob_name].data[i_datum,...]
# att_data = att_data.reshape((14, 14))
att_data = np.zeros((14, 14))
chosen_parse = datum.parses[model.layout_ids[i_datum]]
fields = [
question,
str(chosen_parse),
"<img src='../../%s'>" % datum.image_path,
att_data,
", ".join(top_answers),
", ".join([ANSWER_INDEX.get(a) for a in datum.answers]),
]
visualizer.show(fields)
def forward(data, model, config, train, vis):
model.reset()
# load batch data
max_len = max(len(d.question) for d in data)
max_layouts = max(len(d.layouts) for d in data)
channels, size, trailing = data[0].load_features().shape
assert trailing == 1
has_rel_features = data[0].load_rel_features() is not None
if has_rel_features:
rel_channels, size_1, size_2 = data[0].load_rel_features().shape
assert size_1 == size_2 == size
questions = np.ones((config.opt.batch_size, max_len)) * NULL_ID
features = np.zeros((config.opt.batch_size, channels, size, 1))
if has_rel_features:
rel_features = np.zeros((config.opt.batch_size, rel_channels, size, size))
else:
rel_features = None
layout_reprs = np.zeros((config.opt.batch_size, max_layouts, len(MODULE_INDEX) + 7))
for i, datum in enumerate(data):
questions[i, max_len - len(datum.question) :] = datum.question
features[i, ...] = datum.load_features()
if has_rel_features:
rel_features[i, ...] = datum.load_rel_features()
layout_reprs[i, ...] = featurize_layouts(datum, max_layouts)
layouts = [d.layouts for d in data]
# apply model
model.forward(
layouts,
layout_reprs,
questions,
features,
rel_features,
dropout=(train and config.opt.dropout),
deterministic=not train,
)
# extract predictions
if config.opt.multiclass:
pred_words = []
for i in range(model.prediction_data.shape[0]):
preds = model.prediction_data[i, :]
chosen = np.where(preds > 0.5)[0]
pred_words.append(set(ANSWER_INDEX.get(w) for w in chosen))
else:
pred_ids = np.argmax(model.prediction_data, axis=1)
pred_words = [ANSWER_INDEX.get(w) for w in pred_ids]
predictions = list()
for i in range(len(data)):
qid = data[i].id
answer = pred_words[i]
predictions.append({"question_id": qid, "answer": answer})
return predictions
def forward(data, model, config, train, vis):
model.reset()
# load batch data
max_len = max(len(d.question) for d in data)
max_layouts = max(len(d.layouts) for d in data)
channels, size, trailing = data[0].load_features().shape
assert trailing == 1
has_rel_features = data[0].load_rel_features() is not None
if has_rel_features:
rel_channels, size_1, size_2 = data[0].load_rel_features().shape
assert size_1 == size_2 == size
questions = np.ones((config.opt.batch_size, max_len)) * NULL_ID
features = np.zeros((config.opt.batch_size, channels, size, 1))
if has_rel_features:
rel_features = np.zeros(
(config.opt.batch_size, rel_channels, size, size))
else:
rel_features = None
layout_reprs = np.zeros(
(config.opt.batch_size, max_layouts, len(MODULE_INDEX) + 7))
for i, datum in enumerate(data):
questions[i, max_len - len(datum.question):] = datum.question
features[i, ...] = datum.load_features()
if has_rel_features:
rel_features[i, ...] = datum.load_rel_features()
layout_reprs[i, ...] = featurize_layouts(datum, max_layouts)
layouts = [d.layouts for d in data]
# apply model
model.forward(layouts,
layout_reprs,
questions,
features,
rel_features,
dropout=(train and config.opt.dropout),
deterministic=not train)
# extract predictions
if config.opt.multiclass:
pred_words = []
for i in range(model.prediction_data.shape[0]):
preds = model.prediction_data[i, :]
chosen = np.where(preds > 0.5)[0]
pred_words.append(set(ANSWER_INDEX.get(w) for w in chosen))
else:
pred_ids = np.argmax(model.prediction_data, axis=1)
pred_words = [ANSWER_INDEX.get(w) for w in pred_ids]
predictions = list()
for i in range(len(data)):
qid = data[i].id
answer = pred_words[i]
predictions.append({"question_id": qid, "answer": answer})
return predictions
def compute_acc(predictions, data, config):
score = 0.0
for prediction, datum in zip(predictions, data):
pred_answer = prediction["answer"]
if config.opt.multiclass:
answers = [set(ANSWER_INDEX.get(aa) for aa in a) for a in datum.answers]
else:
answers = [ANSWER_INDEX.get(a) for a in datum.answers]
matching_answers = [a for a in answers if a == pred_answer]
if len(answers) == 1:
score += len(matching_answers)
else:
score += min(len(matching_answers) / 3.0, 1.0)
score /= len(data)
return score
def compute_acc(predictions, data, config):
score = 0.0
for prediction, datum in zip(predictions, data):
pred_answer = prediction["answer"]
if config.opt.multiclass:
answers = [
set(ANSWER_INDEX.get(aa) for aa in a) for a in datum.answers
]
else:
answers = [ANSWER_INDEX.get(a) for a in datum.answers]
matching_answers = [a for a in answers if a == pred_answer]
if len(answers) == 1:
score += len(matching_answers)
else:
score += min(len(matching_answers) / 3.0, 1.0)
score /= len(data)
return score
def visualize(batch_data, model):
i_datum = 0
#mod_layout_choice = model.module_layout_choices[i_datum]
#print model.apollo_net.blobs.keys()
#att_blob_name = "Find_%d_softmax" % (mod_layout_choice * 100 + 1)
#
datum = batch_data[i_datum]
question = " ".join([QUESTION_INDEX.get(w) for w in datum.question[1:-1]]),
preds = model.prediction_data[i_datum, :]
top = np.argsort(preds)[-5:]
top_answers = reversed([ANSWER_INDEX.get(p) for p in top])
#att_data = model.apollo_net.blobs[att_blob_name].data[i_datum,...]
#att_data = att_data.reshape((14, 14))
att_data = np.zeros((14, 14))
chosen_parse = datum.parses[model.layout_ids[i_datum]]
fields = [
question,
str(chosen_parse),
"<img src='../../%s'>" % datum.image_path, att_data,
", ".join(top_answers),
", ".join([ANSWER_INDEX.get(a) for a in datum.answers])
]
visualizer.show(fields)
