def visualize(batch_data, model): i_datum = 0 # mod_layout_choice = model.module_layout_choices[i_datum] # print model.apollo_net.blobs.keys() # att_blob_name = "Find_%d_softmax" % (mod_layout_choice * 100 + 1) # datum = batch_data[i_datum] question = (" ".join([QUESTION_INDEX.get(w) for w in datum.question[1:-1]]),) preds = model.prediction_data[i_datum, :] top = np.argsort(preds)[-5:] top_answers = reversed([ANSWER_INDEX.get(p) for p in top]) # att_data = model.apollo_net.blobs[att_blob_name].data[i_datum,...] # att_data = att_data.reshape((14, 14)) att_data = np.zeros((14, 14)) chosen_parse = datum.parses[model.layout_ids[i_datum]] fields = [ question, str(chosen_parse), "<img src='../../%s'>" % datum.image_path, att_data, ", ".join(top_answers), ", ".join([ANSWER_INDEX.get(a) for a in datum.answers]), ] visualizer.show(fields)
def forward(data, model, config, train, vis): model.reset() # load batch data max_len = max(len(d.question) for d in data) max_layouts = max(len(d.layouts) for d in data) channels, size, trailing = data[0].load_features().shape assert trailing == 1 has_rel_features = data[0].load_rel_features() is not None if has_rel_features: rel_channels, size_1, size_2 = data[0].load_rel_features().shape assert size_1 == size_2 == size questions = np.ones((config.opt.batch_size, max_len)) * NULL_ID features = np.zeros((config.opt.batch_size, channels, size, 1)) if has_rel_features: rel_features = np.zeros((config.opt.batch_size, rel_channels, size, size)) else: rel_features = None layout_reprs = np.zeros((config.opt.batch_size, max_layouts, len(MODULE_INDEX) + 7)) for i, datum in enumerate(data): questions[i, max_len - len(datum.question) :] = datum.question features[i, ...] = datum.load_features() if has_rel_features: rel_features[i, ...] = datum.load_rel_features() layout_reprs[i, ...] = featurize_layouts(datum, max_layouts) layouts = [d.layouts for d in data] # apply model model.forward( layouts, layout_reprs, questions, features, rel_features, dropout=(train and config.opt.dropout), deterministic=not train, ) # extract predictions if config.opt.multiclass: pred_words = [] for i in range(model.prediction_data.shape[0]): preds = model.prediction_data[i, :] chosen = np.where(preds > 0.5)[0] pred_words.append(set(ANSWER_INDEX.get(w) for w in chosen)) else: pred_ids = np.argmax(model.prediction_data, axis=1) pred_words = [ANSWER_INDEX.get(w) for w in pred_ids] predictions = list() for i in range(len(data)): qid = data[i].id answer = pred_words[i] predictions.append({"question_id": qid, "answer": answer}) return predictions
def forward(data, model, config, train, vis): model.reset() # load batch data max_len = max(len(d.question) for d in data) max_layouts = max(len(d.layouts) for d in data) channels, size, trailing = data[0].load_features().shape assert trailing == 1 has_rel_features = data[0].load_rel_features() is not None if has_rel_features: rel_channels, size_1, size_2 = data[0].load_rel_features().shape assert size_1 == size_2 == size questions = np.ones((config.opt.batch_size, max_len)) * NULL_ID features = np.zeros((config.opt.batch_size, channels, size, 1)) if has_rel_features: rel_features = np.zeros( (config.opt.batch_size, rel_channels, size, size)) else: rel_features = None layout_reprs = np.zeros( (config.opt.batch_size, max_layouts, len(MODULE_INDEX) + 7)) for i, datum in enumerate(data): questions[i, max_len - len(datum.question):] = datum.question features[i, ...] = datum.load_features() if has_rel_features: rel_features[i, ...] = datum.load_rel_features() layout_reprs[i, ...] = featurize_layouts(datum, max_layouts) layouts = [d.layouts for d in data] # apply model model.forward(layouts, layout_reprs, questions, features, rel_features, dropout=(train and config.opt.dropout), deterministic=not train) # extract predictions if config.opt.multiclass: pred_words = [] for i in range(model.prediction_data.shape[0]): preds = model.prediction_data[i, :] chosen = np.where(preds > 0.5)[0] pred_words.append(set(ANSWER_INDEX.get(w) for w in chosen)) else: pred_ids = np.argmax(model.prediction_data, axis=1) pred_words = [ANSWER_INDEX.get(w) for w in pred_ids] predictions = list() for i in range(len(data)): qid = data[i].id answer = pred_words[i] predictions.append({"question_id": qid, "answer": answer}) return predictions
def compute_acc(predictions, data, config): score = 0.0 for prediction, datum in zip(predictions, data): pred_answer = prediction["answer"] if config.opt.multiclass: answers = [set(ANSWER_INDEX.get(aa) for aa in a) for a in datum.answers] else: answers = [ANSWER_INDEX.get(a) for a in datum.answers] matching_answers = [a for a in answers if a == pred_answer] if len(answers) == 1: score += len(matching_answers) else: score += min(len(matching_answers) / 3.0, 1.0) score /= len(data) return score
def compute_acc(predictions, data, config): score = 0.0 for prediction, datum in zip(predictions, data): pred_answer = prediction["answer"] if config.opt.multiclass: answers = [ set(ANSWER_INDEX.get(aa) for aa in a) for a in datum.answers ] else: answers = [ANSWER_INDEX.get(a) for a in datum.answers] matching_answers = [a for a in answers if a == pred_answer] if len(answers) == 1: score += len(matching_answers) else: score += min(len(matching_answers) / 3.0, 1.0) score /= len(data) return score
def visualize(batch_data, model): i_datum = 0 #mod_layout_choice = model.module_layout_choices[i_datum] #print model.apollo_net.blobs.keys() #att_blob_name = "Find_%d_softmax" % (mod_layout_choice * 100 + 1) # datum = batch_data[i_datum] question = " ".join([QUESTION_INDEX.get(w) for w in datum.question[1:-1]]), preds = model.prediction_data[i_datum, :] top = np.argsort(preds)[-5:] top_answers = reversed([ANSWER_INDEX.get(p) for p in top]) #att_data = model.apollo_net.blobs[att_blob_name].data[i_datum,...] #att_data = att_data.reshape((14, 14)) att_data = np.zeros((14, 14)) chosen_parse = datum.parses[model.layout_ids[i_datum]] fields = [ question, str(chosen_parse), "<img src='../../%s'>" % datum.image_path, att_data, ", ".join(top_answers), ", ".join([ANSWER_INDEX.get(a) for a in datum.answers]) ] visualizer.show(fields)