def predict_image(path, question, model=model, idx2word=idx2word, input_embedding=input_embedding): input_seq_len = 21 embedding_size = 50 X1 = get_feature(path) question = preprocess_text(question) padding = ['<pad>'] * (input_seq_len - len(question)) question = question + padding X2 = np.zeros((input_seq_len, embedding_size)) for i in range(input_seq_len): if question[i] not in input_embedding.keys(): question[i] = '<unk>' X2[i] = input_embedding[question[i]] X2 = torch.from_numpy(X2).float() X1 = X1.unsqueeze(0) X2 = X2.unsqueeze(0) if torch.cuda.is_available(): X1 = X1.cuda() X2 = X2.cuda() model = model.cuda() output = model(X1, X2) index = output.max(-1)[1].item() return idx2word[index]
def run_step(self, text): cur_sent = preprocess_text(text.strip(), lower=FLAGS.uncased) tokens, ids = self.sp.encode_ids(cur_sent) sent_len, diff_len = len(ids) - 1, FLAGS.seq_len - len(ids) input_ids = ids + [SEP_ID] * (diff_len - 1) + [ CLS_ID ] # cat_data = np.concatenate([inp, a_data, sep_array, b_data, sep_array, cls_array]) input_tokens = tokens + ["<sep>"] * (diff_len - 1) + ["<cls>"] input_mask = [1] + [0] * sent_len + [1] * diff_len segment_ids = [0] * (sent_len + 1) + [ 2 ] * diff_len # seg_id = ([0] * (reuse_len + a_data.shape[0]) + [0] + [1] * b_data.shape[0] + [1] + [2]) input_ids, input_tokens, input_mask, segment_ids = input_ids[:FLAGS. seq_len], input_tokens[: FLAGS . seq_len], input_mask[: FLAGS . seq_len], segment_ids[: FLAGS . seq_len] ''' logging.info("text: %s, seg_text: %s" % (text, " ".join([str(x) for x in tokens]))) logging.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) logging.info("input_mask: %s" % " ".join([str(x) for x in input_mask])) logging.info("segment_ids: %s" % " ".join([str(x) for x in segment_ids])) ''' il={'text':text,'seg_text':" ".join([str(x) for x in tokens]),'input_ids':" ".join([str(x) for x in input_ids]), \ 'input_mask':" ".join([str(x) for x in input_mask]),'segment_ids':" ".join([str(x) for x in segment_ids])} logging.info(json.dumps(il, ensure_ascii=False)) feed_dict = { self.input_ids: [input_ids], self.segment_ids: [segment_ids], self.input_mask: [input_mask] } fetch = self.sess.run( [self.output, self.attn_prob, self.attention_out], feed_dict) out_encode, atten_prob = fetch[0], fetch[1] #weight0 = normalization(self.cal_weight(out_encode, input_tokens)) weight_attn = normalization(self.weight_attenprob(atten_prob, tokens)) weight_idf = normalization(self.sp.cal_weight_idf(tokens[1:])) weight_lm = normalization(self.lm.cal_weight_lm(tokens[1:])) weight_rule = self.merge_weight([(weight_attn, 0.5), (weight_idf, 0.5), (weight_lm, 0.5)]) self.weight_attn, self.weight_idf, self.weight_lm = weight_attn, weight_idf, weight_lm sen2terms = [e for e in tokens[1:]] weightrank = self.rank_weight(sen2terms, weight_attn, weight_idf, weight_lm) weight_rank = normalization(weightrank) weight = self.merge_weight([(weight_rank, 0.7), (weight_rule, 0.0)]) # 0.6-0.4 wl = {'weight_rank':' '.join([str(k)+':'+str(v) for k, v in weight_rank]),'weight_rule':' '.join([str(k)+':'+str(v) for k, v in weight_rule]), \ 'weight': ' '.join([str(k) + ':' + str(v) for k, v in weight])} logging.info(json.dumps(wl, ensure_ascii=False)) return weight
def __getitem__(self, idx): feature, path, question, answer = self.image_features[idx] original_question = question if isinstance(feature, list): X1 = random.choice(feature) # Data Augmentation for training else: X1 = feature # No Data Augmentation for validation X1 = torch.Tensor(X1) question = preprocess_text(question) answer = preprocess_text(answer)[0] answer = change(answer) padding = ['<pad>'] * (self.config['input_seq_len'] - len(question)) question = question + padding assert len(question) == self.config[ 'input_seq_len'], "Length of question is %d" % len(question) X2 = np.zeros( (self.config['input_seq_len'], self.config['embedding_size'])) for i in range(self.config['input_seq_len']): if question[i] not in self.input_embedding.keys(): question[i] = '<unk>' X2[i] = self.input_embedding[question[i]] X2 = torch.from_numpy(X2).float() if answer not in self.word2idx.keys(): answer = '<unk>' if answer not in self.word_list: answer = '<unk>' Y = self.word2idx[answer] return X1, X2, Y
def cal_ndcg_train_data(topk=1): ndcg_sum = 0.0 matchObj = re.compile(r'(.+)\t([0-9]+)', re.M | re.I) qw = query_weight() text = [ e.strip().split("\t") for e in open("get_jdcv_data/label.data", encoding="utf8").readlines() if e.strip() ] for line in tqdm(text, total=len(text)): seg_line = [(preprocess_text(e.split(":")[0]), e.split(":")[1]) for e in line] sorted_seg_line = sorted(seg_line, key=lambda d: d[1], reverse=True) rel = { k: len(sorted_seg_line) - i - 1 for i, (k, v) in enumerate(sorted_seg_line) } query = " ".join([e[0] for e in seg_line]) dcg, idcg, ndcg = get_one_query_ndcg(qw, query, rel, topk) ndcg_sum += ndcg ndcg_avg = ndcg_sum / len(text) print("ndcg_avg@%d: %.3f" % (topk, ndcg_avg))
print('Loaded GloVe') ### Loading Image Features with open('dumps/train_features_vgg16.pkl', 'rb') as f: original_features = pickle.load(f) print('len(original_features) : ', len(original_features)) with open('dumps/val_features_vgg16.pkl', 'rb') as f: original_val_features = pickle.load(f) print('len(original_val_features) : ', len(original_val_features)) for i in range(len(original_features)): features, path, question, answer = original_features[i] answer = preprocess_text(answer) if (len(answer) == 2): train_features.append(original_features[i]) for i in range(len(original_val_features)): feature, path, question, answer = original_val_features[i] answer = preprocess_text(answer) if (len(answer) == 2): val_features.append(original_val_features[i]) print('len(train_features) : ', len(train_features)) print('len(val_features) : ', len(val_features)) print('\n----------------------------------------\n') question, answer = [], ['<unk>']