def evaluate(self, queue, dataset, global_step=None, generate_results=False, tag=''): log.info("Evaluate Phase") self.sess.run(self.eval_reset) iter_length = int(len(dataset) / self.model.batch_size + 1) for i in range(iter_length): loss, pred, logit, sim, input_gate = self.eval_single_step(queue) if i%50 == 0: log.infov("{}/{}".format(i, iter_length)) total_acc = self.sess.run([self.accuracy]) log.infov("[MC] total accuracy: {} ".format(total_acc) + tag)
def evaluate(self, queue, dataset, global_step=None, generate_results=False, tag=''): log.info("Evaluate Phase") batch_size = self.model.batch_size dataset_length = len(dataset) iter_num = int(dataset_length / batch_size) results = [] iter_length = int(dataset_length / self.model.batch_size + 1) scores = [] margin_mat = np.zeros([dataset_length, dataset_length]) print('Iteration number ', iter_num) for i in range(iter_num): for j in range(iter_num): loss, logit, output_score = self.eval_single_step(queue) margin_mat[i * batch_size:(i + 1) * batch_size, j * batch_size:(j + 1) * batch_size] = output_score if i % 5 == 0 and j % 5 == 0: ii = int(i / 5) jj = int(j / 5) if i % 10 == 0: log.infov("{}/{}, margin-loss: {}".format( i, iter_length, loss)) acc = np.mean(np.diagonal(margin_mat)) rank_list = [] for i in range(dataset_length): col = -margin_mat[i, :] order = col.argsort() ranks = order.argsort() rank_list.append(ranks[i]) c = [x for x in rank_list if x < 1] c5 = [x for x in rank_list if x < 5] c10 = [x for x in rank_list if x < 10] medr = np.median(rank_list) log.infov("[RET] R@1: {}, R@5: {}, R@10: {}, medr: {medr:.4f}".format( len(c), len(c5), len(c10), medr=medr)) log.infov("[RET] total accuracy: {acc:.5f}".format(acc=np.sum(acc))) if generate_results: drive_dir = '/content/drive/My Drive/Graduation Project/Output/checkpoint_lsmdc_InceptionResNet_squeeze_dropout_1024/' with open(drive_dir + 'evaluate_log.tsv', 'a') as f: f.write( '[RET] Step [{}]\t, R@1: {}\t, R@5: {}\t, R@10: {}\t, medr: {}\t, acc: {}\n' .format(global_step, len(c), len(c5), len(c10), medr, np.sum(acc)))
def read_feat_from_hdf5(self): if self.image_feature_net.lower() == 'resnet': if self.layer.lower() == 'pool5': if self.wav_data: feature_file = os.path.join(VIDEO_FEATURE_DIR, self.image_feature_net.upper() + "_" + self.layer.lower() + "wav.hdf5") else: feature_file = os.path.join(VIDEO_FEATURE_DIR, self.image_feature_net.upper() + "_" + self.layer.lower() + ".hdf5") assert_exists(feature_file) log.info("Load %s hdf5 file : %s", self.image_feature_net.upper(), feature_file) if self.more_data: self.msr_h5 = h5py.File(os.path.join(VIDEO_FEATURE_DIR,"MSR_RESNET_pool5.hdf5")) return h5py.File(feature_file, 'r')
def evaluate(self, queue, dataset,generate_results,tag): log.info("Evaluate Phase") iter_length = int(len(dataset) / self.model.batch_size + 1) self.sess.run(self.eval_reset) total_input_gate = [] for i in range(iter_length): loss, predictions, target_indices, logits, ids, input_gate = self.eval_single_step(queue) for j,key in enumerate(ids): total_input_gate.append((str(dataset.idx2word[predictions[j]]), input_gate[0][j],input_gate[1][j],input_gate[2][j],input_gate[3][j])) if i%100 == 0: target_word = dataset.idx2word[target_indices[0]] output_word = dataset.idx2word[predictions[0]] log.infov("[FIB {step:3d}/{total_length:3d}] target: {target}, prediction: {prediction}".format( step=i, total_length=iter_length, target=target_word, prediction=output_word)) total_acc = self.sess.run(self.accuracy) log.infov("[FIB] total accurycy: {acc:.5f}".format(acc=total_acc))
def evaluate(self, queue, dataset, global_step=None, generate_results=False, tag=''): log.info("Evaluate Phase") batch_size = self.model.batch_size iter_num = int(1000 / batch_size) results = [] iter_length = int(len(dataset) / self.model.batch_size + 1) scores = [] margin_mat = np.zeros([1000, 1000]) for i in range(iter_num): for j in range(iter_num): loss, logit, output_score = self.eval_single_step(queue) margin_mat[i * batch_size:(i + 1) * batch_size, j * batch_size:(j + 1) * batch_size] = output_score if i % 5 == 0 and j % 5 == 0: ii = int(i / 5) jj = int(j / 5) if i % 10 == 0: log.infov("{}/{}, margin-loss: {}".format( i, iter_length, loss)) acc = np.mean(np.diagonal(margin_mat)) rank_list = [] for i in range(1000): col = -margin_mat[i, :] order = col.argsort() ranks = order.argsort() rank_list.append(ranks[i]) c = [x for x in rank_list if x < 1] c5 = [x for x in rank_list if x < 5] c10 = [x for x in rank_list if x < 10] medr = np.median(rank_list) log.infov("[RET] R@1: {}, R@5: {}, R@10: {}, medr : {medr:.4f}".format( len(c), len(c5), len(c10), medr=medr)) log.infov("[RET] total accuracy: {acc:.5f}".format(acc=np.sum(acc))) if generate_results: with open('./checkpoint/evaluate_log.tsv', 'a') as f: f.write('[{}]\t{}\t{}\t{}\t{}\t{}\n'.format( global_step, len(c), len(c5), len(c10), medr, np.sum(acc)))
def test(self, queue, dataset): log.info("Testing Phase") batch_size = self.model.batch_size print('batch size = ', batch_size) dataset_length = len(dataset) iter_num = int(dataset_length/batch_size) results = [] iter_length = int(dataset_length / self.model.batch_size + 1) margin_mat = np.zeros([dataset_length, dataset_length]) print('Testing on {} videos'.format(dataset_length)) for i in range(iter_num): for j in range(iter_num): loss, logit, output_score = self.test_single_step(queue) margin_mat[i*batch_size:(i+1)*batch_size, j*batch_size:(j+1)*batch_size] = output_score print('Scores = ', margin_mat[:,0]) scores = margin_mat[:,0] ranks = scores.argsort() print('Ranks = ', ranks); print('Finished Testing on {} videos'.format(dataset_length)) return ranks
def build_word_vocabulary(self): word_matrix_path = os.path.join(VOCABULARY_DIR, 'common_word_matrix.hkl') assert_exists(word_matrix_path) word2idx_path = os.path.join(VOCABULARY_DIR, 'common_word_to_index.hkl') assert_exists(word2idx_path) idx2word_path = os.path.join(VOCABULARY_DIR, 'common_index_to_word.hkl') assert_exists(idx2word_path) with open(word_matrix_path, 'r') as f: self.word_matrix = hkl.load(f) log.info("Load word_matrix from hkl file : %s", word_matrix_path) with open(word2idx_path, 'r') as f: self.word2idx = hkl.load(f) log.info("Load word2idx from hkl file : %s", word2idx_path) with open(idx2word_path, 'r') as f: self.idx2word = hkl.load(f) log.info("Load idx2word from hkl file : %s", idx2word_path)
def main(argv): model_config = ModelConfig() train_config = TrainConfig() drive_dir = '/content/drive/My Drive/Graduation Project/Output/checkpoint_lsmdc_InceptionResNet_squeeze_dropout/' base_dir = os.path.join(drive_dir, train_config.train_tag + "_" + FLAGS.tag) checkpoint_dir = os.path.join(base_dir, "model.ckpt") logits_dir = os.path.join(base_dir, "logits_") if not os.path.exists(base_dir): os.mkdir(base_dir) # train_dataset = DatasetLSMDC( dataset_name='train', image_feature_net=model_config.image_feature_net, layer=model_config.layer, max_length=model_config.caption_length, max_vid_length=model_config.video_steps, max_n_videos=None, data_type=train_config.train_tag, attr_length=model_config.attr_length, wav_data=model_config.wav_data) validation_dataset = DatasetLSMDC( dataset_name='validation', image_feature_net=model_config.image_feature_net, layer=model_config.layer, max_length=model_config.caption_length, max_vid_length=model_config.video_steps, max_n_videos=None, data_type=train_config.train_tag, attr_length=model_config.attr_length, wav_data=model_config.wav_data) train_dataset.build_word_vocabulary() validation_dataset.share_word_vocabulary_from(train_dataset) if train_config.train_tag == 'RET': model_config.batch_size = model_config.ret_batch_size train_iter = train_dataset.batch_iter(train_config.num_epochs, model_config.batch_size) train_queue = BatchQueue(train_iter, name='train') if train_config.train_tag == 'RET': val_queue = BatchQueue(validation_dataset.batch_tile( 20 * train_config.num_epochs, model_config.batch_size), name='validation') else: val_iter = validation_dataset.batch_iter(20 * train_config.num_epochs, model_config.batch_size, shuffle=False) val_queue = BatchQueue(val_iter, name='validation') train_queue.start_threads() val_queue.start_threads() g = tf.Graph() with g.as_default(): global session, model, trainer tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True session = tf.Session(graph=g, config=tf_config) model = MODELS[train_config.train_tag](model_config, train_dataset.word_matrix) log.info("Build the model...") model.build_model(**model.get_placeholder()) trainer = MODEL_TRAINERS[train_config.train_tag](train_config, model, session) steps_in_epoch = int(np.ceil(len(train_dataset) / model.batch_size)) saver = tf.train.Saver(max_to_keep=10) if train_config.load_from_ckpt is not None: log.info("Restoring parameter from {}".format( train_config.load_from_ckpt)) session.run(tf.global_variables_initializer()) saver.restore(session, train_config.load_from_ckpt) else: session.run(tf.global_variables_initializer()) for step in range(train_config.max_steps): skip = False if step <= train_config.last_step_taken: skip = True if step % 1000 == 0: print("Before Run single step = ", step) step_result = trainer.run_single_step(skip, queue=train_queue, is_train=True) if step % 1000 == 0: print("After Run single step = ", step) if not skip: if step % train_config.steps_per_logging == 0: step_result['steps_in_epoch'] = steps_in_epoch trainer.log_step_message(**step_result) if step % train_config.steps_per_evaluate == 0 or train_config.print_evaluate: trainer.evaluate(queue=val_queue, dataset=validation_dataset, global_step=step, generate_results=True, tag=FLAGS.tag) if step % (2 * train_config.steps_per_evaluate) == 0: print("SAVE MODEL" + FLAGS.tag) saver.save(session, checkpoint_dir, global_step=step) if step == train_config.max_steps - 1: trainer.evaluate(queue=val_queue, dataset=validation_dataset, global_step=step, generate_results=True, tag=FLAGS.tag) print("SAVE MODEL" + FLAGS.tag) saver.save(session, checkpoint_dir, global_step=step) print("Finished Epoch")
def start_threads(self): log.info("Start {} dataset Queue".format(self.name)) self.thread = threading.Thread(target=self.thread_main, args=()) self.thread.daemon = True self.thread.start()