def evaluate(): tf.set_random_seed(0) # for reproducibility # Write json of flags model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json") if not os.path.exists(model_flags_path): raise IOError(("Cannot find file %s. Did you run train.py on the same " "--train_dir?") % model_flags_path) flags_dict = json.loads(open(model_flags_path).read()) with tf.Graph().as_default(): # convert feature_names and feature_sizes to lists of values feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes( flags_dict["feature_names"], flags_dict["feature_sizes"]) if flags_dict["frame_features"]: reader = readers.YT8MFrameFeatureReader( feature_names=feature_names, feature_sizes=feature_sizes) else: reader = readers.YT8MAggregatedFeatureReader( feature_names=feature_names, feature_sizes=feature_sizes) model = find_class_by_name(flags_dict["model"], [frame_level_models, video_level_models])() label_loss_fn = find_class_by_name(flags_dict["label_loss"], [losses])() if FLAGS.eval_data_pattern is "": raise IOError("'eval_data_pattern' was not specified. " + "Nothing to evaluate.") build_graph(reader=reader, model=model, eval_data_pattern=FLAGS.eval_data_pattern, label_loss_fn=label_loss_fn, num_readers=FLAGS.num_readers, batch_size=FLAGS.batch_size) logging.info("built evaluation graph") video_id_batch = tf.get_collection("video_id_batch")[0] prediction_batch = tf.get_collection("predictions")[0] label_batch = tf.get_collection("labels")[0] loss = tf.get_collection("loss")[0] summary_op = tf.get_collection("summary_op")[0] saver = tf.train.Saver(tf.global_variables()) summary_writer = tf.summary.FileWriter(FLAGS.train_dir, graph=tf.get_default_graph()) evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k) last_global_step_val = -1 with tf.device("/gpu:0"): while True: last_global_step_val = evaluation_loop( video_id_batch, prediction_batch, label_batch, loss, summary_op, saver, summary_writer, evl_metrics, last_global_step_val) if FLAGS.run_once: break
def evaluate(): tf.set_random_seed(0) # for reproducibility with tf.Graph().as_default(): # convert feature_names and feature_sizes to lists of values feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes( FLAGS.feature_names, FLAGS.feature_sizes) # prepare a reader for each single model prediction result all_readers = [] all_patterns = FLAGS.eval_data_patterns all_patterns = map(lambda x: x.strip(), all_patterns.strip().strip(",").split(",")) for i in xrange(len(all_patterns)): reader = readers.EnsembleReader(feature_names=feature_names, feature_sizes=feature_sizes) all_readers.append(reader) input_reader = None input_data_pattern = None if FLAGS.input_data_pattern is not None: input_reader = readers.EnsembleReader(feature_names=["input"], feature_sizes=[1024 + 128]) input_data_pattern = FLAGS.input_data_pattern # find the model model = find_class_by_name(FLAGS.model, [ensemble_level_models])() label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])() if FLAGS.eval_data_patterns is "": raise IOError("'eval_data_patterns' was not specified. " + "Nothing to evaluate.") build_graph(all_readers=all_readers, input_reader=input_reader, all_eval_data_patterns=all_patterns, input_data_pattern=input_data_pattern, model=model, label_loss_fn=label_loss_fn, batch_size=FLAGS.batch_size) logging.info("built evaluation graph") video_id_batch = tf.get_collection("video_id_batch")[0] prediction_batch = tf.get_collection("predictions")[0] label_batch = tf.get_collection("labels")[0] loss = tf.get_collection("loss")[0] summary_op = tf.get_collection("summary_op")[0] saver = tf.train.Saver(tf.global_variables()) summary_writer = tf.summary.FileWriter(FLAGS.train_dir, graph=tf.get_default_graph()) evl_metrics = eval_util.EvaluationMetrics(FLAGS.num_classes, FLAGS.top_k) last_global_step_val = -1 last_global_step_val = evaluation_loop(video_id_batch, prediction_batch, label_batch, loss, summary_op, saver, summary_writer, evl_metrics, last_global_step_val)
def evaluate(): tf.set_random_seed(0) # for reproducibility with tf.Graph().as_default(): # convert feature_names and feature_sizes to lists of values feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes( FLAGS.feature_names, FLAGS.feature_sizes) if FLAGS.frame_features: reader = readers.YT8MFrameFeatureReader( feature_names=feature_names, feature_sizes=feature_sizes) else: reader = readers.YT8MAggregatedFeatureReader( feature_names=feature_names, feature_sizes=feature_sizes) model = find_class_by_name(FLAGS.model, [frame_level_models, video_level_models])() label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])() if FLAGS.eval_data_pattern is "": raise IOError("'eval_data_pattern' was not specified. " + "Nothing to evaluate.") build_graph(reader=reader, model=model, eval_data_pattern=FLAGS.eval_data_pattern, label_loss_fn=label_loss_fn, num_readers=FLAGS.num_readers, batch_size=FLAGS.batch_size) logging.info("built evaluation graph") video_id_batch = tf.get_collection("video_id_batch")[0] prediction_batch = tf.get_collection("predictions")[0] ### Newly # coarse_prediction_batch = tf.get_collection("coarse_predictions")[0] # coarse_label_batch = tf.get_collection("coarse_labels")[0] # coarse_loss = tf.get_collection("coarse_loss")[0] ### label_batch = tf.get_collection("labels")[0] loss = tf.get_collection("loss")[0] summary_op = tf.get_collection("summary_op")[0] saver = tf.train.Saver(tf.global_variables()) summary_writer = tf.summary.FileWriter(FLAGS.train_dir, graph=tf.get_default_graph()) evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k) last_global_step_val = -1 while True: last_global_step_val = evaluation_loop(video_id_batch, prediction_batch, label_batch, loss, summary_op, saver, summary_writer, evl_metrics, last_global_step_val) ### Newly # coarse_prediction_batch, coarse_label_batch, coarse_loss) ### if FLAGS.run_once: break
def evaluate(preds, labels, loss): evl_metrics = eval_util.EvaluationMetrics(3862, 20) evl_metrics.clear() iteration_info_dict = evl_metrics.accumulate(preds, labels, loss) # calculate the metrics for the entire epoch epoch_info_dict = evl_metrics.get() return epoch_info_dict['gap']
def cal_gap(self): self.predict() evl_metrics = eval_util.EvaluationMetrics(4716, self.top_k) predictions_val = ensemble(self.pred_list, self.weights) print predictions_val.shape iteration_info_dict = evl_metrics.accumulate(predictions_val, self.labels_val, np.zeros(predictions_val.shape[0])) epoch_info_dict = evl_metrics.get() print(("GAP@%d:" %self.top_k) + str(epoch_info_dict['gap']))
def evaluate(): tf.set_random_seed(0) # for reproducibility with tf.Graph().as_default(): if FLAGS.use_mnist: reader = readers.MnistReader() else: reader = readers.FaceReader() generator_model = find_class_by_name(FLAGS.generator_model, [models])() discriminator_model = find_class_by_name(FLAGS.discriminator_model, [models])() label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])() if FLAGS.eval_data_pattern is "": raise IOError("'eval_data_pattern' was not specified. " + "Nothing to evaluate.") build_graph(reader=reader, generator_model=generator_model, discriminator_model=discriminator_model, eval_data_pattern=FLAGS.eval_data_pattern, label_loss_fn=label_loss_fn, num_readers=FLAGS.num_readers, batch_size=FLAGS.batch_size) logging.info("built evaluation graph") p_fake_batch = tf.get_collection("p_for_fake")[0] p_real_batch = tf.get_collection("p_for_data")[0] G_loss = tf.get_collection("G_loss")[0] D_loss = tf.get_collection("D_loss")[0] noise_input = tf.get_collection("noise_input_placeholder")[0] summary_op = tf.get_collection("summary_op")[0] saver = tf.train.Saver(tf.global_variables()) summary_writer = tf.summary.FileWriter(FLAGS.train_dir, graph=tf.get_default_graph()) evl_metrics = eval_util.EvaluationMetrics() last_global_step_val = -1 while True: last_global_step_val = evaluation_loop(p_fake_batch, p_real_batch, G_loss, D_loss, noise_input, summary_op, saver, summary_writer, evl_metrics, last_global_step_val, FLAGS.batch_size) if FLAGS.run_once: break
def evaluate(): tf.set_random_seed(0) # for reproducibility # Write json of flags model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json") if not file_io.file_exists(model_flags_path): raise IOError(("Cannot find file %s. Did you run train.py on the same " "--train_dir?") % model_flags_path) flags_dict = json.loads(file_io.FileIO(model_flags_path, mode="r").read()) with tf.Graph().as_default(): # convert feature_names and feature_sizes to lists of values feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes( flags_dict["feature_names"], flags_dict["feature_sizes"]) if flags_dict["frame_features"]: reader = readers.YT8MFrameFeatureReader(feature_names=feature_names, feature_sizes=feature_sizes) else: reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names, feature_sizes=feature_sizes) model = find_class_by_name(flags_dict["model"], [frame_level_models, video_level_models])() label_loss_fn = find_class_by_name(flags_dict["label_loss"], [losses])() if FLAGS.eval_data_pattern is "": raise IOError("'eval_data_pattern' was not specified. " + "Nothing to evaluate.") build_graph( reader=reader, model=model, eval_data_pattern=FLAGS.eval_data_pattern, label_loss_fn=label_loss_fn, num_readers=FLAGS.num_readers, batch_size=FLAGS.batch_size) logging.info("built evaluation graph") summary_writer = tf.summary.FileWriter( FLAGS.train_dir, graph=tf.get_default_graph()) evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k)
def evaluate(): tf.set_random_seed(0) # for reproducibility with tf.Graph().as_default(): reader = readers.CatsVsDogsFeatureReader() model = find_class_by_name(FLAGS.model, [cvd_models])() label_loss_fn = find_class_by_name(FLAGS.label_loss, [losses])() if FLAGS.eval_data_pattern is "": raise IOError("'eval_data_pattern' was not specified. " + "Nothing to evaluate.") build_graph(reader=reader, model=model, eval_data_pattern=FLAGS.eval_data_pattern, label_loss_fn=label_loss_fn, num_readers=FLAGS.num_readers, batch_size=FLAGS.batch_size) logging.info("built evaluation graph") image_id_batch = tf.get_collection("id_batch")[0] prediction_batch = tf.get_collection("predictions")[0] label_batch = tf.get_collection("labels")[0] loss = tf.get_collection("loss")[0] summary_op = tf.get_collection("summary_op")[0] saver = tf.train.Saver(tf.global_variables()) summary_writer = tf.summary.FileWriter(FLAGS.train_dir, graph=tf.get_default_graph()) evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, 2) last_global_step_val = -1 while True: last_global_step_val = evaluation_loop(image_id_batch, prediction_batch, label_batch, loss, summary_op, saver, summary_writer, evl_metrics, last_global_step_val) if FLAGS.run_once: break
def evaluate(): if os.path.exists(FLAGS.Ensemble_Models + FLAGS.ensemble_output_path): shutil.rmtree(FLAGS.Ensemble_Models + FLAGS.ensemble_output_path) subdirs = [ os.path.join(FLAGS.Ensemble_Models, x) for x in FLAGS.ensemble_model_path.split(',') ] flags_dict = [] save_flag = True for subdir in subdirs: model_flags_path = os.path.join( os.path.join(FLAGS.Ensemble_Models, subdir), "model_flags.json") print "Load model from " + model_flags_path + "\n" flags_dict.append( json.loads(file_io.FileIO(model_flags_path, mode="r").read())) # save model_flags.json to inference dictory if save_flag: if os.path.exists(FLAGS.Ensemble_Models + FLAGS.ensemble_output_path) == False: os.mkdir(FLAGS.Ensemble_Models + FLAGS.ensemble_output_path) shutil.copyfile( model_flags_path, os.path.join( FLAGS.Ensemble_Models + FLAGS.ensemble_output_path, "model_flags.json")) save_flag = False g = tf.Graph() with g.as_default(): models = [] label_loss_fns = [] readers = [] model_nums = len(subdirs) for m in range(model_nums): model, label_loss_fn, reader = get_params(flags_dict[m]) models.append(model) label_loss_fns.append(label_loss_fn) readers.append(reader) # start build graph build_graph( reader=readers[0], # anyone is ok models=models, eval_data_pattern=FLAGS.eval_data_pattern, label_loss_fns=label_loss_fns, batch_size=FLAGS.batch_size, num_readers=FLAGS.num_readers) logging.info("built evaluation graph") video_id_batch = tf.get_collection("video_id_batch")[0] prediction_batch = tf.get_collection("predictions")[0] label_batch = tf.get_collection("labels")[0] loss = tf.get_collection("loss")[0] summary_op = tf.get_collection("summary_op")[0] all_vars = tf.global_variables() # remove global_step or inference can't find uninitialize parameter all_vars = [v for v in all_vars if "global_step" not in v.op.name] saver = tf.train.Saver(all_vars) evl_metrics = eval_util.EvaluationMetrics(readers[0].num_classes, FLAGS.top_k) summary_writer = tf.summary.FileWriter(FLAGS.Ensemble_Models, graph=tf.get_default_graph()) last_global_step_val = -1 last_global_step_val = evaluation_loop(model_nums, subdirs, video_id_batch, prediction_batch, label_batch, loss, summary_op, saver, summary_writer, evl_metrics)
def evaluate(model, checkpoint_name, k_frame): tf.set_random_seed(0) # for reproducibility # Write json of flags model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json") if not os.path.exists(model_flags_path): raise IOError(("Cannot find file %s. Did you run train.py on the same " "--train_dir?") % model_flags_path) flags_dict = json.loads(open(model_flags_path).read()) with tf.Graph().as_default() as graph: # scope_name # if FLAGS.model_type=='KD': # scope_name='student' # with tf.variable_scope(, reuse=False): # convert feature_names and feature_sizes to lists of values feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes( flags_dict["feature_names"], flags_dict["feature_sizes"]) if flags_dict["frame_features"]: reader = readers.YT8MFrameFeatureReader( feature_names=feature_names, feature_sizes=feature_sizes) else: reader = readers.YT8MAggregatedFeatureReader( feature_names=feature_names, feature_sizes=feature_sizes) label_loss_fn = find_class_by_name(flags_dict["label_loss"], [losses])() if FLAGS.eval_data_pattern is "": raise IOError("'eval_data_pattern' was not specified. " + "Nothing to evaluate.") scope_name = 'teacher' if k_frame != 300: scope_name = 'student' with tf.variable_scope(scope_name, reuse=None): build_graph(reader=reader, model=model, eval_data_pattern=FLAGS.eval_data_pattern, label_loss_fn=label_loss_fn, num_readers=FLAGS.num_readers, batch_size=FLAGS.batch_size, k_frame=k_frame) # print("See variables in student in EVAL") # for i in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,scope='student'): # print(i) # i.name if you want just a name logging.info("built evaluation graph") video_id_batch = tf.get_collection("video_id_batch")[0] prediction_batch = tf.get_collection("predictions")[0] label_batch = tf.get_collection("labels")[0] loss = tf.get_collection("loss")[0] summary_op = tf.get_collection("summary_op")[0] saver = tf.train.Saver(tf.global_variables()) if k_frame != 300: student_variables = tf.get_collection( tf.GraphKeys.GLOBAL_VARIABLES, scope='student') saver = tf.train.Saver(var_list=student_variables) # student_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='student') # print("See variables in student") # for i in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='student'): # print(i) # i.name if you want just a name # student_saver = tf.train.Saver(var_list=student_variables) summary_writer = tf.summary.FileWriter(FLAGS.train_dir, graph=tf.get_default_graph()) evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k) last_global_step_val = -1 while True: last_global_step_val, EvalGAP = evaluation_loop( video_id_batch, prediction_batch, label_batch, loss, summary_op, saver, summary_writer, evl_metrics, last_global_step_val, checkpoint_name) if FLAGS.run_once: break return EvalGAP
ground-truth labels, hence for train and validate data sets. ''' def pairwise(t): it = iter(t) return izip(it, it) records = glob.glob("/media/6TB/video/yt8m-v2/video/validate???5.tfrecord") csv = "val_gatednetvladLF-256k-1024-80-0002-300iter-norelu-basic-gatedmoe-ens2.csv" num_classes = 3862 batch_size = 1 top_k = 20 evl_metrics = eval_util.EvaluationMetrics(num_classes, top_k) all_labels = dict() for record in records: for example in tf.python_io.tf_record_iterator(record): tf_example = tf.train.Example.FromString(example) vid_id = tf_example.features.feature['id'].bytes_list.value[0].decode( encoding='UTF-8') labels = tf_example.features.feature['labels'].int64_list.value all_labels[vid_id] = labels evl_metrics.clear() with open(csv, 'r') as fid: next(fid) for line in fid: vid, label_scores = line.split(',')
def evaluate(): ema_tensors = None if FLAGS.use_EMA: latest_checkpoint = get_latest_checkpoint() assert latest_checkpoint, "No checkpoint found" with tf.device("/cpu:0"): saver = tf.train.import_meta_graph(latest_checkpoint + ".meta", clear_devices=True) # saver.restore(sess, "../trained_models/attention_frames_v0_EMA/model.ckpt-15512") xvars = tf.get_collection("ema_vars") assert len(xvars) > 0, "No EMA shadow variables found. Did you train with EMA?" ema_tensors = list(set([x.name for x in xvars])) tf.reset_default_graph() tf.set_random_seed(0) # for reproducibility # Write json of flags model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json") if not os.path.exists(model_flags_path): raise IOError(("Cannot find file %s. Did you run train.py on the same " "--train_dir?") % model_flags_path) flags_dict = json.loads(open(model_flags_path).read()) with tf.Graph().as_default(): # convert feature_names and feature_sizes to lists of values feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes( flags_dict["feature_names"], flags_dict["feature_sizes"]) if flags_dict["frame_features"]: reader = readers.YT8MFrameFeatureReader(feature_names=feature_names, feature_sizes=feature_sizes) else: reader = readers.YT8MAggregatedFeatureReader(feature_names=feature_names, feature_sizes=feature_sizes) model = find_class_by_name(flags_dict["model"], [frame_level_models, video_level_models])() label_loss_fn = find_class_by_name(flags_dict["label_loss"], [losses])() if FLAGS.eval_data_pattern is "": raise IOError("'eval_data_pattern' was not specified. " + "Nothing to evaluate.") build_graph( reader=reader, model=model, eval_data_pattern=FLAGS.eval_data_pattern, label_loss_fn=label_loss_fn, num_readers=FLAGS.num_readers, batch_size=FLAGS.batch_size) logging.info("built evaluation graph") video_id_batch = tf.get_collection("video_id_batch")[0] prediction_batch = tf.get_collection("predictions")[0] label_batch = tf.get_collection("labels")[0] loss = tf.get_collection("loss")[0] summary_op = tf.get_collection("summary_op")[0] saver = tf.train.Saver(tf.global_variables()) summary_writer = tf.summary.FileWriter( FLAGS.train_dir, graph=tf.get_default_graph()) evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k) last_global_step_val = -1 while True: last_global_step_val = evaluation_loop(video_id_batch, prediction_batch, label_batch, loss, summary_op, saver, summary_writer, evl_metrics, last_global_step_val, ema_tensors) if FLAGS.run_once: break
def run(self): tf.set_random_seed(0) # for reproducibility # Setup logging & log the version. tf.set_random_seed(0) # for reproducibility # Setup logging & log the version. tf.logging.set_verbosity(logging.INFO) logging.info("Tensorflow version: {}.".format(tf.__version__)) if os.environ.get('CUDA_VISIBLE_DEVICES') is None: if FLAGS.eval_num_gpu == 0: os.environ['CUDA_VISIBLE_DEVICES'] = '-1' else: os.environ['CUDA_VISIBLE_DEVICES'] = ','.join( map(str, range(FLAGS.eval_num_gpu))) # self.train_dir = join(FLAGS.path, FLAGS.train_dir) self.train_dir = FLAGS.train_dir pp = pprint.PrettyPrinter(indent=2, compact=True) logging.info(pp.pformat(FLAGS.values())) with tf.Graph().as_default(): if FLAGS.eval_num_gpu: self.batch_size = \ FLAGS.eval_batch_size * FLAGS.eval_num_gpu else: self.batch_size = FLAGS.eval_batch_size self.reader = find_class_by_name(FLAGS.reader, [readers])(self.batch_size, is_training=False) self.model = find_class_by_name(FLAGS.model, [models])() self.loss_fn = find_class_by_name(FLAGS.loss, [losses])() data_pattern = FLAGS.data_pattern if data_pattern is "": raise IOError("'data_pattern' was not specified. " "Nothing to evaluate.") self.build_graph() logging.info("Built evaluation graph") self.saver = tf.train.Saver(tf.global_variables()) filename_suffix = "_{}_{}".format( "eval", re.findall("[a-z0-9]+", data_pattern.lower())[0]) self.summary_writer = tf.summary.FileWriter( self.train_dir, filename_suffix=filename_suffix, graph=tf.get_default_graph()) evl_metrics = eval_util.EvaluationMetrics(self.reader.n_classes, 20) self.counter = 0 last_global_step_val = 0 while self.counter < FLAGS.stopped_at_n: last_global_step_val = self.eval_loop(last_global_step_val, evl_metrics) logging.info("Done evaluation -- number of eval reached.")
def evaluate(): tf.compat.v1.set_random_seed(0) # for reproducibility # Write json of flags model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json") if not file_io.file_exists(model_flags_path): raise IOError(("Cannot find file %s. Did you run train.py on the same " "--train_dir?") % model_flags_path) flags_dict = json.loads(open(model_flags_path).read()) with tf.Graph().as_default(): # convert feature_names and feature_sizes to lists of values feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes( flags_dict["feature_names"], flags_dict["feature_sizes"]) if flags_dict["frame_features"]: reader = readers.YT8MFrameFeatureReader( feature_names=feature_names, feature_sizes=feature_sizes, segment_labels=FLAGS.segment_labels, ) else: reader = readers.YT8MAggregatedFeatureReader( feature_names=feature_names, feature_sizes=feature_sizes) model = find_class_by_name( flags_dict["model"], [frame_level_models, video_level_models, nextvlad])() label_loss_fn = find_class_by_name(flags_dict["label_loss"], [losses])() if FLAGS.eval_data_pattern: raise IOError("'eval_data_pattern' was not specified. " + "Nothing to evaluate.") build_graph( reader=reader, model=model, eval_data_pattern=FLAGS.eval_data_pattern, label_loss_fn=label_loss_fn, num_readers=FLAGS.num_readers, batch_size=FLAGS.batch_size, ) logging.info("built evaluation graph") # xxx 2018 # video_id_batch = tf.compat.v1.get_collection("video_id_batch")[0] # prediction_batch = tf.compat.v1.get_collection("predictions")[0] # label_batch = tf.compat.v1.get_collection("labels")[0] # loss = tf.compat.v1.get_collection("loss")[0] # summary_op = tf.compat.v1.get_collection("summary_op")[0] # A dict of tensors to be run in Session. fetches = { "video_id": tf.compat.v1.get_collection("video_id_batch")[0], "predictions": tf.compat.v1.get_collection("predictions")[0], "labels": tf.compat.v1.get_collection("labels")[0], "loss": tf.compat.v1.get_collection("loss")[0], "summary": tf.compat.v1.get_collection("summary_op")[0], } if FLAGS.segment_labels: fetches["label_weights"] = tf.compat.v1.get_collection( "label_weights")[0] saver = tf.compat.v1.train.Saver(tf.compat.v1.global_variables()) summary_writer = tf.compat.v1.summary.FileWriter( os.path.join(FLAGS.train_dir, "eval"), graph=tf.compat.v1.get_default_graph(), ) evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k, None) last_global_step_val = -1 while True: last_global_step_val = evaluation_loop(fetches, saver, summary_writer, evl_metrics, last_global_step_val) if FLAGS.run_once: break
return dense_predictions def sparse_labels_to_dense(sparse_labels, num_classes): dense_labels = np.zeros([num_classes], dtype=int) for label in sparse_labels: dense_labels[label] = 1 return dense_labels with open(labels_file) as f: labels = json.load(f) eval = eval_util.EvaluationMetrics(4716, 20) count = 0 batch_num = 4096 batch_predictions = [] batch_labels = [] with open(predictions_file) as f: reader = csv.DictReader(f) for row in reader: sparse_predictions = row['LabelConfidencePairs'].split(' ') sparse_labels = labels[row['VideoId']] dense_predictions = sparse_predictions_to_dense( sparse_predictions, 4716) dense_labels = sparse_labels_to_dense(sparse_labels, 4716)
def evaluate(): tf.set_random_seed(0) # for reproducibility # Write json of flags # model_flags_path = os.path.join(FLAGS.train_dir, "model_flags.json") # if not file_io.file_exists(model_flags_path): # raise IOError(("Cannot find file %s. Did you run train.py on the same " # "--train_dir?") % model_flags_path) # flags_dict = json.loads(file_io.FileIO(model_flags_path, mode="r").read()) all_eval_data_patterns = [] with open(FLAGS.eval_data_config) as f: all_eval_data_patterns = f.read().splitlines() with tf.Graph().as_default(): # convert feature_names and feature_sizes to lists of values # feature_names, feature_sizes = utils.GetListOfFeatureNamesAndSizes( # flags_dict["feature_names"], flags_dict["feature_sizes"]) # prepare a reader for each single model prediction result all_readers = [] for i in xrange(len(all_eval_data_patterns)): reader = readers.EnsembleReader( feature_names=[FLAGS.feature_names], feature_sizes=[FLAGS.feature_sizes]) all_readers.append(reader) input_reader = None input_data_pattern = None # model = find_class_by_name(flags_dict["model"], # [frame_level_models, video_level_models])() model = ensemble_model.MeanModel() label_loss_fn = find_class_by_name("CrossEntropyLoss", [losses])() build_graph( all_readers=all_readers, all_eval_data_patterns = all_eval_data_patterns, model=model, label_loss_fn=label_loss_fn, num_readers=FLAGS.num_readers, batch_size=FLAGS.batch_size) logging.info("built evaluation graph") video_id_batch = tf.get_collection("video_id_batch")[0] prediction_batch = tf.get_collection("predictions")[0] label_batch = tf.get_collection("labels")[0] loss = tf.get_collection("loss")[0] summary_op = tf.get_collection("summary_op")[0] saver = tf.train.Saver(tf.global_variables()) summary_writer = tf.summary.FileWriter( FLAGS.train_dir, graph=tf.get_default_graph()) evl_metrics = eval_util.EvaluationMetrics(reader.num_classes, FLAGS.top_k) last_global_step_val = -1 while True: last_global_step_val = evaluation_loop(video_id_batch, prediction_batch, label_batch, loss, summary_op, saver, summary_writer, evl_metrics, last_global_step_val) if FLAGS.run_once: break