def evaluate(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels, all_filenames, filename_queue = image_processing.inputs( dataset) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class num_classes = dataset.num_classes() + 1 print("there are %d classes!" % dataset.num_classes()) # Build a Graph that computes the logits predictions from the # inference model. logits, _, end_points, net2048, sel_end_points = inception.inference( images, num_classes) # Calculate predictions. #max_percent = tf.argmax(logits,1) #max_percent = tf.reduce_max(logits, reduction_indices=[1]) / tf.add_n(logits) max_percent = end_points['predictions'] # max_percent = len(end_points) #for kk in range(len(labels)): # #max_percent.append(end_points['predictions'][kk][labels[kk]]) # max_percent.append(labels[kk]) if FLAGS.mode == '0_softmax': top_1_op = tf.nn.in_top_k(logits, labels, 1) top_5_op = tf.nn.in_top_k(logits, labels, 5) elif FLAGS.mode == '1_sigmoid': top_1_op = None top_5_op = None # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() graph_def = tf.get_default_graph().as_graph_def() summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, graph_def=graph_def) while True: precision_at_1, current_score = _eval_once( saver, summary_writer, top_1_op, top_5_op, summary_op, max_percent, all_filenames, filename_queue, net2048, sel_end_points, logits, labels) print("%s: Precision: %.4f " % (datetime.now(), precision_at_1)) if FLAGS.run_once: break time.sleep(FLAGS.eval_interval_secs) return precision_at_1, current_score
def evaluate(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels, all_filenames, filename_queue = image_processing.inputs(dataset) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class num_classes = dataset.num_classes() + 1 print("there are %d classes!" % dataset.num_classes()) # Build a Graph that computes the logits predictions from the # inference model. logits, _, end_points, net2048, sel_end_points = inception.inference(images, num_classes) # Calculate predictions. #max_percent = tf.argmax(logits,1) #max_percent = tf.reduce_max(logits, reduction_indices=[1]) / tf.add_n(logits) max_percent = end_points['predictions'] # max_percent = len(end_points) #for kk in range(len(labels)): # #max_percent.append(end_points['predictions'][kk][labels[kk]]) # max_percent.append(labels[kk]) if FLAGS.mode == '0_softmax': top_1_op = tf.nn.in_top_k(logits, labels, 1) top_5_op = tf.nn.in_top_k(logits, labels, 5) elif FLAGS.mode == '1_sigmoid': top_1_op = None top_5_op = None # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() graph_def = tf.get_default_graph().as_graph_def() summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, graph_def=graph_def) while True: precision_at_1, current_score = _eval_once(saver, summary_writer, top_1_op, top_5_op, summary_op, max_percent, all_filenames, filename_queue, net2048, sel_end_points, logits, labels) print("%s: Precision: %.4f " % (datetime.now(), precision_at_1) ) if FLAGS.run_once: break time.sleep(FLAGS.eval_interval_secs) return precision_at_1, current_score
def evaluate_op(dataset): # Get images and labels from the dataset. images, labels, _ = image_processing.inputs(dataset) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class. #num_classes = dataset.num_classes() + 1 num_classes = dataset.num_classes() # Build a Graph that computes the logits predictions from the # inference model. logits, _ = inception.inference(images, num_classes) # Calculate predictions. top_1_op = tf.nn.in_top_k(logits, labels, 1) top_5_op = tf.nn.in_top_k(logits, labels, 5) return top_1_op, top_5_op
def evaluate(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels = image_processing.inputs(dataset) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class. num_classes = dataset.num_classes() + 1 # Build a Graph that computes the logits predictions from the # inference model. logits, _ = inception.inference(images, num_classes) # print(logits.get_shape()) # print(labels.get_shape()) # Calculate predictions. # top_1_op = tf.nn.in_top_k(logits, labels, 1) # top_5_op = tf.nn.in_top_k(logits, labels, 5) label_bool_op = evaluate_multilabel(logits, labels) # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() graph_def = tf.get_default_graph().as_graph_def() summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, graph_def=graph_def) # while True: # _eval_once(saver, summary_writer, top_1_op, top_5_op, summary_op) # if FLAGS.run_once: # break # time.sleep(FLAGS.eval_interval_secs) while True: _eval_once(saver, summary_writer, label_bool_op, summary_op) if FLAGS.run_once: break time.sleep(FLAGS.eval_interval_secs)
def evaluate(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels,filenames = image_processing.inputs(dataset) #print(images.shape) #print(labels.shape) #print(filenames.shape) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class. num_classes = dataset.num_classes() + 1 #for i in range(FLAGS.num_gpus): #with tf.device('/gpu:%d' % (FLAGS.eval_gpu_id)): # Build a Graph that computes the logits predictions from the # inference model. endpoints = inception.inference_endpoint(images, num_classes) #positive_labels = 2 # Calculate predictions. logits = endpoints['logits'] top_1_op = tf.nn.in_top_k(logits, labels, 1) #top_5_op = tf.nn.in_top_k(logits, labels, 5) positive_op = endpoints['predictions'] # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() graph_def = tf.get_default_graph().as_graph_def() summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, graph_def=graph_def) while True: _eval_once(saver, summary_writer, top_1_op,positive_op, summary_op,filenames) if FLAGS.run_once: break time.sleep(FLAGS.eval_interval_secs)
def predict(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels, filenames = image_processing.inputs(dataset) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class. num_classes = dataset.num_classes() # Build a Graph that computes the logits predictions from the # inference model. logits, _ = inception.inference(images, num_classes) # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) _predict_once(saver, filenames, logits)
def evaluate(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels = image_processing.inputs(dataset) # Number of classes in the Dataset label. num_classes = dataset.num_classes() # Number of examples in the Dataset. num_examples_dataset = dataset.num_examples_per_epoch() # Build a Graph that computes the logits predictions from the # inference model. logits, _ = inception.inference(images, num_classes) # Calculate predictions. if not FLAGS.sparse_labels: labels = tf.argmax(labels, axis=1) top_1_op = tf.nn.in_top_k(logits, labels, 1) top_5_op = tf.nn.in_top_k(logits, labels, 5) # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() graph_def = tf.get_default_graph().as_graph_def() summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, graph_def=graph_def) while True: _eval_once(saver, summary_writer, top_1_op, top_5_op, summary_op, num_examples_dataset) if FLAGS.run_once: break time.sleep(FLAGS.eval_interval_secs)
def test(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, _, filenames = image_processing.inputs(dataset) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class. num_classes = dataset.num_classes() + 1 # Build a Graph that computes the logits predictions from the # inference model. logits, _ = inception.inference(images, num_classes) output = tf.nn.softmax(tf.slice(logits, [0,1], [-1,-1]), name='output') # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) results = _test(saver, filenames, output) current_time = datetime.now().strftime('%Y-%m-%d-%Hh%Mm%Ss') csvfilename = os.path.join(FLAGS.test_dir, 'submission-{}.csv'.format(current_time)) zipfilename = os.path.join(FLAGS.test_dir, '{}.zip'.format(csvfilename)) with open(csvfilename, 'wb') as csvfile: writer = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) writer.writerow(['img', 'c0', 'c1', 'c2', 'c3', 'c4', 'c5', 'c6', 'c7', 'c8', 'c9']) for batch_result in results: for filename, result in batch_result: writer.writerow([filename] + result.tolist()) with zipfile.ZipFile(zipfilename, 'w') as myzip: myzip.write(csvfilename) print('Submission available at: %s' % (zipfilename))
def build_input(dataset, data_path, batch_size, standardize_images, mode): if dataset == 'mnist': from datasets import mnist return mnist.build_input(data_path, batch_size, standardize_images, mode) elif dataset == 'svhn': from datasets import svhn return svhn.build_input(data_path, batch_size, standardize_images, mode) elif dataset == 'cifar10': from datasets import cifar return cifar.build_input(dataset, data_path, batch_size, standardize_images, mode) elif dataset == 'cifar100': from datasets import cifar return cifar.build_input(dataset, data_path, batch_size, standardize_images, mode) elif dataset == 'imagenet': from inception import image_processing from inception.imagenet_data import ImagenetData images, labels = image_processing.inputs(ImagenetData('validation'), batch_size=batch_size) import tensorflow as tf labels = tf.one_hot(labels, 1001) return images, labels else: raise ValueError("Dataset {} not supported".format(dataset))
def evaluate(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels = image_processing.inputs(dataset) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class. num_classes = dataset.num_classes() + 1 # Build a Graph that computes the logits predictions from the # inference model. logits, _ = inception.inference(images, num_classes) max_percent = end_points['predictions'] # Calculate predictions. # top_1_op = tf.nn.in_top_k(logits, labels, 1) # top_5_op = tf.nn.in_top_k(logits, labels, 5) # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() graph_def = tf.get_default_graph().as_graph_def() summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, graph_def=graph_def) while True: _eval_once(saver, summary_writer, summary_op) if FLAGS.run_once: break time.sleep(FLAGS.eval_interval_secs)
def evaluate(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels, all_filenames, filename_queue = image_processing.inputs(dataset) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class. num_classes = dataset.num_classes() + 1 # Build a Graph that computes the logits predictions from the # inference model. logits, _, end_points, net2048, sel_end_points = inception.inference(images, num_classes) # Calculate predictions. #max_percent = tf.argmax(logits,1) #max_percent = tf.reduce_max(logits, reduction_indices=[1]) / tf.add_n(logits) max_percent = end_points['predictions'] # max_percent = len(end_points) #for kk in range(len(labels)): # #max_percent.append(end_points['predictions'][kk][labels[kk]]) # max_percent.append(labels[kk]) #top_1_op = tf.nn.in_top_k(logits, labels, 1) #top_5_op = tf.nn.in_top_k(logits, labels, 5) # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() graph_def = tf.get_default_graph().as_graph_def() summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, graph_def=graph_def) # Label 0 is reserved for an (unused) background class. num_classes = dataset.num_classes() + 1 ''' # Split the batch of images and labels for towers. images_splits = tf.split(axis=0, num_or_size_splits=1, value=images) labels_splits = tf.split(axis=0, num_or_size_splits=1, value=labels) # Calculate the gradients for each model tower. tower_grads = [] reuse_variables = None for i in range(1): with tf.device('/gpu:%d' % i): with tf.name_scope('%s_%d' % (inception.TOWER_NAME, i)) as scope: # Force all Variables to reside on the CPU. with slim.arg_scope([slim.variables.variable], device='/cpu:0'): # Calculate the loss for one tower of the ImageNet model. This # function constructs the entire ImageNet model but shares the # variables across all towers. loss = _tower_loss(images_splits[i], labels_splits[i], num_classes, scope, reuse_variables) # Reuse variables for the next tower. reuse_variables = True ''' loss = False while True: precision_at_1, current_score = _eval_once(saver, summary_writer, summary_op, max_percent, all_filenames, filename_queue, net2048, sel_end_points, logits, labels, loss) print("%s: Precision: %.4f --------------------" % (datetime.now(), precision_at_1) ) if FLAGS.run_once: break time.sleep(FLAGS.eval_interval_secs) return precision_at_1, current_score
def retrieve(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(), tf.Session() as sess: # Get images and labels from the dataset. images, labels, filenames_tensor = image_processing.inputs(dataset, return_filenames=True) # Build a Graph that computes the features. num_classes = dataset.num_classes() + 1 _, _ = inception.inference(images, num_classes, restore_logits=False) # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Restore checkpoint. ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir) if ckpt and ckpt.model_checkpoint_path: if os.path.isabs(ckpt.model_checkpoint_path): # Restores from checkpoint with absolute path. saver.restore(sess, ckpt.model_checkpoint_path) else: # Restores from checkpoint with relative path. saver.restore(sess, os.path.join(FLAGS.checkpoint_dir, ckpt.model_checkpoint_path)) # Assuming model_checkpoint_path looks something like: # /my-favorite-path/imagenet_train/model.ckpt-0, # extract global_step from it. global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] print('Succesfully loaded model from %s at step=%s.' % (ckpt.model_checkpoint_path, global_step)) else: print('No checkpoint file found') return # Start the queue runners. coord = tf.train.Coordinator() try: threads = [] for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS): threads.extend(qr.create_threads(sess, coord=coord, daemon=True, start=True)) num_iter = int(math.ceil(FLAGS.num_examples / FLAGS.batch_size)) print('%s: starting evaluation on (%s).' % (datetime.now(), FLAGS.subset)) start_time = time.time() features_tensor = tf.get_default_graph().get_tensor_by_name(FLAGS.features_tensor_name) features = [] filenames = [] step = 0 while step < num_iter and not coord.should_stop(): features_batch, filenames_batch = sess.run([features_tensor, filenames_tensor]) features.append(features_batch) filenames.extend(filenames_batch) step += 1 if step % 20 == 0: duration = time.time() - start_time sec_per_batch = duration / 20.0 examples_per_sec = FLAGS.batch_size / sec_per_batch print('%s: [%d batches out of %d] (%.1f examples/sec; %.3f' 'sec/batch)' % (datetime.now(), step, num_iter, examples_per_sec, sec_per_batch)) start_time = time.time() features = features[:FLAGS.num_examples] filenames = filenames[:FLAGS.num_examples] except Exception as e: # pylint: disable=broad-except coord.request_stop(e) coord.request_stop() coord.join(threads, stop_grace_period_secs=10) return np.vstack(features), filenames
def evaluate(dataset): """Evaluate model on Dataset for a number of steps.""" with tf.Graph().as_default(): # Get images and labels from the dataset. images, labels = image_processing.inputs(dataset) # Number of classes in the Dataset label set plus 1. # Label 0 is reserved for an (unused) background class. num_classes = dataset.num_classes() + 1 # Build a Graph that computes the logits predictions from the # inference model. logits, _ = inception.inference(images, num_classes) pred = tf.nn.softmax(logits) top_1_op = tf.nn.in_top_k(logits, labels, 1) # Calculate predictions. # Restore the moving average version of the learned variables for eval. variable_averages = tf.train.ExponentialMovingAverage( inception.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.merge_all_summaries() graph_def = tf.get_default_graph().as_graph_def() summary_writer = tf.train.SummaryWriter(FLAGS.eval_dir, graph_def=graph_def) with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir) if ckpt and ckpt.model_checkpoint_path: if os.path.isabs(ckpt.model_checkpoint_path): # Restores from checkpoint with absolute path. saver.restore(sess, ckpt.model_checkpoint_path) else: # Restores from checkpoint with relative path. saver.restore( sess, os.path.join(FLAGS.checkpoint_dir, ckpt.model_checkpoint_path)) # Assuming model_checkpoint_path looks something like: # /my-favorite-path/imagenet_train/model.ckpt-0, # extract global_step from it. global_step = ckpt.model_checkpoint_path.split('/')[-1].split( '-')[-1] print('Succesfully loaded model from %s at step=%s.' % (ckpt.model_checkpoint_path, global_step)) else: print('No checkpoint file found') return # Start the queue runners. coord = tf.train.Coordinator() try: threads = [] for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS): threads.extend( qr.create_threads(sess, coord=coord, daemon=True, start=True)) num_iter = int(math.ceil(FLAGS.num_examples / FLAGS.batch_size)) # Counts the number of correct predictions. test_acc = 0.0 count_top_1 = 0 confusion_m_all = [] total_sample_count = num_iter * FLAGS.batch_size step = 0 print('%s: starting evaluation on (%s).' % (datetime.now(), FLAGS.subset)) start_time = time.time() while step < num_iter and not coord.should_stop(): pred, labels, top_1 = sess.run([pred, labels, top_1_op]) print(pred.shape) print(labels.shape) #correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(labels, 1)) correct_pred = np.equal(np.argmax(pred, 1), labels) #print (correct_pred) test_acc += np.sum(correct_pred.astype(float)) confu_m = confusion_matrix(labels, np.argmax( pred, 1)) #(np.argmax(labels,1), np.argmax(pred,1)) confusion_m_all.append(confu_m) #top_1, top_5 = sess.run([top_1_op, top_5_op]) count_top_1 += np.sum(top_1) #count_top_5 += np.sum(top_5) step += 1 ''' if step % 20 == 0: duration = time.time() - start_time sec_per_batch = duration / 20.0 examples_per_sec = FLAGS.batch_size / sec_per_batch print('%s: [%d batches out of %d] (%.1f examples/sec; %.3f' 'sec/batch)' % (datetime.now(), step, num_iter, examples_per_sec, sec_per_batch)) start_time = time.time() ''' # Compute precision @ 1 ''' precision_at_1 = count_top_1 / total_sample_count #recall_at_5 = count_top_5 / total_sample_count print('%s: precision @ 1 = %.4f [%d examples]' % (datetime.now(), precision_at_1, total_sample_count)) ''' print(confusion_m_all.shape) exit() confusion_m_average = np.sum(confusion_m_all, axis=0) print(confusion_m_average) test_acc = float(test_acc) / float(total_sample_count) print("Test Accuracy: {} \n".format(test_acc)) summary = tf.Summary() summary.ParseFromString(sess.run(summary_op)) summary.value.add(tag='Precision @ 1', simple_value=precision_at_1) #summary.value.add(tag='Recall @ 5', simple_value=recall_at_5) summary_writer.add_summary(summary, global_step) except Exception as e: # pylint: disable=broad-except coord.request_stop(e) coord.request_stop() coord.join(threads, stop_grace_period_secs=10)