def compute_accuracy(x, l, mask): """Compute model accuracy.""" preds = ch_model.get_probs(x) preds = tf.squeeze(preds) preds = tf.argmax(preds, -1, output_type=l.dtype) _, acc_update_op = tf.metrics.accuracy(l, preds, weights=mask) if FLAGS.surrogate_attack: preds = sur_ch_model.get_probs(x) preds = tf.squeeze(preds) preds = tf.argmax(preds, -1, output_type=l.dtype) acc_update_op = tf.tuple((acc_update_op, tf.metrics.accuracy(l, preds, weights=mask)[1])) sess.run(tf.initialize_local_variables()) for i in range(FLAGS.eval_steps): tf.logging.info( "\tEvaluating batch [%d / %d]" % (i + 1, FLAGS.eval_steps)) acc = sess.run(acc_update_op) if FLAGS.surrogate_attack: tf.logging.info("\tFinal acc: (%.4f, %.4f)" % (acc[0], acc[1])) else: tf.logging.info("\tFinal acc: %.4f" % acc) return acc
def testRecallAtK(self): labels = tf.convert_to_tensor([[1, 0, 0], [0, 1, 1], [0, 0, 0]], dtype=tf.float32) predictions = tf.convert_to_tensor([[0.5, 1.0, 0.0], [0.5, 0.0, 1.0], [0.25, 0.25, 0.4]]) k_values = [0, 1, 2] expected_recall_values_for_each_k = [0, 0.25, 0.75] # For k = 0 we get 0% from both of first two (3rd example is always NA) # For k = 1 we get 0% from first example and 50% from second # For k = 2 we get 100% from first example and 50% from second values_and_updates = [ protein_model._custom_recall_at_k(labels_as_multi_hot=labels, predictions=predictions, k=k) for k in k_values ] with tf.Session() as sess: for i, value_and_update in enumerate(values_and_updates): value, update_op = value_and_update sess.run(tf.initialize_local_variables()) sess.run(update_op) actual_recall = sess.run(value) self.assertEqual(actual_recall, expected_recall_values_for_each_k[i])
def run_on_files(self, input_filenames, batch_size, max_size=None, keys=None): """Run evaluation on the given files. Args: input_filenames: list of strings providing paths to input sstables. batch_size: integer mini-batch size. max_size: optional maximum number of examples to evalute. keys: Optional sequence of string tensor names to evaluate. By default, uses all known tensors. Returns: xarray.Dataset with data variables for each of the given keys. """ with tf.Graph().as_default(): examples = examples_from_sstable(input_filenames, batch_size) tensors = self.evaluation_tensors(examples, keys=keys) sess = tf.Session() # initialize the num_epochs counter sess.run(tf.initialize_local_variables()) self._restore(sess) # TODO(shoyer): after cl/133322369 is merged, switch to use # tf.contrib.metrics.streaming_concat instead of fetch_across_batches. arrays_list = fetch_across_batches(sess, list(tensors.values()), max_size) arrays_dict = dict(list(zip(list(tensors.keys()), arrays_list))) return convert_labeled_tensor_to_xarray(arrays_dict, tensors)
def eval_model(): preds = spec.predictions["predictions"] preds = tf.argmax(preds, -1, output_type=labels.dtype) _, acc_update_op = tf.metrics.accuracy(labels=labels, predictions=preds) sess.run(tf.initialize_local_variables()) for _ in range(FLAGS.eval_steps): acc = sess.run(acc_update_op) return acc
def load_preaggregated_data(self): # Return objects of this function X = None Y = None X_valid = None Y_valid = None # Load pre-aggregated training dataset tfrecord_file_list = os.listdir(self.preaggregated_data_path) tfrecord_file_list = [ os.path.join(self.preaggregated_data_path, k) for k in tfrecord_file_list ] print('Pre-aggregated file list = ' + str(tfrecord_file_list)) reader = tf.TFRecordReader() key, examples = reader.read( tf.train.string_input_producer( tfrecord_file_list, num_epochs=1)) # Only generate all data once name_to_features = { "input_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "input_mask": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "segment_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), } parsed_example = tf.parse_single_example(examples, name_to_features) parsed_example_values = list(parsed_example.values()) # Reuse Keras Session sess = K.get_session() # Just read all data into array for now. # TODO: Implment generator to support very large dataset that is not fit into RAM all_data = [] sess.run(tf.initialize_local_variables()) tf.train.start_queue_runners(sess=sess) try: while True: data = sess.run(parsed_example_values) for i in range(len(data)): if len(all_data) <= i: all_data.append([]) all_data[i].append(data[i]) except tf.errors.OutOfRangeError: pass all_data = [np.array(a) for a in all_data] X = all_data Y = all_data[0] # Y is only 'input_ids' tensor K.clear_session() # sess object is not valid anymore after this # Load pre-aggregated validation dataset tfrecord_file_list = os.listdir( self.preaggregated_validation_data_path) tfrecord_file_list = [ os.path.join(self.preaggregated_validation_data_path, k) for k in tfrecord_file_list ] print('Pre-aggregated file list = ' + str(tfrecord_file_list)) reader = tf.TFRecordReader() key, examples = reader.read( tf.train.string_input_producer( tfrecord_file_list, num_epochs=1)) # Only generate all data once name_to_features = { "input_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "input_mask": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "segment_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), } parsed_example = tf.parse_single_example(examples, name_to_features) parsed_example_values = list(parsed_example.values()) # Reuse Keras Session sess = K.get_session() # Just read all data into array for now. # TODO: Implment generator to support very large dataset that is not fit into RAM all_data = [] sess.run(tf.initialize_local_variables()) tf.train.start_queue_runners(sess=sess) try: while True: data = sess.run(parsed_example_values) for i in range(len(data)): if len(all_data) <= i: all_data.append([]) all_data[i].append(data[i]) except tf.errors.OutOfRangeError: pass all_data = [np.array(a) for a in all_data] X_valid = all_data Y_valid = all_data[0] # Y is only 'input_ids' tensor K.clear_session() # sess object is not valid anymore after this #print(len(X_valid)) #print(len(Y_valid)) return (X, Y, X_valid, Y_valid)
def run(config, target='', cluster_spec=None, is_chief=True, job_name=None, task_index=None, get_model_fn=get_model, get_dataset_fn=get_dataset, environment=None): model_class = get_model_fn(config.model.type) image_vis = config.train.get('image_vis') var_vis = config.train.get('var_vis') if config.train.get('seed') is not None: tf.set_random_seed(config.train.seed) log_prefix = '[{}-{}] - '.format(job_name, task_index) \ if job_name is not None and task_index is not None else '' if config.train.debug or config.train.tf_debug: tf.logging.set_verbosity(tf.logging.DEBUG) else: tf.logging.set_verbosity(tf.logging.INFO) model = model_class(config) # Placement of ops on devices using replica device setter # which automatically places the parameters on the `ps` server # and the `ops` on the workers # # See: # https://www.tensorflow.org/api_docs/python/tf/train/replica_device_setter with tf.device(tf.train.replica_device_setter(cluster=cluster_spec)): try: config['dataset']['type'] except KeyError: raise KeyError('dataset.type should be set on the custom config.') try: dataset_class = get_dataset_fn(config.dataset.type) dataset = dataset_class(config) train_dataset = dataset() except InvalidDataDirectory as exc: tf.logging.error("Error while reading dataset, {}".format(exc)) sys.exit(1) train_image = train_dataset['image'] train_filename = train_dataset['filename'] train_bboxes = train_dataset['bboxes'] prediction_dict = model(train_image, train_bboxes, is_training=True) total_loss = model.loss(prediction_dict) global_step = tf.train.get_or_create_global_step() optimizer = get_optimizer(config.train, global_step) # TODO: Is this necesarry? Couldn't we just get them from the # trainable vars collection? We should probably improve our # usage of collections. trainable_vars = model.get_trainable_vars() # Compute, clip and apply gradients with tf.name_scope('gradients'): grads_and_vars = optimizer.compute_gradients( total_loss, trainable_vars) if config.train.clip_by_norm: grads_and_vars = clip_gradients_by_norm(grads_and_vars) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Create custom init for slots in optimizer, as we don't save them to # our checkpoints. An example of slots in an optimizer are the Momentum # variables in MomentumOptimizer. We do this because slot variables can # effectively duplicate the size of your checkpoint! slot_variables = [ optimizer.get_slot(var, name) for name in optimizer.get_slot_names() for var in trainable_vars ] slot_init = tf.variables_initializer( slot_variables, name='optimizer_slots_initializer') # Create saver for saving/restoring model model_saver = tf.train.Saver( set(tf.global_variables()) - set(slot_variables), name='model_saver', max_to_keep=config.train.get('checkpoints_max_keep', 1), ) # Create saver for loading pretrained checkpoint into base network base_checkpoint_vars = model.get_base_network_checkpoint_vars() checkpoint_file = model.get_checkpoint_file() if base_checkpoint_vars and checkpoint_file: base_net_checkpoint_saver = tf.train.Saver( base_checkpoint_vars, name='base_net_checkpoint_saver') # We'll send this fn to Scaffold init_fn def load_base_net_checkpoint(_, session): base_net_checkpoint_saver.restore(session, checkpoint_file) else: load_base_net_checkpoint = None tf.logging.info('{}Starting training for {}'.format(log_prefix, model)) run_options = None if config.train.full_trace: run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) # Create custom Scaffold to make sure we run our own init_op when model # is not restored from checkpoint. summary_op = [model.summary] summaries = tf.summary.merge_all() if summaries is not None: summary_op.append(summaries) summary_op = tf.summary.merge(summary_op) # `ready_for_local_init_op` is hardcoded to 'ready' as local init doesn't # depend on global init and `local_init_op` only runs when it is set as # 'ready' (an empty string tensor sets it as ready). scaffold = tf.train.Scaffold( saver=model_saver, init_op=tf.global_variables_initializer() if is_chief else tf.no_op(), local_init_op=tf.group(tf.initialize_local_variables(), slot_init), ready_for_local_init_op=tf.constant([], dtype=tf.string), summary_op=summary_op, init_fn=load_base_net_checkpoint, ) # Custom hooks for our session hooks = [] chief_only_hooks = [] if config.train.tf_debug: debug_hook = tf_debug.LocalCLIDebugHook() debug_hook.add_tensor_filter('has_inf_or_nan', tf_debug.has_inf_or_nan) hooks.extend([debug_hook]) if not config.train.job_dir: tf.logging.warning( '`job_dir` is not defined. Checkpoints and logs will not be saved.' ) checkpoint_dir = None elif config.train.run_name: # Use run_name when available checkpoint_dir = os.path.join(config.train.job_dir, config.train.run_name) else: checkpoint_dir = config.train.job_dir should_add_hooks = (config.train.display_every_steps or config.train.display_every_secs and checkpoint_dir is not None) if should_add_hooks: if not config.train.debug and image_vis == 'debug': tf.logging.warning('ImageVisHook will not run without debug mode.') elif image_vis is not None: # ImageVis only runs on the chief. chief_only_hooks.append( ImageVisHook(prediction_dict, image=train_dataset['image'], gt_bboxes=train_dataset['bboxes'], config=config.model, output_dir=checkpoint_dir, every_n_steps=config.train.display_every_steps, every_n_secs=config.train.display_every_secs, image_visualization_mode=image_vis)) if var_vis is not None: # VarVis only runs on the chief. chief_only_hooks.append( VarVisHook( every_n_steps=config.train.display_every_steps, every_n_secs=config.train.display_every_secs, mode=var_vis, output_dir=checkpoint_dir, vars_summary=model.vars_summary, )) step = -1 with tf.train.MonitoredTrainingSession( master=target, is_chief=is_chief, checkpoint_dir=checkpoint_dir, scaffold=scaffold, hooks=hooks, chief_only_hooks=chief_only_hooks, save_checkpoint_secs=config.train.save_checkpoint_secs, save_summaries_steps=config.train.save_summaries_steps, save_summaries_secs=config.train.save_summaries_secs, ) as sess: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) try: while not coord.should_stop(): before = time.time() _, train_loss, step, filename = sess.run( [train_op, total_loss, global_step, train_filename], options=run_options) # TODO: Add image summary every once in a while. tf.logging.info( '{}step: {}, file: {}, train_loss: {}, in {:.2f}s'.format( log_prefix, step, filename, train_loss, time.time() - before)) if is_chief and step == 1: # We save the run after first batch to make sure everything # works properly. save_run(config, environment=environment) except tf.errors.OutOfRangeError: tf.logging.info('{}finished training after {} epoch limit'.format( log_prefix, config.train.num_epochs)) # TODO: Print summary finally: coord.request_stop() # Wait for all threads to stop. coord.join(threads) return step
def predict(model_root, datasets_dir, model_name, test_image_name): with tf.Graph().as_default(): tf_global_step = slim.get_or_create_global_step() test_image = os.path.join(datasets_dir, test_image_name) # dataset = convert_data.get_datasets('test',dataset_dir=datasets_dir) network_fn = net_select.get_network_fn(model_name, num_classes=20, is_training=False) batch_size = 1 eval_image_size = network_fn.default_image_size # images, images_raw, labels = load_batch(datasets_dir, # height=eval_image_size, # width=eval_image_size) image_preprocessing_fn = preprocessing_select.get_preprocessing( model_name, is_training=False) image_data = tf.io.read_file(test_image) image_data = tf.image.decode_jpeg(image_data, channels=3) image_data = image_preprocessing_fn(image_data, eval_image_size, eval_image_size) image_data = tf.expand_dims(image_data, 0) logits_1, end_points_1 = network_fn(image_data) attention_maps = tf.reduce_mean(end_points_1['attention_maps'], axis=-1, keepdims=True) attention_maps = tf.image.resize(attention_maps, [eval_image_size, eval_image_size], method=tf.image.ResizeMethod.BILINEAR) bboxes = tf_v1.py_func(mask2bbox, [attention_maps], [tf.float32]) bboxes = tf.reshape(bboxes, [batch_size, 4]) # print(bboxes) box_ind = tf.range(batch_size, dtype=tf.int32) images = tf.image.crop_and_resize( image_data, bboxes, box_ind, crop_size=[eval_image_size, eval_image_size]) logits_2, end_points_2 = network_fn(images, reuse=True) logits = tf.math.log( tf.nn.softmax(logits_1) * 0.5 + tf.nn.softmax(logits_2) * 0.5) checkpoint_path = os.path.join(model_root, model_name) if tf.io.gfile.isdir(checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) else: checkpoint_path = checkpoint_path init_fn = slim.assign_from_checkpoint_fn( checkpoint_path, slim.get_variables_to_restore()) # with tf_v1.Session() as sess: # with slim.queues.QueueRunners(sess): # sess.run(tf_v1.initialize_local_variables()) # init_fn(sess) # np_probabilities, np_images_raw, np_labels = sess.run([logits, images_raw, labels]) # # for i in range(batch_size): # image = np_images_raw[i, :, :, :] # true_label = np_labels[i] # predicted_label = np.argmax(np_probabilities[i, :]) # print('true is {}, predict is {}'.format(true_label, predicted_label)) with tf_v1.Session() as sess: with slim.queues.QueueRunners(sess): sess.run(tf_v1.initialize_local_variables()) init_fn(sess) np_images, np_probabilities = sess.run([image_data, logits]) predicted_label = np.argmax(np_probabilities[0, :]) print(predicted_label)