def main(_): pipeline_proto = _load_pipeline_proto(FLAGS.pipeline_proto) if FLAGS.model_dir: pipeline_proto.model_dir = FLAGS.model_dir tf.logging.info("Override model checkpoint dir: %s", FLAGS.model_dir) tf.logging.info("Pipeline configure: %s", '=' * 128) tf.logging.info(pipeline_proto) g = tf.Graph() with g.as_default(): model = builder.build(pipeline_proto.model, is_training=False) predictions = model.build_prediction( examples={}, prediction_task=GAPPredictionTasks.word_saliency) saver = tf.train.Saver() invalid_variable_names = tf.report_uninitialized_variables() with tf.Session(graph=g) as sess: sess.run(tf.tables_initializer()) checkpoint_path = tf.train.latest_checkpoint(pipeline_proto.model_dir) assert checkpoint_path is not None saver.restore(sess, checkpoint_path) assert len(sess.run(invalid_variable_names)) == 0 predictions = sess.run(predictions) (vocabulary, word_saliency, word_embedding) = (predictions[GAPPredictions.vocabulary], predictions[GAPPredictions.word_saliency], predictions[GAPPredictions.word_embedding]) vocabulary = [word.decode('UTF8') for word in vocabulary.tolist()] # Read the MSCOCO vocabulary. name_to_class_id = {} with open(FLAGS.name_to_class_id_file, 'r') as fid: for line in fid.readlines(): name, class_id = line.strip('\n').split('\t') name_to_class_id[name] = class_id queries = list(name_to_class_id) query_embedding = word_embedding[[ vocabulary.index(query) for query in queries ]] # Rank the words by importance. similarity_to_coco = np.matmul(word_embedding, query_embedding.transpose()) most_similar_coco_word_index = similarity_to_coco.argmax(axis=-1) indices = np.argsort(word_saliency)[::-1] for i in indices[:1000]: word, score = vocabulary[i], word_saliency[i] coco_i = most_similar_coco_word_index[i] print('%s,%.4lf,%s,%.4lf' % (word, score, queries[coco_i], similarity_to_coco[i][coco_i])) tf.logging.info('Done')
def main(_): pipeline_proto = _load_pipeline_proto(FLAGS.pipeline_proto) tf.logging.info("Pipeline configure: %s", '=' * 128) tf.logging.info(pipeline_proto) g = tf.Graph() with g.as_default(): # Infer saliency. image = tf.placeholder(tf.uint8, shape=[None, None, 3]) image_resized = tf.image.resize_images(image, [ pipeline_proto.eval_reader.image_height, pipeline_proto.eval_reader.image_width ]) model = builder.build(pipeline_proto.model, is_training=False) predictions = model.build_prediction( examples={InputDataFields.image: tf.expand_dims(image_resized, 0)}, prediction_task=GAPPredictionTasks.image_saliency) height, width = tf.shape(image)[0], tf.shape(image)[1] saliency = tf.image.resize_images( tf.expand_dims(predictions[GAPPredictions.image_saliency], axis=-1), [height, width], tf.image.ResizeMethod.NEAREST_NEIGHBOR)[:, :, :, 0] saver = tf.train.Saver() invalid_variable_names = tf.report_uninitialized_variables() with tf.Session(graph=g) as sess: # Load the latest checkpoint. checkpoint_path = tf.train.latest_checkpoint(pipeline_proto.model_dir) assert checkpoint_path is not None saver.restore(sess, checkpoint_path) assert len(sess.run(invalid_variable_names)) == 0 # Iterate the testdir to generate the demo results. saliency_func = lambda x: sess.run(saliency[0], feed_dict={image: x}) for filename in os.listdir(FLAGS.image_path): tf.logging.info('On processing %s', filename) _get_saliency(input_path=os.path.join(FLAGS.image_path, filename), output_path=os.path.join(FLAGS.demo_path, filename), saliency_func=saliency_func) tf.logging.info('Done')
def main(_): pipeline_proto = _load_pipeline_proto(FLAGS.pipeline_proto) tf.logging.info("Pipeline configure: %s", '=' * 128) tf.logging.info(pipeline_proto) g = tf.Graph() with g.as_default(): # Infer saliency. model = builder.build(pipeline_proto.model, is_training=False) predictions = model.build_prediction(examples={}, prediction_task=GAPPredictionTasks.word_saliency) vocabulary = predictions[GAPPredictions.vocabulary] saliency = predictions[GAPPredictions.word_saliency] saver = tf.train.Saver() invalid_variable_names = tf.report_uninitialized_variables() with tf.Session(graph=g) as sess: sess.run(tf.tables_initializer()) # Load the latest checkpoint. checkpoint_path = tf.train.latest_checkpoint(pipeline_proto.model_dir) assert checkpoint_path is not None saver.restore(sess, checkpoint_path) assert len(sess.run(invalid_variable_names)) == 0 # Print word importance. vocabulary, saliency = sess.run([vocabulary, saliency]) if FLAGS.ascending_order: indices = np.argsort(saliency) else: indices = np.argsort(saliency)[::-1] for i in indices[:FLAGS.top_k]: tf.logging.info("%12s: %.4lf", vocabulary[i].decode('UTF-8'), saliency[i]) print('%s\t%.4lf' % (vocabulary[i].decode('UTF-8'), saliency[i])) tf.logging.info('Done')
def _model_fn(features, labels, mode, params): """ Args: features: a dict mapping from names to tensors, denoting the features. labels: a dict mapping from names to tensors, denoting the labels. mode: mode parameter required by the estimator. params: additional parameters used for creating the model. Returns: an instance of EstimatorSpec. """ is_training = (tf.estimator.ModeKeys.TRAIN == mode) tf.logging.info("Current mode is %s, is_training=%s", mode, is_training) model = builder.build(pipeline_proto.model, is_training) predictions = model.build_prediction(features) # Get scaffold and variables_to_train. scaffold = model.get_scaffold() variables_to_train = model.get_variables_to_train() # Compute losses. Note: variables created in build_loss are not trainable. losses = model.build_loss(predictions, examples=features) for name, loss in losses.items(): tf.losses.add_loss(loss) tf.summary.scalar('loss/' + name, loss) for loss in tf.losses.get_regularization_losses(): tf.summary.scalar( "loss/regularization/" + '/'.join(loss.op.name.split('/')[:2]), loss) total_loss = tf.losses.get_total_loss(add_regularization_losses=True) train_op = None eval_metric_ops = None training_hooks = [] #update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) #for op in update_ops: # if 'cnn' not in op.name: # tf.logging.info(op.name) #import pdb #pdb.set_trace() if tf.estimator.ModeKeys.TRAIN == mode: train_config = pipeline_proto.train_config # Create the optimizer. learning_rate = train_config.learning_rate global_step = tf.train.get_or_create_global_step() if train_config.HasField('learning_rate_decay'): learning_rate = tf.train.exponential_decay( learning_rate, global_step, train_config.learning_rate_decay.decay_steps, train_config.learning_rate_decay.decay_rate, staircase=train_config.learning_rate_decay.staircase) tf.summary.scalar('loss/learning_rate', learning_rate) optimizer = training_utils.build_optimizer( train_config.optimizer, learning_rate=learning_rate) # Setup the replicas_hook for the SyncReplicasOptimizer. if train_config.sync_replicas: optimizer = tf.train.SyncReplicasOptimizer( optimizer, replicas_to_aggregate=4) sync_replicas_hook = optimizer.make_session_run_hook(is_chief) training_hooks.append(sync_replicas_hook) # Enable MovingAverageOptimizer if specified. if train_config.HasField('moving_average_decay'): optimizer = tf.contrib.opt.MovingAverageOptimizer( optimizer, average_decay=train_config.moving_average_decay) # Apply gradient multipliers. gradient_multipliers = {} for var in variables_to_train: for multiplier in train_config.gradient_multiplier: if var.op.name.startswith(multiplier.scope): if var.op.name in gradient_multipliers: tf.logging.warn('Override gradient multiplier: %s', var.op.name) gradient_multipliers[var.op.name] = multiplier.multiplier tf.logging.info('Set gradient multiplier for %s', var.op.name) tf.logging.info('Variable to train: %s, %s', var.op.name, var.get_shape()) tf.logging.info('Apply gradient multipliers: \n%s', json.dumps(gradient_multipliers, indent=2)) def transform_grads_fn(grads): if not gradient_multipliers: return grads return tf.contrib.training.multiply_gradients(grads, gradient_multipliers) # The train_op is required for mode `TRAIN`. train_op = tf.contrib.training.create_train_op( total_loss, optimizer, variables_to_train=variables_to_train, transform_grads_fn=transform_grads_fn, summarize_gradients=True) if train_config.HasField('moving_average_decay'): scaffold = tf.train.Scaffold( saver=optimizer.swapping_saver(), copy_from_scaffold=scaffold) elif tf.estimator.ModeKeys.EVAL == mode: # The eval_metric_ops is optional for mode `EVAL`. eval_metric_ops = model.build_evaluation(predictions, examples=features) elif tf.estimator.ModeKeys.PREDICT == mode: # The predictions is required for mode `PREDICT`. predictions.update(features) predictions.update({'summary': tf.summary.merge_all()}) return tf.estimator.EstimatorSpec( mode=mode, predictions=predictions, loss=total_loss, train_op=train_op, training_hooks=training_hooks, eval_metric_ops=eval_metric_ops, scaffold=scaffold)
def main(_): logging.set_verbosity(logging.DEBUG) for gpu in tf.config.experimental.list_physical_devices('GPU'): tf.config.experimental.set_memory_growth(gpu, True) pipeline_proto = _load_pipeline_proto(FLAGS.pipeline_proto) vocab = _load_vocab_file(FLAGS.vocab_file) # Get `next_examples_ts' tensor. if 'train' in FLAGS.output_jsonl_file: input_fn = reader.get_input_fn(pipeline_proto.train_reader, is_training=False) else: input_fn = reader.get_input_fn(pipeline_proto.eval_reader, is_training=False) iterator = input_fn().make_initializable_iterator() next_examples_ts = iterator.get_next() # Build model that takes placeholder as inputs, and predicts the logits. frcnn_dims = pipeline_proto.eval_reader.vcr_text_frcnn_reader.frcnn_feature_dims (label_pl, choices_pl, choices_tag_pl, choices_len_pl) = (tf.placeholder(tf.int32, [1]), tf.placeholder(tf.int32, [1, NUM_CHOICES, None]), tf.placeholder(tf.int32, [1, NUM_CHOICES, None]), tf.placeholder(tf.int32, [1, NUM_CHOICES])) (num_detections_pl, detection_boxes_pl, detection_classes_pl, detection_scores_pl, detection_features_pl) = (tf.placeholder(tf.int32, [1]), tf.placeholder(tf.float32, [1, None, 4]), tf.placeholder(tf.int32, [1, None]), tf.placeholder(tf.float32, [1, None]), tf.placeholder(tf.float32, [1, None, frcnn_dims])) model = builder.build(pipeline_proto.model, is_training=False) logits_ts = model.predict({ InputFields.num_detections: num_detections_pl, InputFields.detection_boxes: detection_boxes_pl, InputFields.detection_classes: detection_classes_pl, InputFields.detection_scores: detection_scores_pl, InputFields.detection_features: detection_features_pl, model._field_choices: choices_pl, model._field_choices_tag: choices_tag_pl, model._field_choices_len: choices_len_pl, })[FIELD_ANSWER_PREDICTION] losses_ts = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits_ts, labels=tf.one_hot( label_pl, depth=NUM_CHOICES)) saver = tf.train.Saver() # Find the latest checkpoint file. ckpt_path = tf.train.latest_checkpoint(FLAGS.model_dir) assert ckpt_path is not None def _calc_score_and_loss(choices, choice_tag, choices_len, label, num_detections, detection_boxes, detection_clases, detection_scores, detection_features): """Get the VCR matching scores and losses.""" (scores, losses) = sess.run( [logits_ts, losses_ts], feed_dict={ label_pl: np.expand_dims(label, 0), choices_pl: np.expand_dims(choices, 0), choices_tag_pl: np.expand_dims(choices_tag, 0), choices_len_pl: np.expand_dims(choices_len, 0), num_detections_pl: np.expand_dims(num_detections, 0), detection_boxes_pl: np.expand_dims(detection_boxes, 0), detection_classes_pl: np.expand_dims(detection_clases, 0), detection_scores_pl: np.expand_dims(detection_scores, 0), detection_features_pl: np.expand_dims(detection_features, 0), }) return scores[0], losses[0] # Run inference using the pretrained Bert model. with tf.Session() as sess, open(FLAGS.output_jsonl_file, 'w') as output_fp: sess.run(iterator.initializer) sess.run(tf.tables_initializer()) saver.restore(sess, ckpt_path) logging.info('Restore from %s.', ckpt_path) batch_id = 0 while True: batch_id += 1 try: inputs_batched = sess.run(next_examples_ts) batch_size = len(inputs_batched[InputFields.annot_id]) masks = np.array([[MASK_ID], [MASK_ID], [MASK_ID], [MASK_ID]]) ones = np.array([[1], [1], [1], [1]]) for example_id in range(batch_size): (annot_id, choices, choices_tag, choices_len, label) = ( inputs_batched[ InputFields.annot_id][example_id].decode('utf8'), inputs_batched[model._field_choices][example_id], inputs_batched[model._field_choices_tag][example_id], inputs_batched[model._field_choices_len][example_id], inputs_batched[model._field_label][example_id]) (num_detections, detection_boxes, detection_clases, detection_scores, detection_features) = ( inputs_batched[InputFields.num_detections] [example_id], inputs_batched[ InputFields.detection_boxes][example_id], inputs_batched[InputFields.detection_classes] [example_id], inputs_batched[ InputFields.detection_scores][example_id], inputs_batched[ InputFields.detection_features][example_id]) # Scores of the original choices. orig_scores, orig_losses = _calc_score_and_loss( choices, choices_tag, choices_len, label, num_detections, detection_boxes, detection_clases, detection_scores, detection_features) # Adversarial atacking. max_losses = np.zeros(NUM_CHOICES) max_losses_choices = choices if FLAGS.rationale: sep_pos = np.where(choices == SEP_ID)[1].take( [1, 3, 5, 7]) else: sep_pos = np.where(choices == SEP_ID)[1] result_losses = [[] for _ in range(4)] result_tokens = [[] for _ in range(4)] for pos_id in range(sep_pos.min() + 1, choices_len.max()): # Compute the new losses. new_choices = np.concatenate([ choices[:, :pos_id], masks, choices[:, pos_id + 1:] ], -1) new_choices_tag = np.concatenate([ choices_tag[:, :pos_id], -ones, choices_tag[:, pos_id + 1:] ], -1) scores, losses = _calc_score_and_loss( new_choices, new_choices_tag, choices_len, label, num_detections, detection_boxes, detection_clases, detection_scores, detection_features) # Update the maximum values. token_id = choices[:, pos_id] is_valid = np.logical_not( np.logical_or( token_id == PAD_ID, np.logical_or(token_id == CLS_ID, token_id == SEP_ID))) for choice_id in range(4): if is_valid[choice_id]: result_losses[choice_id].append( round(float(losses[choice_id]), 4)) result_tokens[choice_id].append( vocab[choices[choice_id][pos_id]]) # Maximize loss. adversarial_select_cond = np.logical_and( losses > max_losses, is_valid) max_losses_choices = np.where( np.expand_dims(adversarial_select_cond, -1), new_choices, max_losses_choices) max_losses = np.maximum(max_losses, losses) #END: for pos_id in range(sep_pos.min() + 1, choices_len.max()): choices = pack_tensor_values(choices, choices_len, vocab) adversarial_choices = pack_tensor_values( max_losses_choices, choices_len, vocab) output_annot = { 'annot_id': annot_id, 'label': int(label), 'choices': choices, 'adversarial_choices': adversarial_choices, 'result_losses': result_losses, 'result_tokens': result_tokens, } # print(label) # for i in range(4): # print(choices[i]) # print(adversarial_choices[i]) output_fp.write(json.dumps(output_annot) + '\n') if batch_id % 10 == 0: logging.info('batch_id=%i', batch_id) except tf.errors.OutOfRangeError as ex: logging.info('Done!') break output_fp.close()
def main(_): logging.set_verbosity(tf.logging.INFO) assert os.path.isfile(FLAGS.pipeline_proto) g = tf.Graph() with g.as_default(): pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto) logging.info("Pipeline configure: %s", '=' * 128) logging.info(pipeline_proto) # Get examples from reader. examples = ads_examples.get_examples(pipeline_proto.example_reader) # Build model for evaluation. global_step = slim.get_or_create_global_step() model = builder.build(pipeline_proto.model, is_training=False) predictions = model.build_inference(examples) loss_dict = model.build_loss(predictions) uninitialized_variable_names = tf.report_uninitialized_variables() saver = tf.train.Saver() init_op = tf.group(tf.local_variables_initializer(), tf.global_variables_initializer()) session_config = train_utils.default_session_config( FLAGS.per_process_gpu_memory_fraction) # Start session. logging.info('=' * 128) eval_config = pipeline_proto.eval_config model_path = train_utils.get_latest_model(FLAGS.saved_ckpts_dir) if model_path is not None: with tf.Session(graph=g, config=session_config) as sess: # Initialize model. sess.run(init_op) saver.restore(sess, model_path) logging.info('Restore from %s.', model_path) warn_names = sess.run(uninitialized_variable_names) assert len(warn_names) == 0 # Evaluation loop. step = sess.run(global_step) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) evaluate_once(sess, None, step, examples['video_id'], predictions=predictions) coord.request_stop() coord.join(threads, stop_grace_period_secs=10) logging.info('Done')
def _model_fn(features, labels, mode, params): """Creates the model. Args: features: A dict mapping from names to tensors, denoting the features. labels: A dict mapping from names to tensors, denoting the labels. mode: Mode parameter required by the estimator. params: Additional parameters used for creating the model. Returns: An instance of EstimatorSpec. """ is_training = (tf.estimator.ModeKeys.TRAIN == mode) logging.info("Current mode is %s, is_training=%s", mode, is_training) model = builder.build(pipeline_proto.model, is_training) # Predict resutls. predictions = model.predict(features) # Compute losses. Note: variables created in build_loss are not trainable. losses = model.build_losses(features, predictions) for name, loss in losses.items(): tf.compat.v1.summary.scalar('losses/' + name, loss) tf.losses.add_loss(loss) for loss in tf.compat.v1.losses.get_regularization_losses(): tf.summary.scalar( "regularization/" + '/'.join(loss.op.name.split('/')[:2]), loss) total_loss = tf.compat.v1.losses.get_total_loss( add_regularization_losses=True) # Get variables_to_train. variables_to_train = model.get_variables_to_train() scaffold = model.get_scaffold() train_op = None eval_metric_ops = None if tf.estimator.ModeKeys.TRAIN == mode: _summarize_variables(tf.compat.v1.global_variables()) global_step = tf.compat.v1.train.get_global_step() # Set learning rate. train_config = pipeline_proto.train_config lr_schedule_fn = learning_rate_schedule.create_learning_rate_schedule( train_config.learning_rate_schedule) learning_rate = lr_schedule_fn(global_step) tf.compat.v1.summary.scalar('losses/learning_rate', learning_rate) # Use optimizer to minimize loss. optimizer = optimization.create_optimizer( train_config.optimizer, learning_rate=learning_rate) def transform_grads_fn(grads): if train_config.HasField('max_gradient_norm'): grads = tf.contrib.training.clip_gradient_norms( grads, max_norm=train_config.max_gradient_norm) return grads train_op = tf.contrib.training.create_train_op( total_loss, optimizer, variables_to_train=variables_to_train, transform_grads_fn=transform_grads_fn, summarize_gradients=True) elif tf.estimator.ModeKeys.EVAL == mode: eval_metric_ops = model.build_metrics(features, predictions) for name, loss in losses.items(): loss_metric = tf.keras.metrics.Mean() loss_metric.update_state(loss) eval_metric_ops['losses/' + name] = loss_metric elif tf.estimator.ModeKeys.PREDICT == mode: # Add input tensors to the predictions. predictions.update(features) # Create additional tensors if specified. create_additional_predictions = params.get( 'create_additional_predictions', None) if create_additional_predictions: assert callable(create_additional_predictions) predictions.update( create_additional_predictions(tf.get_default_graph())) # Merge summaries. summary_saver_hook = tf.estimator.SummarySaverHook( summary_op=tf.compat.v1.summary.merge_all(), save_steps=pipeline_proto.train_config.save_summary_steps) return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions, loss=total_loss, train_op=train_op, eval_metric_ops=eval_metric_ops, training_hooks=[summary_saver_hook], scaffold=scaffold)
def main(_): pipeline_proto = _load_pipeline_proto(FLAGS.pipeline_proto) if FLAGS.model_dir: pipeline_proto.model_dir = FLAGS.model_dir tf.logging.info("Override model checkpoint dir: %s", FLAGS.model_dir) tf.logging.info("Pipeline configure: %s", '=' * 128) tf.logging.info(pipeline_proto) categories = [ 'bear', 'beach', 'car', 'motorcycle', 'light', 'donut', 'plate', 'person', 'skateboard' ] g = tf.Graph() with g.as_default(): image = tf.placeholder(tf.uint8, shape=[None, None, 3]) image_resized = tf.image.resize_images(image, [ pipeline_proto.eval_reader.image_height, pipeline_proto.eval_reader.image_width ]) model = builder.build(pipeline_proto.model, is_training=False) prediction_dict = model.build_prediction( examples={ InputDataFields.image: tf.expand_dims(image_resized, 0), InputDataFields.category_strings: tf.constant(categories) }, prediction_task=GAPPredictionTasks.image_score_map) # height, width = tf.shape(image)[0], tf.shape(image)[1] (saliency_map, score_maps) = (prediction_dict[GAPPredictions.image_saliency], prediction_dict[GAPPredictions.image_score_map]) score_map_list = [tf.squeeze(saliency_map, axis=-1)] + tf.unstack( score_maps, axis=-1) tf.logging.info("score map list size: %d", len(score_map_list)) for i, x in enumerate(score_map_list): tf.logging.info("%d: shape: %s", i, x.get_shape().as_list()) saver = tf.train.Saver() invalid_variable_names = tf.report_uninitialized_variables() with tf.Session(graph=g) as sess: sess.run(tf.tables_initializer()) # Load the latest checkpoint. checkpoint_path = tf.train.latest_checkpoint(pipeline_proto.model_dir) assert checkpoint_path is not None saver.restore(sess, checkpoint_path) assert len(sess.run(invalid_variable_names)) == 0 # Iterate the testdir to generate the demo results. score_map_func = lambda x: sess.run(score_map_list, feed_dict={image: x}) for filename in os.listdir(FLAGS.image_path): tf.logging.info('On processing %s', filename) _get_score_map( input_path=os.path.join(FLAGS.image_path, filename), output_path=os.path.join(FLAGS.demo_path, filename), names=['saliency'] + categories, score_map_func=score_map_func, shape=(pipeline_proto.eval_reader.image_height, pipeline_proto.eval_reader.image_width)) tf.logging.info('Done')
def main(_): logging.set_verbosity(tf.logging.INFO) assert os.path.isfile(FLAGS.pipeline_proto) assert os.path.isfile(FLAGS.action_reason_annot_path) pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto) logging.info("Pipeline configure: %s", '=' * 128) logging.info(pipeline_proto) groundtruths = load_action_reason_annots(FLAGS.action_reason_annot_path) g = tf.Graph() with g.as_default(): # Get examples from reader. split = 'valid' if not FLAGS.continuous_evaluation: split = 'test' examples, feed_init_fn = ads_mem_examples.get_examples( pipeline_proto.example_reader, split) # Build model for training. global_step = slim.get_or_create_global_step() model = builder.build(pipeline_proto.model, is_training=False) predictions = model.build_evaluation_graph(examples) init_fn = model.get_init_fn() uninitialized_variable_names = tf.report_uninitialized_variables() saver = tf.train.Saver() init_op = tf.group(tf.local_variables_initializer(), tf.global_variables_initializer()) session_config = train_utils.default_session_config( FLAGS.per_process_gpu_memory_fraction) # evaluation on test set. logging.info('Start evaluating.') eval_config = pipeline_proto.eval_config # One time evaluation and inference. if not FLAGS.continuous_evaluation: model_path = train_utils.get_latest_model(FLAGS.saved_ckpt_dir) with tf.Session(graph=g, config=session_config) as sess: feed_init_fn(sess) sess.run(init_op) saver.restore(sess, model_path) logging.info('Restore model from %s.', model_path) warn_names = sess.run(uninitialized_variable_names) assert len(warn_names) == 0 # Evaluate the best model in terms of recall@3. step = sess.run(global_step) evaluate_once(sess, None, step, predictions, groundtruths) logging.info('Done') exit(0) # Continuous evaluation on valid set. writer = tf.summary.FileWriter(FLAGS.eval_log_dir, g) step = prev_step = -1 while True: start = time.time() try: model_path = tf.train.latest_checkpoint(FLAGS.train_log_dir) if model_path is not None: with tf.Session(graph=g, config=session_config) as sess: # Restore model. feed_init_fn(sess) sess.run(init_op) saver.restore(sess, model_path) logging.info('Restore model from %s.', model_path) warn_names = sess.run(uninitialized_variable_names) assert len(warn_names) == 0 step = sess.run(global_step) if step != prev_step and step > eval_config.eval_min_global_steps: # Evaluate the latest model. prev_step = step metric = evaluate_once(sess, writer, step, predictions, groundtruths) step_best, metric_best = train_utils.save_model_if_it_is_better( step, metric, model_path, FLAGS.saved_ckpt_dir, reverse=False) if step_best == step: summary = tf.Summary() summary.value.add(tag='metrics/model_metric', simple_value=metric_best) writer.add_summary(summary, global_step=step) writer.flush() # with tf.Session # if model_path is not None except Exception as ex: pass if step >= eval_config.number_of_steps: break sleep_secs = eval_config.eval_interval_secs - (time.time() - start) if sleep_secs > 0: logging.info('Now sleep for %.2lf secs.', sleep_secs) time.sleep(sleep_secs) writer.close() logging.info('Done')
def main(_): logging.set_verbosity(tf.logging.INFO) assert os.path.isfile(FLAGS.pipeline_proto) assert os.path.isfile(FLAGS.sentiment_vocab_path) assert os.path.isfile(FLAGS.sentiment_anno_vocab) assert os.path.isfile(FLAGS.sentiment_raw_annot_path) assert os.path.isfile(FLAGS.sentiment_clean_annot_path) sentiment_raw_annot = load_raw_annot(FLAGS.sentiment_raw_annot_path) sentiment_clean_annot = eval_utils.load_clean_annot(FLAGS.sentiment_clean_annot_path) topic_clean_annot = load_clean_annot(FLAGS.topic_clean_annot_path) g = tf.Graph() with g.as_default(): pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto) logging.info("Pipeline configure: %s", '=' * 128) logging.info(pipeline_proto) # Get examples from reader. examples = ads_examples.get_examples(pipeline_proto.example_reader) # Build model for evaluation. global_step = slim.get_or_create_global_step() model = builder.build(pipeline_proto.model, is_training=False) predictions = model.build_inference(examples) loss_dict = model.build_loss(predictions) uninitialized_variable_names = tf.report_uninitialized_variables() saver = tf.train.Saver() init_op = tf.group(tf.local_variables_initializer(), tf.global_variables_initializer()) session_config = train_utils.default_session_config( FLAGS.per_process_gpu_memory_fraction) # Start session. logging.info('=' * 128) eval_config = pipeline_proto.eval_config writer = tf.summary.FileWriter(FLAGS.eval_log_dir, g) prev_step = -1 while True: start = time.time() try: model_path = tf.train.latest_checkpoint(FLAGS.train_log_dir) if model_path is not None: with tf.Session(graph=g, config=session_config) as sess: # Initialize model. sess.run(init_op) saver.restore(sess, model_path) logging.info('Restore from %s.', model_path) warn_names = sess.run(uninitialized_variable_names) assert len(warn_names) == 0 step = sess.run(global_step) if step != prev_step and step > eval_config.eval_min_global_steps: prev_step = step # Evaluation loop. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) model_metric = evaluate_sentiment_once(sess, writer, step, examples['video_id'], predictions=predictions, clean_annot=sentiment_clean_annot, raw_annot=sentiment_raw_annot, use_sigmoid=True) step_best, metric_best = train_utils.save_model_if_it_is_better( step, model_metric, model_path, FLAGS.saved_ckpts_dir, reverse=False) # We improved the model, record it. if step_best == step: summary = tf.Summary() summary.value.add( tag='metrics/metric_best', simple_value=metric_best) writer.add_summary(summary, global_step=step) writer.flush() coord.request_stop() coord.join(threads, stop_grace_period_secs=10) if step >= eval_config.number_of_steps: break except Exception as ex: pass # Sleep a while. sleep_secs = eval_config.eval_interval_secs - (time.time() - start) if sleep_secs > 0: logging.info('Now sleep for %.2lf secs.', sleep_secs) time.sleep(sleep_secs) writer.close() logging.info('Done')
def main(_): logging.set_verbosity(tf.logging.INFO) assert os.path.isfile(FLAGS.pipeline_proto) g = tf.Graph() with g.as_default(): pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto) logging.info("Pipeline configure: %s", '=' * 128) logging.info(pipeline_proto) train_config = pipeline_proto.train_config # Get examples from reader. examples, feed_init_fn = ads_mem_examples.get_examples( pipeline_proto.example_reader, split='train') # Build model for training. model = builder.build(pipeline_proto.model, is_training=True) predictions = model.build_inference_graph(examples) loss_dict = model.build_loss(predictions) model_init_fn = model.get_init_fn() uninitialized_variable_names = tf.report_uninitialized_variables() if FLAGS.restore_from: variables_to_restore = slim.get_variables_to_restore( exclude=[name for name in train_config.exclude_variable]) restore_init_fn = slim.assign_from_checkpoint_fn( FLAGS.restore_from, variables_to_restore) def init_fn(sess): model_init_fn(sess) if FLAGS.restore_from: restore_init_fn(sess) # Loss and optimizer. for loss_name, loss_tensor in loss_dict.iteritems(): tf.losses.add_loss(loss_tensor) tf.summary.scalar('losses/{}'.format(loss_name), loss_tensor) total_loss = tf.losses.get_total_loss() tf.summary.scalar('losses/total_loss', total_loss) for reg_loss in tf.losses.get_regularization_losses(): name = 'losses/reg_loss_{}'.format(reg_loss.op.name.split('/')[0]) tf.summary.scalar(name, reg_loss) optimizer = train_utils.build_optimizer(train_config) if train_config.moving_average: optimizer = tf.contrib.opt.MovingAverageOptimizer( optimizer, average_decay=0.99) gradient_multipliers = train_utils.build_multipler( train_config.gradient_multiplier) variables_to_train = model.get_variables_to_train() logging.info('=' * 128) for var in variables_to_train: logging.info(var) train_op = slim.learning.create_train_op( total_loss, variables_to_train=variables_to_train, clip_gradient_norm=0.0, gradient_multipliers=gradient_multipliers, summarize_gradients=True, optimizer=optimizer) saver = None if train_config.moving_average: saver = optimizer.swapping_saver() # Start checking. logging.info('Start checking...') session_config = train_utils.default_session_config( FLAGS.per_process_gpu_memory_fraction) def _session_wrapper_fn(sess): feed_init_fn(sess) return sess slim.learning.train(train_op, logdir=FLAGS.train_log_dir, graph=g, master='', is_chief=True, number_of_steps=train_config.number_of_steps, log_every_n_steps=train_config.log_every_n_steps, save_interval_secs=train_config.save_interval_secs, save_summaries_secs=train_config.save_summaries_secs, session_config=session_config, session_wrapper=_session_wrapper_fn, init_fn=init_fn, saver=saver) logging.info('Done')
def main(_): pipeline_proto = _load_pipeline_proto(FLAGS.pipeline_proto) tf.logging.info("Pipeline configure: %s", '=' * 128) tf.logging.info(pipeline_proto) g = tf.Graph() with g.as_default(): # Infer saliency. model = builder.build(pipeline_proto.model, is_training=False) predictions = model.build_prediction( examples={}, prediction_task=GAPPredictionTasks.word_saliency) saver = tf.train.Saver() invalid_variable_names = tf.report_uninitialized_variables() with tf.Session(graph=g) as sess: sess.run(tf.tables_initializer()) # Load the latest checkpoint. checkpoint_path = tf.train.latest_checkpoint(pipeline_proto.model_dir) assert checkpoint_path is not None saver.restore(sess, checkpoint_path) assert len(sess.run(invalid_variable_names)) == 0 predictions = sess.run(predictions) # Process kNN retrieval. name_to_class_id = {} with open(FLAGS.name_to_class_id_file, 'r') as fid: for line in fid.readlines(): name, class_id = line.strip('\n').split('\t') name_to_class_id[name] = class_id (vocabulary, word_saliency, word_embedding) = (predictions[GAPPredictions.vocabulary], predictions[GAPPredictions.word_saliency], predictions[GAPPredictions.word_embedding]) queries = list(name_to_class_id) synonyms_list = _knn_retrieval(queries, vocabulary, word_embedding, word_saliency) # Print to the terminal. expanded_name_to_class_id = [] for query, synonyms in zip(queries, synonyms_list): elems = [] for synonym in synonyms: if synonym['saliency'] < FLAGS.saliency_threshold: continue if synonym['similarity'] < FLAGS.similarity_threshold: continue elems.append(synonym['word']) expanded_name_to_class_id.append((synonym['word'], name_to_class_id[query])) print('%s\t%s' % (query, ','.join(elems))) # Write to output file. with open(FLAGS.expanded_name_to_class_id_file, 'w') as fid: for word, class_id in expanded_name_to_class_id: fid.write('%s\t%s\n' % (word, class_id)) tf.logging.info('Done')
def main(_): logging.set_verbosity(tf.logging.INFO) assert os.path.isfile(FLAGS.pipeline_proto), FLAGS.pipeline_proto g = tf.Graph() with g.as_default(): pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto) logging.info("Pipeline configure: %s", '=' * 128) logging.info(pipeline_proto) train_config = pipeline_proto.train_config # Get examples from reader. examples = ads_examples.get_examples(pipeline_proto.example_reader) # Build model for training. global_step = slim.get_or_create_global_step() model = builder.build(pipeline_proto.model, is_training=True) predictions = model.build_inference(examples) loss_dict = model.build_loss(predictions) init_fn = model.get_init_fn() uninitialized_variable_names = tf.report_uninitialized_variables() # Loss and optimizer. for loss_name, loss_tensor in loss_dict.iteritems(): tf.losses.add_loss(loss_tensor) tf.summary.scalar('losses/{}'.format(loss_name), loss_tensor) total_loss = tf.losses.get_total_loss() tf.summary.scalar('losses/total_loss', total_loss) optimizer = train_utils.build_optimizer(train_config) if train_config.moving_average: optimizer = tf.contrib.opt.MovingAverageOptimizer(optimizer, average_decay=0.99) gradient_multipliers = train_utils.build_multipler( train_config.gradient_multiplier) variables_to_train = model.get_variables_to_train() for var in variables_to_train: logging.info(var) train_op = slim.learning.create_train_op(total_loss, variables_to_train=variables_to_train, clip_gradient_norm=5.0, gradient_multipliers=gradient_multipliers, summarize_gradients=True, optimizer=optimizer) saver = None if train_config.moving_average: saver = optimizer.swapping_saver() # Starts training. logging.info('Start training.') session_config = train_utils.default_session_config( FLAGS.per_process_gpu_memory_fraction) slim.learning.train(train_op, logdir=FLAGS.train_log_dir, graph=g, master='', is_chief=True, number_of_steps=train_config.number_of_steps, log_every_n_steps=train_config.log_every_n_steps, save_interval_secs=train_config.save_interval_secs, save_summaries_secs=train_config.save_summaries_secs, session_config=session_config, init_fn=init_fn, saver=saver) logging.info('Done')