def evaluate(): g = tf.Graph() with g.as_default(): # load data get iterator data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE, NUM_EPOCHS) iterator = data_loader.load_data(TEST_TFR_PATH, False) frameNo, image, label = iterator.get_next() # define model graph # VGG FACE network VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE) image_batch = tf.reshape(image, [-1, 96, 96, 3]) VGGFace_network.setup( image_batch ) # image_batch is a tensor of shape (batch_size*seq_length,image_dim,image_dim,3) face_output = VGGFace_network.get_face_fc0() # RNN part rnn_in = reshape_to_rnn(face_output) prediction = models.get_prediction_atten(rnn_in, attn_length=30) prediction = tf.reshape(prediction, [-1, 2]) label_batch = tf.reshape(label, [-1, 2]) with tf.Session(graph=g) as sess: restore_variables(sess) evaluated_predictions = [] evaluated_labels = [] while True: try: pred, lab = sess.run([prediction, label_batch]) evaluated_predictions.append(pred) evaluated_labels.append(lab) except tf.errors.OutOfRangeError: break print 'Finish read data' predictions = np.reshape(evaluated_predictions, (-1, 2)) labels = np.reshape(evaluated_labels, (-1, 2)) for i, name in enumerate(['valence', 'arousal']): result = np.stack((predictions[:, i], labels[:, i]), axis=-1) np.savetxt(SUMMARY_PATH + name + '_test.txt', result, fmt="%.3f") conc_arousal = concordance_cc2(predictions[:, 1], labels[:, 1]) conc_valence = concordance_cc2(predictions[:, 0], labels[:, 0]) mse_arousal = sum( (predictions[:, 1] - labels[:, 1])**2) / len(labels[:, 1]) mse_valence = sum( (predictions[:, 0] - labels[:, 0])**2) / len(labels[:, 0]) print '#####################Summary#######################' print 'Concordance on valence : {}'.format(conc_valence) print 'Concordance on arousal : {}'.format(conc_arousal) print 'Concordance on total : {}'.format( (conc_arousal + conc_valence) / 2) print 'MSE Arousal : {}'.format(mse_arousal) print 'MSE Valence : {}'.format(mse_valence)
def train(): g = tf.Graph() with g.as_default(): # load data get iterator data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE, NUM_EPOCHS) iterator = data_loader.load_data(TRAIN_TFR_PATH, True) with tf.Session(graph=g) as sess: frameNo, image, label = iterator.get_next() # Construct and return the model image_batch = reshape_to_cnn(image) network_fn = nets_factory.get_network_fn( name='densenet121', num_classes=None, weight_decay=0.00004, data_format='NHWC', is_training=True ) face_output, _ = network_fn(image_batch) # print face_output.get_shape().as_list() # RNN part rnn_in = reshape_to_rnn(face_output) prediction = models.get_prediction_atten(rnn_in) prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) # compute losses using slim # compute_loss(prediction, label_batch) # total_loss = slim.losses.get_total_loss() # optimizer = tf.train.AdamOptimizer(LEARNING_RATE) # restore VGG-FACE model at the beginning # when the network only contains DenseNet all variables can be restored, # so here we restore global variables with ignore_missing_vars = True variables_to_restore = tf.global_variables() init_fn = slim.assign_from_checkpoint_fn(DenseNet_RESTORE_PATH, variables_to_restore, ignore_missing_vars=True) # summarize_gradients : Whether or not add summaries for each gradient. # variables_to_train: an optional list of variables to train. If None, it will default to all tf.trainable_variables(). trainable_names = ['attention-layer', 'fcatten'] variables_to_train = tf.contrib.framework.get_variables_to_restore(include=trainable_names) print_var(variables_to_train) print 'global vars' print_var(tf.global_variables())
def evaluate(): g = tf.Graph() with g.as_default(): # load data get iterator data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE, NUM_EPOCHS) iterator = data_loader.load_data(TEST_TFR_PATH, False) frameNo, image, label = iterator.get_next() # define model graph image_batch = tf.reshape(image, [-1, 96, 96, 3]) # Construct and return the model network_fn = nets_factory.get_network_fn(name='densenet161', num_classes=None, weight_decay=0.00004, data_format='NHWC', is_training=False) face_output, _ = network_fn(image_batch) # RNN part rnn_in = reshape_to_rnn(face_output) prediction = models.get_prediction_atten(rnn_in) prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) # Computing MSE and Concordance values, and adding them to summary names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ 'eval/mse_valence': slim.metrics.streaming_mean_squared_error(prediction[:, :, 0], label_batch[:, :, 0]), 'eval/mse_arousal': slim.metrics.streaming_mean_squared_error(prediction[:, :, 1], label_batch[:, :, 1]), }) summary_ops = [] conc_total = 0 mse_total = 0 for i, name in enumerate(['valence', 'arousal']): with tf.name_scope(name) as scope: concordance_cc2, values, updates = metrics.concordance_cc2( tf.reshape(prediction[:, :, i], [-1]), tf.reshape(label_batch[:, :, i], [-1])) for n, v in updates.items(): names_to_updates[n + '/' + name] = v op = tf.summary.scalar('eval/concordance_' + name, concordance_cc2) op = tf.Print(op, [concordance_cc2], 'eval/concordance_' + name) summary_ops.append(op) mse_eval = 'eval/mse_' + name op = tf.summary.scalar(mse_eval, names_to_values[mse_eval]) op = tf.Print(op, [names_to_values[mse_eval]], mse_eval) summary_ops.append(op) mse_total += names_to_values[mse_eval] conc_total += concordance_cc2 conc_total = conc_total / 2 mse_total = mse_total / 2 op = tf.summary.scalar('eval/concordance_total', conc_total) op = tf.Print(op, [conc_total], 'eval/concordance_total') summary_ops.append(op) op = tf.summary.scalar('eval/mse_total', mse_total) op = tf.Print(op, [mse_total], 'eval/mse_total') summary_ops.append(op) num_batches = int(NUM_BATCHES) loggingTF.set_verbosity(1) if not os.path.exists(SUMMARY_PATH): os.makedirs(SUMMARY_PATH) slim.evaluation.evaluate_once( '', MODEL_PATH, SUMMARY_PATH, num_evals=num_batches, eval_op=list(names_to_updates.values()), summary_op=tf.summary.merge(summary_ops), )
def train(): g = tf.Graph() with g.as_default(): # load data get iterator data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE, NUM_EPOCHS) iterator = data_loader.load_data(TRAIN_TFR_PATH, True) with tf.Session(graph=g) as sess: frameNo, image, label = iterator.get_next() # Construct and return the model image_batch = reshape_to_cnn(image) with slim.arg_scope(resnet_v1.resnet_arg_scope()): face_output, _ = resnet_v1.resnet_v1_50( inputs=image_batch, num_classes=None, is_training=True, global_pool=True, ) # print face_output.get_shape().as_list() # RNN part rnn_in = reshape_to_rnn(face_output) prediction = models.get_prediction_atten(rnn_in, attn_length=30) prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) # compute losses using slim compute_loss(prediction, label_batch) total_loss = slim.losses.get_total_loss() optimizer = tf.train.AdamOptimizer(LEARNING_RATE) # restore VGG-FACE model at the beginning # when the network only contains DenseNet all variables can be restored, # so here we restore global variables with ignore_missing_vars = True variables_to_restore = tf.global_variables() init_fn = slim.assign_from_checkpoint_fn(ResNet_RESTORE_PATH, variables_to_restore, ignore_missing_vars=True) # init_fn(sess) # summarize_gradients : Whether or not add summaries for each gradient. # variables_to_train: an optional list of variables to train. If None, it will default to all tf.trainable_variables(). trainable_names = ['attention-layer', 'fcatten'] variables_to_train = tf.contrib.framework.get_variables_to_restore(include=trainable_names) # print_var(variables_to_train) # print_var(tf.global_variables()) train_op = slim.learning.create_train_op(total_loss, optimizer, variables_to_train=variables_to_train, summarize_gradients=True # Whether or not add summaries for each gradient. ) loggingTF.set_verbosity(1) # keep 10000 ckpts saver = tf.train.Saver(max_to_keep=10000) # including initialize local and global variables slim.learning.train(train_op, TRAIN_DIR, init_fn=init_fn, save_summaries_secs=60 * 15, # How often, in seconds, to save summaries. log_every_n_steps=500, # The frequency, in terms of global steps, that the loss and global step are logged. save_interval_secs=60 * 15, # How often, in seconds, to save the model to `logdir`. saver=saver )
def train(): g = tf.Graph() with g.as_default(): # load data get iterator data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE, NUM_EPOCHS) iterator = data_loader.load_data(TRAIN_TFR_PATH, True) with tf.Session(graph=g) as sess: frameNo, image, label = iterator.get_next() # VGG FACE network VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE) image_batch = reshape_to_cnn(image) VGGFace_network.setup(image_batch, trainable=True) face_output = VGGFace_network.get_face_fc0() # RNN part rnn_in = reshape_to_rnn(face_output) prediction = models.get_prediction_atten(rnn_in, attn_length=30) prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) # compute losses using slim compute_loss(prediction, label_batch) total_loss = slim.losses.get_total_loss() optimizer = tf.train.AdamOptimizer(LEARNING_RATE) # restore VGG-FACE model at the beginning flow_init_assign_op, flow_init_feed_dict = slim.assign_from_checkpoint( RNN_RESTORE_PATH, get_RNN_variables(), ignore_missing_vars=False) resnet_init_assign_op, resnet_init_feed_dict = slim.assign_from_checkpoint( VGG_RESTORE_PATH, get_CNN_variables(), ignore_missing_vars=False) def init_fn(sess): sess.run(flow_init_assign_op, flow_init_feed_dict) sess.run(resnet_init_assign_op, resnet_init_feed_dict) # print_var(tf.global_variables()) # print_var(get_CNN_variables()) # print_var(get_RNN_variables()) # summarize_gradients : Whether or not add summaries for each gradient. # variables_to_train: an optional list of variables to train. If None, it will default to all tf.trainable_variables(). train_op = slim.learning.create_train_op(total_loss, optimizer, summarize_gradients=True # Whether or not add summaries for each gradient. ) loggingTF.set_verbosity(1) # keep 10000 ckpts saver = tf.train.Saver(max_to_keep=10000) # including initialize local and global variables slim.learning.train(train_op, TRAIN_DIR, init_fn=init_fn, save_summaries_secs=60 * 15, # How often, in seconds, to save summaries. log_every_n_steps=500, # The frequency, in terms of global steps, that the loss and global step are logged. save_interval_secs=60 * 15, # How often, in seconds, to save the model to `logdir`. saver=saver )
def train(): g = tf.Graph() with g.as_default(): # load data get iterator data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE, NUM_EPOCHS) iterator = data_loader.load_data(TRAIN_TFR_PATH, True) with tf.Session(graph=g) as sess: frameNo, image, label = iterator.get_next() # Construct and return the model image_batch = reshape_to_cnn(image) # 'face2_gru_atten' : trainable=False, 'face2_gru_atten_trainCNN' : trainable=True, resnet = ResNet50({'data': image_batch}, trainable=False) face_output = resnet.get_output() # print face_output.get_shape().as_list() # resnet.load(VF2_RESTORE_PATH, sess) # saver_model = tf.train.Saver() # MODEL_PATH = '/vol/gpudata/ml9915/caffe_model/resnet50_scratch_caffe/model.ckpt-0' # saver_model.save(sess, MODEL_PATH, global_step=0) # RNN part rnn_in = reshape_to_rnn(face_output) prediction = models.get_prediction_atten(rnn_in) prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) # compute losses using slim compute_loss(prediction, label_batch) total_loss = slim.losses.get_total_loss() optimizer = tf.train.AdamOptimizer(LEARNING_RATE) # restore VGG-FACE model at the beginning # restore_names = get_restore_variable() # variables_to_restore = tf.contrib.framework.get_variables_to_restore(include=restore_names) # print_var(variables_to_restore) init_fn = slim.assign_from_checkpoint_fn(ResNet50_RESTORE_PATH, tf.global_variables(), ignore_missing_vars=True) # init_fn(sess) # summarize_gradients : Whether or not add summaries for each gradient. # variables_to_train: an optional list of variables to train. If None, it will default to all tf.trainable_variables(). trainable_names = ['attention-layer', 'fcatten'] variables_to_train = tf.contrib.framework.get_variables_to_restore( include=trainable_names) # print_var(tf.trainable_variables()) train_op = slim.learning.create_train_op( total_loss, optimizer, variables_to_train=variables_to_train, summarize_gradients=True # Whether or not add summaries for each gradient. ) loggingTF.set_verbosity(1) # keep 10000 ckpts saver = tf.train.Saver(max_to_keep=10000) # including initialize local and global variables slim.learning.train( train_op, TRAIN_DIR, init_fn=init_fn, save_summaries_secs=60 * 15, # How often, in seconds, to save summaries. log_every_n_steps=500, # The frequency, in terms of global steps, that the loss and global step are logged. save_interval_secs=60 * 15, # How often, in seconds, to save the model to `logdir`. saver=saver)
def evaluate(): g = tf.Graph() with g.as_default(): # load data get iterator data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE, NUM_EPOCHS) iterator = data_loader.load_data(VALID_TFR_PATH, False) frameNo, image, label = iterator.get_next() # define model graph # VGG FACE network VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE) image_batch = tf.reshape(image, [-1, 96, 96, 3]) VGGFace_network.setup( image_batch, trainable=False ) # image_batch is a tensor of shape (batch_size*seq_length,image_dim,image_dim,3) face_output = VGGFace_network.get_face_fc0() # RNN part rnn_in = reshape_to_rnn(face_output) prediction = models.get_prediction_atten(rnn_in, attn_length=30) prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2]) # Computing MSE and Concordance values, and adding them to summary names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ 'eval/mse_valence': slim.metrics.streaming_mean_squared_error(prediction[:, :, 0], label_batch[:, :, 0]), 'eval/mse_arousal': slim.metrics.streaming_mean_squared_error(prediction[:, :, 1], label_batch[:, :, 1]), }) summary_ops = [] conc_total = 0 mse_total = 0 for i, name in enumerate(['valence', 'arousal']): with tf.name_scope(name) as scope: concordance_cc2, values, updates = metrics.concordance_cc2( tf.reshape(prediction[:, :, i], [-1]), tf.reshape(label_batch[:, :, i], [-1])) for n, v in updates.items(): names_to_updates[n + '/' + name] = v op = tf.summary.scalar('eval/concordance_' + name, concordance_cc2) op = tf.Print(op, [concordance_cc2], 'eval/concordance_' + name) summary_ops.append(op) mse_eval = 'eval/mse_' + name op = tf.summary.scalar(mse_eval, names_to_values[mse_eval]) op = tf.Print(op, [names_to_values[mse_eval]], mse_eval) summary_ops.append(op) mse_total += names_to_values[mse_eval] conc_total += concordance_cc2 conc_total = conc_total / 2 mse_total = mse_total / 2 op = tf.summary.scalar('eval/concordance_total', conc_total) op = tf.Print(op, [conc_total], 'eval/concordance_total') summary_ops.append(op) op = tf.summary.scalar('eval/mse_total', mse_total) op = tf.Print(op, [mse_total], 'eval/mse_total') summary_ops.append(op) num_batches = int(NUM_BATCHES) loggingTF.set_verbosity(1) if not os.path.exists(SUMMARY_PATH): os.makedirs(SUMMARY_PATH) # always check latest ckpt and wait for next. slim.evaluation.evaluation_loop( '', CHECK_POINT_DIR, SUMMARY_PATH, num_evals=num_batches, eval_op=list(names_to_updates.values()), summary_op=tf.summary.merge(summary_ops), eval_interval_secs=EVAL_INTERVAL_SECS, )