def _init_decoder_train_connectors(self):
with tf.name_scope('DecoderTrainFeeds'):
sequence_size,batch_size = tf.unstack(tf.shape(self.decoder_targets))
self.EOS_SLICE = tf.ones([1, batch_size], dtype=tf.int32) * self.EOS
self.PAD_SLICE = tf.ones([1, batch_size], dtype=tf.int32) * self.PAD
self.decoder_train_inputs = tf.concat([self.EOS_SLICE, self.decoder_targets], axis=0)
self.decoder_train_length = self.decoder_targets_length + 1
decoder_train_targets = tf.concat([self.decoder_targets, self.PAD_SLICE], axis=0)
self.decoder_train_targets_seq_len,_= tf.unstack(tf.shape(decoder_train_targets))
decoder_train_targets_eos_mask = tf.one_hot(self.decoder_train_length - 1,
self.decoder_train_targets_seq_len,
on_value=self.EOS, off_value=self.PAD,
dtype=tf.int32)
self.decoder_train_targets_eos_mask = tf.transpose(decoder_train_targets_eos_mask, [1, 0])
self.temp_decoder_train_targets = decoder_train_targets
# hacky way using one_hot to put EOS symbol at the end of target sequence
decoder_train_targets = tf.add(decoder_train_targets,
self.decoder_train_targets_eos_mask)
self.decoder_train_targets = decoder_train_targets
self.loss_weights = tf.ones([
batch_size,
tf.reduce_max(self.decoder_train_length)
], dtype=tf.float32, name="loss_weights")
def merge(inputs, targets): """Split inputs and targets into lists.""" inputs = tf.unstack(inputs, axis=1) targets = tf.unstack(targets, axis=1) assert len(inputs) == hparams.video_num_input_frames assert len(targets) == hparams.video_num_target_frames return inputs + targets
def gather_neighbors(X, nbr_indices, B, N, M, d):
"""Gathers the neighbor subsets of the atoms in X.
B = batch_size, N = max_num_atoms, M = max_num_neighbors, d = num_features
Parameters
----------
X: tf.Tensor of shape (B, N, d)
Coordinates/features tensor.
atom_indices: tf.Tensor of shape (B, M)
Neighbor list for single atom.
Returns
-------
neighbors: tf.Tensor of shape (B, M, d)
Neighbor coordinates/features tensor for single atom.
"""
example_tensors = tf.unstack(X, axis=0)
example_nbrs = tf.unstack(nbr_indices, axis=0)
all_nbr_coords = []
for example, (example_tensor, example_nbr) in enumerate(
zip(example_tensors, example_nbrs)):
nbr_coords = tf.gather(example_tensor, example_nbr)
all_nbr_coords.append(nbr_coords)
neighbors = tf.stack(all_nbr_coords)
return neighbors
def vec2mtrxBatch(pBatch,opt):
with tf.name_scope("vec2mtrx"):
batchSize = tf.shape(pBatch)[0]
O = tf.zeros([batchSize])
I = tf.ones([batchSize])
if opt.warpType=="translation":
tx,ty = tf.unstack(pBatch,axis=1)
pMtrxBatch = tf.transpose(tf.stack([[I,O,tx],
[O,I,ty],
[O,O,I]]),perm=[2,0,1])
elif opt.warpType=="similarity":
pc,ps,tx,ty = tf.unstack(pBatch,axis=1)
pMtrxBatch = tf.transpose(tf.stack([[I+pc,-ps,tx],
[ps,I+pc,ty],
[O,O,I]]),perm=[2,0,1])
elif opt.warpType=="affine":
p1,p2,p3,p4,p5,p6 = tf.unstack(pBatch,axis=1)
pMtrxBatch = tf.transpose(tf.stack([[I+p1,p2,p3],
[p4,I+p5,p6],
[O,O,I]]),perm=[2,0,1])
elif opt.warpType=="homography":
p1,p2,p3,p4,p5,p6,p7,p8 = tf.unstack(pBatch,axis=1)
pMtrxBatch = tf.transpose(tf.stack([[I+p1,p2,p3],
[p4,I+p5,p6],
[p7,p8,I]]),perm=[2,0,1])
return pMtrxBatch
def _decode_and_random_crop(image_buffer, bbox, image_size):
"""Randomly crops image and then scales to target size."""
with tf.name_scope('distorted_bounding_box_crop',
values=[image_buffer, bbox]):
sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
tf.image.extract_jpeg_shape(image_buffer),
bounding_boxes=bbox,
min_object_covered=0.1,
aspect_ratio_range=[0.75, 1.33],
area_range=[0.08, 1.0],
max_attempts=10,
use_image_if_no_bounding_boxes=True)
bbox_begin, bbox_size, _ = sample_distorted_bounding_box
# Crop the image to the specified bounding box.
offset_y, offset_x, _ = tf.unstack(bbox_begin)
target_height, target_width, _ = tf.unstack(bbox_size)
crop_window = tf.stack([offset_y, offset_x, target_height, target_width])
image = tf.image.decode_and_crop_jpeg(image_buffer, crop_window, channels=3)
image = tf.image.convert_image_dtype(
image, dtype=tf.float32)
image = tf.image.resize_bicubic([image],
[image_size, image_size])[0]
return image
def lstm_word_embedding_from_chars(chars, lengths, embed_dim, scope='lstm-word-embed', reuse=False):
"""
Word embeddings via LSTM encoding of character sequences.
Args:
chars: Tensor of shape [batch_size, word sequence length, char sequence length, num characters].
lengths: Tensor of shape [batch_size, word_sequence length].
embed_dim: Dimension of word embeddings. Integer.
Returns:
Sequence of embedding vectors. Tensor of shape [batch_size, word sequence length, embed_dim].
"""
chars = tf.cast(chars, tf.float32)
# this is super inefficient
chars = tf.unstack(chars, axis=0)
lengths = tf.unstack(lengths, axis=0)
lstm_word_embeddings = []
for i, (char, length) in enumerate(zip(chars, lengths)):
temp_reuse = i != 0 or reuse
embedding = lstm_encoder(char, length, embed_dim, 1.0, scope=scope, reuse=temp_reuse)
lstm_word_embeddings.append(embedding)
lstm_word_embeddings = tf.stack(lstm_word_embeddings, axis=0)
return lstm_word_embeddings
def buildModel(self, fw_lstm_layer, bw_lstm_layer, is_dynamic_rnn):
# Weights and biases for output softmax layer.
out_weights = tf.Variable(
tf.random_normal([self.num_units * 2, self.n_classes]))
out_bias = tf.Variable(tf.random_normal([self.n_classes]))
# input image placeholder
x = tf.placeholder(
"float", [None, self.time_steps, self.n_input], name="INPUT_IMAGE")
if is_dynamic_rnn:
lstm_inputs = tf.transpose(x, [1, 0, 2])
outputs, _ = bidirectional_dynamic_rnn(
fw_lstm_layer,
bw_lstm_layer,
lstm_inputs,
dtype="float32",
time_major=True)
fw_outputs, bw_outputs = outputs
output = tf.concat([fw_outputs, bw_outputs], 2)
output = tf.unstack(output, axis=0)
output = output[-1]
else:
lstm_input = tf.unstack(x, self.time_steps, 1)
outputs, _, _ = tf.nn.static_bidirectional_rnn(
fw_lstm_layer, bw_lstm_layer, lstm_input, dtype="float32")
output = outputs[-1]
# Compute logits by multiplying output of shape [batch_size,num_units*2]
# by the softmax layer's out_weight of shape [num_units*2,n_classes]
# plus out_bias
prediction = tf.matmul(output, out_weights) + out_bias
output_class = tf.nn.softmax(prediction, name="OUTPUT_CLASS")
return x, prediction, output_class
def __call__(self, inputs, seq_len, keep_prob=1.0, is_train=None, concat_layers=True):
outputs = [tf.transpose(inputs, [1, 0, 2])]
for layer in range(self.num_layers):
gru_fw, gru_bw = self.grus[layer]
init_fw, init_bw = self.inits[layer]
mask_fw, mask_bw = self.dropout_mask[layer]
with tf.variable_scope('fw_{}'.format(layer), reuse=tf.AUTO_REUSE):
with tf.variable_scope('cudnn_gru', reuse=tf.AUTO_REUSE):
out_fw, _ = tf.nn.dynamic_rnn(cell=gru_fw, inputs=outputs[-1] * mask_fw, time_major=True,
initial_state=tuple(tf.unstack(init_fw, axis=0)))
with tf.variable_scope('bw_{}'.format(layer), reuse=tf.AUTO_REUSE):
with tf.variable_scope('cudnn_gru', reuse=tf.AUTO_REUSE):
inputs_bw = tf.reverse_sequence(
outputs[-1] * mask_bw, seq_lengths=seq_len, seq_dim=0, batch_dim=1)
out_bw, _ = tf.nn.dynamic_rnn(cell=gru_bw, inputs=inputs_bw, time_major=True,
initial_state=tuple(tf.unstack(init_bw, axis=0)))
out_bw = tf.reverse_sequence(
out_bw, seq_lengths=seq_len, seq_dim=0, batch_dim=1)
outputs.append(tf.concat([out_fw, out_bw], axis=2))
if concat_layers:
res = tf.concat(outputs[1:], axis=2)
else:
res = outputs[-1]
res = tf.transpose(res, [1, 0, 2])
return res
def input_fn(data_dir, subset, num_shards, batch_size):
"""Create input graph for model.
Args:
data_dir: Directory where TFRecords representing the dataset are located.
subset: one of 'train', 'validate' and 'eval'.
num_shards: num of towers participating in data-parallel training.
batch_size: total batch size for training to be divided by the number of
shards.
Returns:
two lists of tensors for features and labels, each of num_shards length.
"""
with tf.device('/cpu:0'):
dataset = mlp_data.MlpDataSet(data_dir, subset)
image_batch, label_batch = dataset.make_batch(batch_size)
if num_shards <= 1:
# No GPU available or only 1 GPU.
return [image_batch], [label_batch]
# Note that passing num=batch_size is safe here, even though
# dataset.batch(batch_size) can, in some cases, return fewer than batch_size
# examples. This is because it does so only when repeating for a limited
# number of epochs, but our dataset repeats forever.
image_batch = tf.unstack(image_batch, num=batch_size, axis=0)
label_batch = tf.unstack(label_batch, num=batch_size, axis=0)
feature_shards = [[] for i in range(num_shards)]
label_shards = [[] for i in range(num_shards)]
for i in xrange(batch_size):
idx = i % num_shards
feature_shards[idx].append(image_batch[i])
label_shards[idx].append(label_batch[i])
feature_shards = [tf.parallel_stack(x) for x in feature_shards]
label_shards = [tf.parallel_stack(x) for x in label_shards]
return feature_shards, label_shards
def hard_negative_mining():
bboxes_per_batch = tf.unstack(bboxes)
classification_loss_per_batch = tf.unstack(classification_loss)
num_positives_per_batch = tf.unstack(tf.reduce_sum(positives, axis=-1))
neg_class_loss_per_batch = tf.unstack(neg_class_loss_all)
neg_class_losses = []
total_negatives = []
for bboxes_per_image, classification_loss_per_image, num_positives_per_image, neg_class_loss_per_image in \
zip(bboxes_per_batch, classification_loss_per_batch, num_positives_per_batch, neg_class_loss_per_batch):
min_negatives_keep = tf.maximum(self.neg_pos_ratio * num_positives_per_image, 3)
num_negatives_keep = tf.minimum(min_negatives_keep,
tf.count_nonzero(neg_class_loss_per_image, dtype=tf.float32))
indices = tf.image.non_max_suppression(bboxes_per_image, classification_loss_per_image,
tf.to_int32(num_negatives_keep), iou_threshold=0.99)
num_negatives = tf.size(indices)
total_negatives.append(num_negatives)
expanded_indexes = tf.expand_dims(indices, axis=1) # shape: (num_negatives, 1)
negatives_keep = tf.scatter_nd(expanded_indexes, updates=tf.ones_like(indices, dtype=tf.int32),
shape=tf.shape(classification_loss_per_image)) # shape: (num_priors,)
negatives_keep = tf.to_float(tf.reshape(negatives_keep, [num_priors])) # shape: (batch_size, num_priors)
neg_class_losses.append(tf.reduce_sum(classification_loss_per_image * negatives_keep, axis=-1)) # shape: (1,)
return tf.stack(neg_class_losses), tf.reduce_sum(tf.stack(total_negatives))
def wasserstein_disagreement_map(
prediction, ground_truth, weight_map=None, M=None):
"""
Function to calculate the pixel-wise Wasserstein distance between the
flattened prediction and the flattened labels (ground_truth) with respect
to the distance matrix on the label space M.
:param prediction: the logits after softmax
:param ground_truth: segmentation ground_truth
:param M: distance matrix on the label space
:return: the pixelwise distance map (wass_dis_map)
"""
if weight_map is not None:
# raise NotImplementedError
tf.logging.warning('Weight map specified but not used.')
assert M is not None, "Distance matrix is required."
# pixel-wise Wassertein distance (W) between flat_pred_proba and flat_labels
# wrt the distance matrix on the label space M
n_classes = prediction.shape[1].value
unstack_labels = tf.unstack(ground_truth, axis=-1)
unstack_labels = tf.cast(unstack_labels, dtype=tf.float64)
unstack_pred = tf.unstack(prediction, axis=-1)
unstack_pred = tf.cast(unstack_pred, dtype=tf.float64)
# print("shape of M", M.shape, "unstacked labels", unstack_labels,
# "unstacked pred" ,unstack_pred)
# W is a weighting sum of all pairwise correlations (pred_ci x labels_cj)
pairwise_correlations = []
for i in range(n_classes):
for j in range(n_classes):
pairwise_correlations.append(
M[i, j] * tf.multiply(unstack_pred[i], unstack_labels[j]))
wass_dis_map = tf.add_n(pairwise_correlations)
return wass_dis_map
def buildModel(self, lstm_layer, is_dynamic_rnn, is_train):
# Weights and biases for output softmax layer.
out_weights = tf.Variable(
tf.random_normal([self.num_units, self.n_classes]))
out_bias = tf.Variable(tf.random_normal([self.n_classes]))
# input image placeholder
x = tf.placeholder(
"float", [None, self.time_steps, self.n_input], name="INPUT_IMAGE")
# For dynamic_rnn, train with dynamic_rnn and inference with static_rnn.
# x is shaped [batch_size,time_steps,num_inputs]
if is_dynamic_rnn:
if is_train:
lstm_input = x
outputs, _ = tf.nn.dynamic_rnn(lstm_layer, lstm_input, dtype="float32")
outputs = tf.unstack(outputs, axis=1)
else:
lstm_input = tf.unstack(x, self.time_steps, 1)
outputs, _ = tf.nn.static_rnn(lstm_layer, lstm_input, dtype="float32")
else:
lstm_input = tf.unstack(x, self.time_steps, 1)
outputs, _ = tf.nn.static_rnn(lstm_layer, lstm_input, dtype="float32")
# Compute logits by multiplying outputs[-1] of shape [batch_size,num_units]
# by the softmax layer's out_weight of shape [num_units,n_classes]
# plus out_bias
prediction = tf.matmul(outputs[-1], out_weights) + out_bias
output_class = tf.nn.softmax(prediction, name="OUTPUT_CLASS")
return x, prediction, output_class
def testCannotInferNumFromUnknownShape(self):
x = tf.placeholder(np.float32)
with self.assertRaisesRegexp(
ValueError, r'Cannot infer num from shape <unknown>'):
tf.unpack(x)
with self.assertRaisesRegexp(
ValueError, r'Cannot infer num from shape <unknown>'):
tf.unstack(x)
def testCannotInferNumFromNoneShape(self):
x = tf.placeholder(np.float32, shape=(None,))
with self.assertRaisesRegexp(ValueError,
r'Cannot infer num from shape \(\?,\)'):
tf.unpack(x)
with self.assertRaisesRegexp(ValueError,
r'Cannot infer num from shape \(\?,\)'):
tf.unstack(x)
def sample(self, b_enc, b_dec):
"""Generate samples for the batch from the NADE.
Args:
b_enc: External encoder bias terms (`b` in [1]), sized
`[batch_size, num_hidden]`.
b_dec: External decoder bias terms (`c` in [1]), sized
`[batch_size, num_dims]`.
Returns:
sample: The generated samples, sized `[batch_size, num_dims]`.
log_prob: The log probabilities of each observation in the batch, sized
`[batch_size, 1]`.
"""
batch_size = tf.shape(b_enc)[0]
a_0 = b_enc
sample_0 = []
log_p_0 = tf.zeros([batch_size, 1])
w_enc_arr = tf.unstack(self.w_enc)
w_dec_arr = tf.unstack(self.w_dec_t)
b_dec_arr = tf.unstack(
tf.reshape(tf.transpose(b_dec), [self.num_dims, batch_size, 1]))
def loop_body(i, a, sample, log_p):
"""Accumulate hidden state, sample, and log probability for index i."""
# Get weights and bias for time step.
w_enc_i = w_enc_arr[i]
w_dec_i = w_dec_arr[i]
b_dec_i = b_dec_arr[i]
cond_p_i = self._cond_prob(a, w_dec_i, b_dec_i)
bernoulli = tf.contrib.distributions.Bernoulli(probs=cond_p_i,
dtype=tf.float32)
v_i = bernoulli.sample()
# Accumulate sampled values.
sample_new = sample + [v_i]
# Get log probability for this value. Log space avoids numerical issues.
log_p_i = v_i * safe_log(cond_p_i) + (1 - v_i) * safe_log(1 - cond_p_i)
# Accumulate log probability.
log_p_new = log_p + log_p_i
# Encode value and add to hidden units.
a_new = a + tf.matmul(v_i, w_enc_i)
return a_new, sample_new, log_p_new
# Build the actual loop.
a, sample, log_p = a_0, sample_0, log_p_0
for i in range(self.num_dims):
a, sample, log_p = loop_body(i, a, sample, log_p)
return tf.transpose(tf.squeeze(tf.stack(sample), [2])), log_p
def construct_model(self, images, actions, rewards):
images = tf.unstack(images, axis=0)
actions = tf.unstack(actions, axis=0)
rewards = tf.unstack(rewards, axis=0)
batch_size = common_layers.shape_list(images[0])[0]
context_frames = self.hparams.video_num_input_frames
# Predicted images and rewards.
gen_rewards, gen_images, latent_means, latent_stds = [], [], [], []
# LSTM states.
lstm_state = [None] * 7
# Create scheduled sampling function
ss_func = self.get_scheduled_sample_func(batch_size)
pred_image = tf.zeros_like(images[0])
pred_reward = tf.zeros_like(rewards[0])
latent = None
for timestep, image, action, reward in zip(
range(len(images)-1), images[:-1], actions[:-1], rewards[:-1]):
# Scheduled Sampling
done_warm_start = timestep > context_frames - 1
groundtruth_items = [image, reward]
generated_items = [pred_image, pred_reward]
input_image, input_reward = self.get_scheduled_sample_inputs(
done_warm_start, groundtruth_items, generated_items, ss_func)
# Latent
# TODO(mbz): should we use input_image iunstead of image?
latent_images = tf.stack([image, images[timestep+1]], axis=0)
latent_mean, latent_std = self.construct_latent_tower(
latent_images, time_axis=0)
latent = common_video.get_gaussian_tensor(latent_mean, latent_std)
latent_means.append(latent_mean)
latent_stds.append(latent_std)
# Prediction
pred_image, lstm_state, _ = self.construct_predictive_tower(
input_image, input_reward, action, lstm_state, latent)
if self.hparams.reward_prediction:
pred_reward = self.reward_prediction(
pred_image, input_reward, action, latent)
pred_reward = common_video.decode_to_shape(
pred_reward, common_layers.shape_list(input_reward), "reward_dec")
else:
pred_reward = input_reward
gen_images.append(pred_image)
gen_rewards.append(pred_reward)
gen_images = tf.stack(gen_images, axis=0)
gen_rewards = tf.stack(gen_rewards, axis=0)
return gen_images, gen_rewards, latent_means, latent_stds
def buildModel(self,
fw_rnn_layer,
bw_rnn_layer,
is_dynamic_rnn,
is_inference,
use_sequence_length=False):
# Weights and biases for output softmax layer.
out_weights = tf.Variable(
tf.random_normal([self.num_units * 2, self.n_classes]))
out_bias = tf.Variable(tf.random_normal([self.n_classes]))
batch_size = self.batch_size
if is_inference:
batch_size = 1
# input image placeholder
x = tf.placeholder(
"float", [batch_size, self.time_steps, self.n_input],
name="INPUT_IMAGE")
sequence_length = None
if use_sequence_length:
sequence_length = [self.time_steps] * batch_size
if is_dynamic_rnn:
rnn_inputs = tf.transpose(x, [1, 0, 2])
outputs, _ = bidirectional_dynamic_rnn(
fw_rnn_layer,
bw_rnn_layer,
rnn_inputs,
sequence_length,
dtype="float32",
time_major=True)
fw_outputs, bw_outputs = outputs
output = tf.concat([fw_outputs, bw_outputs], 2)
output = tf.unstack(output, axis=0)
output = output[-1]
else:
rnn_inputs = tf.unstack(x, self.time_steps, 1)
# Sequence length is not supported for static since we don't have a
# wrapper for it. At training phase, we can still have sequence_length,
# but inference phase, we change it to None.
if is_inference:
sequence_length = None
outputs, _, _ = tf.nn.static_bidirectional_rnn(
fw_rnn_layer,
bw_rnn_layer,
rnn_inputs,
dtype="float32",
sequence_length=sequence_length)
output = outputs[-1]
# Compute logits by multiplying output of shape [batch_size,num_units*2]
# by the softmax layer's out_weight of shape [num_units*2,n_classes]
# plus out_bias
prediction = tf.matmul(output, out_weights) + out_bias
output_class = tf.nn.softmax(prediction, name="OUTPUT_CLASS")
return x, prediction, output_class
def batch_transform_cloud_xyz(cloud, transform):
results = []
cloud_items = tf.unstack(cloud)
if len(transform.shape.as_list()) == 2:
transform_items = tf.unstack(transform)
else:
transform_items = [transform] * len(cloud_items)
for cloud_item, transform_item in zip(cloud_items, transform_items):
results.append(transform_cloud_xyz(cloud_item, transform_item))
return tf.stack(results)
def _add_seq2seq(self):
"""Add the whole sequence-to-sequence model to the graph."""
hps = self._hps
vsize = self._vocab.size() # size of the vocabulary
with tf.variable_scope('seq2seq'):
# Some initializers
self.rand_unif_init = tf.random_uniform_initializer(-hps.rand_unif_init_mag, hps.rand_unif_init_mag, seed=123)
self.trunc_norm_init = tf.truncated_normal_initializer(stddev=hps.trunc_norm_init_std)
# Add embedding matrix (shared by the encoder and decoder inputs)
with tf.variable_scope('embedding'):
if FLAGS.embedding:
embedding = tf.Variable(self.embedding_place)
else:
embedding = tf.get_variable('embedding', [vsize, hps.emb_dim], dtype=tf.float32, initializer=self.trunc_norm_init)
if hps.mode=="train": self._add_emb_vis(embedding) # add to tensorboard
emb_enc_inputs = tf.nn.embedding_lookup(embedding, self._enc_batch) # tensor with shape (batch_size, max_enc_steps, emb_size)
emb_dec_inputs = [tf.nn.embedding_lookup(embedding, x) for x in tf.unstack(self._dec_batch, axis=1)] # list length max_dec_steps containing shape (batch_size, emb_size)
# Add the encoder.
enc_outputs, fw_st, bw_st = self._add_encoder(emb_enc_inputs, self._enc_lens)
self._enc_states = enc_outputs
# Our encoder is bidirectional and our decoder is unidirectional so we need to reduce the final encoder hidden state to the right size to be the initial decoder hidden state
self._dec_in_state = self._reduce_states(fw_st, bw_st)
# Add the decoder.
with tf.variable_scope('decoder'):
(self.decoder_outputs, self._dec_out_state, self.attn_dists, self.p_gens, self.coverage, self.vocab_scores,
self.final_dists, self.samples, self.greedy_search_samples, self.temporal_es,
self.sampling_rewards, self.greedy_rewards) = self._add_decoder(emb_dec_inputs, embedding)
if FLAGS.use_discounted_rewards and hps.rl_training and hps.mode in ['train', 'eval']:
# Get the sampled and greedy sentence from model output
# self.samples: (max_dec_steps, batch_size, k)
self.sampling_discounted_rewards = tf.stack(self.discount_rewards(tf.unstack(self.sampling_rewards))) # list of max_dec_steps * (batch_size, k)
self.greedy_discounted_rewards = tf.stack(self.discount_rewards(tf.unstack(self.greedy_rewards))) # list of max_dec_steps * (batch_size, k)
elif FLAGS.use_intermediate_rewards and hps.rl_training and hps.mode in ['train', 'eval']:
# Get the sampled and greedy sentence from model output
# self.samples: (max_dec_steps, batch_size, k)
self.sampling_discounted_rewards = tf.stack(self.intermediate_rewards(tf.unstack(self.sampling_rewards))) # list of max_dec_steps * (batch_size, k)
self.greedy_discounted_rewards = tf.stack(self.intermediate_rewards(tf.unstack(self.greedy_rewards))) # list of max_dec_steps * (batch_size, k)
elif hps.ac_training and hps.mode in ['train', 'eval']:
# Get the sampled and greedy sentence from model output
self.sampled_sentences = tf.transpose(tf.stack(self.samples), perm=[1,2,0]) # (batch_size, k, <=max_dec_steps) word indices
self.greedy_search_sentences = tf.transpose(tf.stack(self.greedy_search_samples), perm=[1,2,0]) # (batch_size, k, <=max_dec_steps) word indices
if hps.mode == "decode":
# We run decode beam search mode one decoder step at a time
assert len(self.final_dists)==1 # final_dists is a singleton list containing shape (batch_size, extended_vsize)
self.final_dists = self.final_dists[0]
topk_probs, self._topk_ids = tf.nn.top_k(self.final_dists, hps.batch_size*2) # take the k largest probs. note batch_size=beam_size in decode mode
self._topk_log_probs = tf.log(topk_probs)
def regressAnchors(anchors, bbox_regression, axis=-1):
""" Given preliminary bounding boxes, regress them to their final location
The bounding box regressions are outputs of convolutional layers and of
the form (dx,dy,dw,dh), where dw and dh are the natural logarithms of
the desired changes in width and height and where dx and dy are unscaled
x and y displacements; they must be mulitiplied by the width and height of a given
bounding box to arrive at the actual displacements.
"""
# Our shifted anchors come in the shape (numBaseAnchors, feat_h, feat_w, 4)
# We want to separate out the 4 regression variables, {dx,dy,dw,dh}, out into their
# own fith dimension for the output of the regression as well!
# (Actually, we're going to assume that the regressions are ALSO in the form
# (numBaseAnchors, feat_h, feat_w, 4) ! This can be enforced at another stage.
with tf.device("/cpu:0"):
x1, y1, x2, y2 = tf.unstack(anchors, num=4, axis=axis)
dx, dy, dw, dh = tf.unstack(bbox_regression, num=4, axis=axis)
# We must get the anchors into the same width/height x/y format as the
# bbox_regressions
w = x2 - x1 + [1.]
h = y2 - y1 + [1.]
x = w / [2.] + x1
y = h / [2.] + y1
# The dx and dy given by the regression must be scaled by w and h and added
# to the anchors
x_new = dx * w + x
y_new = dy * h + y
# Since logarithms of the values in question are easier to learn (no regression means
# a logarithm of the change being zero), we learn the logarithms of h, w.
w_new = tf.exp(dw) * w
h_new = tf.exp(dh) * h
# Transform back to the original (x1,y1,x2,y2) coordinate system
x1_final = x_new - [.5] * w_new
y1_final = y_new - [.5] * h_new
# x2, y2 represent bottom-left corner of top-right pixels. Hence we need to
# subtract one, or else calling regressAnchors with trivial regressions augments
# x2 and y2 by one every single time it is called.
x2_final = x_new + [.5] * w_new - [1.]
y2_final = y_new + [.5] * h_new - [1.]
# Stack our anchors back up
regressedAnchors = tf.stack([x1_final, y1_final, x2_final, y2_final], axis,
name="regressed_anchors")
# The output shape is the same as the input shape; Output shape is
# regressedAnchors.shape = (numBaseAnchors, feature_h, feature_w, 4)
return regressedAnchors
def __call__(self, init, match, hidden_size, mask):
with tf.variable_scope(self.scope):
BS, ML, MH = tf.unstack(tf.shape(match))
BS, IH = tf.unstack(tf.shape(init))
match_do = tf.nn.dropout(match, keep_prob=self.keep_prob, noise_shape=[BS, 1, MH])
dropout_mask = tf.nn.dropout(tf.ones([BS, IH], dtype=tf.float32), keep_prob=self.keep_prob)
inp, logits1 = attention(match_do, init * dropout_mask, hidden_size, mask)
inp_do = tf.nn.dropout(inp, keep_prob=self.keep_prob)
_, state = self.gru(inp_do, init)
tf.get_variable_scope().reuse_variables()
_, logits2 = attention(match_do, state * dropout_mask, hidden_size, mask)
return logits1, logits2
def flip_image(image, bboxes=None, left_right=True, up_down=False):
"""Flips image on its axis for data augmentation.
Args:
image: Tensor with image of shape (H, W, 3).
bboxes: Optional Tensor with bounding boxes with shape
(total_bboxes, 5).
left_right: Boolean flag to flip the image horizontally
(left to right).
up_down: Boolean flag to flip the image vertically (upside down)
Returns:
image: Flipped image with the same shape.
bboxes: Tensor with the same shape.
"""
image_shape = tf.shape(image)
height = image_shape[0]
width = image_shape[1]
if bboxes is not None:
# bboxes usually come from dataset as ints, but just in case we are
# using flip for preprocessing, where bboxes usually are represented as
# floats, we cast them.
bboxes = tf.to_int32(bboxes)
if left_right:
image = tf.image.flip_left_right(image)
if bboxes is not None:
x_min, y_min, x_max, y_max, label = tf.unstack(bboxes, axis=1)
new_x_min = width - x_max - 1
new_y_min = y_min
new_x_max = new_x_min + (x_max - x_min)
new_y_max = y_max
bboxes = tf.stack(
[new_x_min, new_y_min, new_x_max, new_y_max, label], axis=1
)
if up_down:
image = tf.image.flip_up_down(image)
if bboxes is not None:
x_min, y_min, x_max, y_max, label = tf.unstack(bboxes, axis=1)
new_x_min = x_min
new_y_min = height - y_max - 1
new_x_max = x_max
new_y_max = new_y_min + (y_max - y_min)
bboxes = tf.stack(
[new_x_min, new_y_min, new_x_max, new_y_max, label], axis=1
)
return_dict = {'image': image}
if bboxes is not None:
return_dict['bboxes'] = bboxes
return return_dict
def distorted_bounding_box_crop(image_bytes,
bbox,
min_object_covered=0.1,
aspect_ratio_range=(0.75, 1.33),
area_range=(0.05, 1.0),
max_attempts=100,
scope=None):
"""Generates cropped_image using one of the bboxes randomly distorted.
See `tf.image.sample_distorted_bounding_box` for more documentation.
Args:
image_bytes: `Tensor` of binary image data.
bbox: `Tensor` of bounding boxes arranged `[1, num_boxes, coords]`
where each coordinate is [0, 1) and the coordinates are arranged
as `[ymin, xmin, ymax, xmax]`. If num_boxes is 0 then use the whole
image.
min_object_covered: An optional `float`. Defaults to `0.1`. The cropped
area of the image must contain at least this fraction of any bounding
box supplied.
aspect_ratio_range: An optional list of `float`s. The cropped area of the
image must have an aspect ratio = width / height within this range.
area_range: An optional list of `float`s. The cropped area of the image
must contain a fraction of the supplied image within in this range.
max_attempts: An optional `int`. Number of attempts at generating a cropped
region of the image of the specified constraints. After `max_attempts`
failures, return the entire image.
scope: Optional `str` for name scope.
Returns:
(cropped image `Tensor`, distorted bbox `Tensor`).
"""
with tf.name_scope(scope, 'distorted_bounding_box_crop', [image_bytes, bbox]):
shape = tf.image.extract_jpeg_shape(image_bytes)
sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
shape,
bounding_boxes=bbox,
min_object_covered=min_object_covered,
aspect_ratio_range=aspect_ratio_range,
area_range=area_range,
max_attempts=max_attempts,
use_image_if_no_bounding_boxes=True)
bbox_begin, bbox_size, _ = sample_distorted_bounding_box
# Crop the image to the specified bounding box.
offset_y, offset_x, _ = tf.unstack(bbox_begin)
target_height, target_width, _ = tf.unstack(bbox_size)
crop_window = tf.stack([offset_y, offset_x, target_height, target_width])
image = tf.image.decode_and_crop_jpeg(image_bytes, crop_window)
image = tf.image.convert_image_dtype(
image, dtype=tf.float32)
return image
def buildModel(self, fw_lstm_layer, bw_lstm_layer, is_dynamic_rnn):
"""Build Mnist recognition model.
Args:
fw_lstm_layer: The forward lstm layer either a single lstm cell or a multi
lstm cell.
bw_lstm_layer: The backward lstm layer either a single lstm cell or a
multi lstm cell.
is_dynamic_rnn: Use dynamic_rnn or not.
Returns:
A tuple containing:
- Input tensor of the model.
- Prediction tensor of the model.
- Output class tensor of the model.
"""
# Weights and biases for output softmax layer.
out_weights = tf.Variable(
tf.random_normal([self.num_units * 2, self.n_classes]))
out_bias = tf.Variable(tf.random_normal([self.n_classes]))
# input image placeholder
x = tf.placeholder(
"float", [None, self.time_steps, self.n_input], name="INPUT_IMAGE")
if is_dynamic_rnn:
lstm_inputs = tf.transpose(x, [1, 0, 2])
outputs, _ = bidirectional_dynamic_rnn(
fw_lstm_layer,
bw_lstm_layer,
lstm_inputs,
dtype="float32",
time_major=True)
fw_outputs, bw_outputs = outputs
output = tf.concat([fw_outputs, bw_outputs], 2)
output = tf.unstack(output, axis=0)
output = output[-1]
else:
lstm_input = tf.unstack(x, self.time_steps, 1)
outputs, _, _ = tf.nn.static_bidirectional_rnn(
fw_lstm_layer, bw_lstm_layer, lstm_input, dtype="float32")
output = outputs[-1]
# Compute logits by multiplying output of shape [batch_size,num_units*2]
# by the softmax layer's out_weight of shape [num_units*2,n_classes]
# plus out_bias
prediction = tf.matmul(output, out_weights) + out_bias
output_class = tf.nn.softmax(prediction, name="OUTPUT_CLASS")
return x, prediction, output_class
def image_summary(seqs, name, num=None):
"""Visualizes sequences as TensorBoard summaries.
Args:
seqs: A tensor of shape [n, t, h, w, c].
name: String name of this summary.
num: Integer for the number of examples to visualize. Defaults to
all examples.
"""
seqs = tf.clip_by_value(seqs, 0., 1.)
seqs = tf.unstack(seqs[:num])
joined_seqs = [tf.concat(tf.unstack(seq), 1) for seq in seqs]
joined_seqs = tf.expand_dims(tf.concat(joined_seqs, 0), 0)
tf.contrib.summary.image(name, joined_seqs, max_images=1)
def _summary_vis(m, batch_size, num_steps, arop_full_summary_iters):
arop = []; arop_summary_iters = []; arop_eval_fns = [];
vis_value_ops = []; vis_goal_ops = []; vis_map_ops = [];
vis_occupancy_ops = []; vis_conf_ops = [];
for i, val_op in enumerate(m.value_ops):
vis_value_op = tf.reduce_mean(tf.abs(val_op), axis=3, keep_dims=True)
vis_value_ops.append(vis_value_op)
vis_occupancy_op = tf.reduce_mean(tf.abs(m.occupancys[i]), 3, True)
vis_occupancy_ops.append(vis_occupancy_op)
vis_conf_op = tf.reduce_max(tf.abs(m.confs[i]), axis=3, keep_dims=True)
vis_conf_ops.append(vis_conf_op)
ego_goal_imgs_i_op = m.input_tensors['step']['ego_goal_imgs_{:d}'.format(i)]
vis_goal_op = tf.reduce_max(ego_goal_imgs_i_op, 4, True)
vis_goal_ops.append(vis_goal_op)
vis_map_op = tf.reduce_mean(tf.abs(m.ego_map_ops[i]), 4, True)
vis_map_ops.append(vis_map_op)
vis_goal_ops = tf.concat(vis_goal_ops, 4)
vis_map_ops = tf.concat(vis_map_ops, 4)
vis_value_ops = tf.concat(vis_value_ops, 3)
vis_occupancy_ops = tf.concat(vis_occupancy_ops, 3)
vis_conf_ops = tf.concat(vis_conf_ops, 3)
sh = tf.unstack(tf.shape(vis_value_ops))[1:]
vis_value_ops = tf.reshape(vis_value_ops, shape=[batch_size, -1] + sh)
sh = tf.unstack(tf.shape(vis_conf_ops))[1:]
vis_conf_ops = tf.reshape(vis_conf_ops, shape=[batch_size, -1] + sh)
sh = tf.unstack(tf.shape(vis_occupancy_ops))[1:]
vis_occupancy_ops = tf.reshape(vis_occupancy_ops, shape=[batch_size,-1] + sh)
# Save memory, only return time steps that need to be visualized, factor of
# 32 CPU memory saving.
id = np.int(num_steps/2)
vis_goal_ops = tf.expand_dims(vis_goal_ops[:,id,:,:,:], axis=1)
vis_map_ops = tf.expand_dims(vis_map_ops[:,id,:,:,:], axis=1)
vis_value_ops = tf.expand_dims(vis_value_ops[:,id,:,:,:], axis=1)
vis_conf_ops = tf.expand_dims(vis_conf_ops[:,id,:,:,:], axis=1)
vis_occupancy_ops = tf.expand_dims(vis_occupancy_ops[:,id,:,:,:], axis=1)
arop += [[vis_value_ops, vis_goal_ops, vis_map_ops, vis_occupancy_ops,
vis_conf_ops]]
arop_summary_iters += [arop_full_summary_iters]
arop_eval_fns += [_vis]
return arop, arop_summary_iters, arop_eval_fns
def sequence_loss_fn(self, chars_logits, chars_labels):
"""Loss function for char sequence.
Depending on values of hyper parameters it applies label smoothing and can
also ignore all null chars after the first one.
Args:
chars_logits: logits for predicted characters,
shape=[batch_size, seq_length, num_char_classes];
chars_labels: ground truth ids of characters,
shape=[batch_size, seq_length];
mparams: method hyper parameters.
Returns:
A Tensor with shape [batch_size] - the log-perplexity for each sequence.
"""
mparams = self._mparams['sequence_loss_fn']
with tf.variable_scope('sequence_loss_fn/SLF'):
if mparams.label_smoothing > 0:
smoothed_one_hot_labels = self.label_smoothing_regularization(
chars_labels, mparams.label_smoothing)
labels_list = tf.unstack(smoothed_one_hot_labels, axis=1)
else:
# NOTE: in case of sparse softmax we are not using one-hot
# encoding.
labels_list = tf.unstack(chars_labels, axis=1)
batch_size, seq_length, _ = chars_logits.shape.as_list()
if mparams.ignore_nulls:
weights = tf.ones((batch_size, seq_length), dtype=tf.float32)
else:
# Suppose that reject character is the last in the charset.
reject_char = tf.constant(
self._params.num_char_classes - 1,
shape=(batch_size, seq_length),
dtype=tf.int64)
known_char = tf.not_equal(chars_labels, reject_char)
weights = tf.to_float(known_char)
logits_list = tf.unstack(chars_logits, axis=1)
weights_list = tf.unstack(weights, axis=1)
loss = tf.contrib.legacy_seq2seq.sequence_loss(
logits_list,
labels_list,
weights_list,
softmax_loss_function=get_softmax_loss_fn(mparams.label_smoothing),
average_across_timesteps=mparams.average_across_timesteps)
tf.losses.add_loss(loss)
return loss
def batch_delta(bboxes, anchors):
"""
Args:
bboxes: [num_bboxes, 4]
anchors: [num_bboxes, 4]
Return:
deltas: [num_bboxes, 4]
"""
bbox_x, bbox_y, bbox_w, bbox_h = tf.unstack(bboxes, axis=1)
anchor_x, anchor_y, anchor_w, anchor_h = tf.unstack(anchors, axis=1)
delta_x = (bbox_x - anchor_x) / bbox_w
delta_y = (bbox_y - anchor_y) / bbox_h
delta_w = tf.log(bbox_w / anchor_w)
delta_h = tf.log(bbox_h / anchor_h)
return tf.stack([delta_x, delta_y, delta_w, delta_h], axis=1)
def layer_op(self, input_tensor, is_training):
n_modality = input_tensor.shape.as_list()[-1]
n_chns = input_tensor.shape.as_list()[-2]
rank = input_tensor.shape.ndims
perm = [i for i in range(rank)]
perm[-2], perm[-1] = perm[-1], perm[-2]
output_tensor = input_tensor
for layer in range(self.n_layers):
# modalities => feature channels
output_tensor = tf.transpose(output_tensor, perm=perm)
output_tensor = tf.unstack(output_tensor, axis=-1)
for (idx, tensor) in enumerate(output_tensor):
block_name = 'M_F_{}_{}'.format(layer, idx)
highresblock_op = HighResBlock(
n_output_chns=n_modality,
kernels=(3, 1),
with_res=True,
w_initializer=self.initializers['w'],
w_regularizer=self.regularizers['w'],
acti_func=self.acti_func,
name=block_name)
output_tensor[idx] = highresblock_op(tensor, is_training)
print(highresblock_op)
output_tensor = tf.stack(output_tensor, axis=-1)
# feature channels => modalities
output_tensor = tf.transpose(output_tensor, perm=perm)
output_tensor = tf.unstack(output_tensor, axis=-1)
for (idx, tensor) in enumerate(output_tensor):
block_name = 'F_M_{}_{}'.format(layer, idx)
highresblock_op = HighResBlock(
n_output_chns=n_chns,
kernels=(3, 1),
with_res=True,
w_initializer=self.initializers['w'],
w_regularizer=self.regularizers['w'],
acti_func=self.acti_func,
name=block_name)
output_tensor[idx] = highresblock_op(tensor, is_training)
print(highresblock_op)
output_tensor = tf.stack(output_tensor, axis=-1)
if self.func == 'MAX':
output_tensor = tf.reduce_max(output_tensor, axis=-1)
elif self.func == 'AVERAGE':
output_tensor = tf.reduce_mean(output_tensor, axis=-1)
return output_tensor
def visualize_boxes_in_image(image, boxlist, normalized=False, scope=None):
"""Overlay bounding box list on image.
Currently this visualization plots a 1 pixel thick red bounding box on top
of the image. Note that tf.image.draw_bounding_boxes essentially is
1 indexed.
Args:
image: an image tensor with shape [height, width, 3]
boxlist: a BoxList
normalized: (boolean) specify whether corners are to be interpreted
as absolute coordinates in image space or normalized with respect to the
image size.
scope: name scope.
Returns:
image_and_boxes: an image tensor with shape [height, width, 3]
"""
with tf.name_scope(scope, 'VisualizeBoxesInImage'):
if not normalized:
height, width, _ = tf.unstack(tf.shape(image))
boxlist = scale(boxlist,
1.0 / tf.cast(height, tf.float32),
1.0 / tf.cast(width, tf.float32))
corners = tf.expand_dims(boxlist.get(), 0)
image = tf.expand_dims(image, 0)
return tf.squeeze(tf.image.draw_bounding_boxes(image, corners), [0])
def deep_learning_model():
tf.reset_default_graph() # rest all graphs
batchX_placeholder = tf.placeholder(
tf.float32, [batch_size, truncated_backprop_length, inputs])
batchY_placeholder = tf.placeholder(
tf.float32, [batch_size, truncated_backprop_length])
keep_prob = tf.placeholder(tf.float32)
time_step = tf.placeholder(tf.float32)
col_min_holder = tf.placeholder(
tf.float32, [None, 1, inputs + num_classes
]) # keeps columns min for denormalization
col_max_holder = tf.placeholder(
tf.float32, [None, 1, inputs + num_classes
]) # keeps columns max for denormalization
# Create cells' status
init_state = tf.placeholder(tf.float32,
[num_layers, 2, batch_size, state_size])
state_per_layer_list = tf.unstack(init_state, axis=0)
rnn_tuple_state = tuple([
tf.nn.rnn_cell.LSTMStateTuple(state_per_layer_list[idx][0, :, :],
state_per_layer_list[idx][1, :, :])
for idx in range(num_layers)
])
# Weights and biases
W2 = tf.Variable(np.zeros((state_size, num_classes)), dtype=tf.float32)
b2 = tf.Variable(np.zeros((1, num_classes)), dtype=tf.float32)
# define cells' structure
stacked_rnn = []
for _ in range(num_layers):
stacked_rnn.append(
tf.nn.rnn_cell.LSTMCell(state_size, state_is_tuple=True))
cell = tf.nn.rnn_cell.MultiRNNCell(stacked_rnn, state_is_tuple=True)
states_series, current_state = tf.nn.dynamic_rnn(
cell, batchX_placeholder, initial_state=rnn_tuple_state)
states_series = tf.reshape(states_series, [-1, state_size])
# generate predition
logits = tf.matmul(states_series, W2) + b2
predictions_series = tf.sigmoid(logits)
# Translate acceleration to headway
acel_min = tf.nn.embedding_lookup(tf.transpose(col_min_holder), [2])
acel_max = tf.nn.embedding_lookup(tf.transpose(col_max_holder), [2])
location = tf.transpose(
tf.nn.embedding_lookup(tf.transpose(batchX_placeholder),
[0])) # extract subject vehicle's location
loc_min = tf.nn.embedding_lookup(tf.transpose(col_min_holder), [0])
loc_max = tf.nn.embedding_lookup(tf.transpose(col_max_holder), [0])
location1 = tf.reshape(
tf.add(tf.multiply(location, tf.subtract(loc_max, loc_min)), loc_min),
[batch_size, truncated_backprop_length, -1
]) # denormalize subject vehicle's location
velo = tf.transpose(
tf.nn.embedding_lookup(tf.transpose(batchX_placeholder),
[1])) # extract subject vehicle's velocity
velo_min = tf.nn.embedding_lookup(tf.transpose(col_min_holder), [1])
velo_max = tf.nn.embedding_lookup(tf.transpose(col_max_holder), [1])
velo1 = tf.reshape(
tf.add(tf.multiply(velo, tf.subtract(velo_max, velo_min)), velo_min),
[batch_size, truncated_backprop_length, -1
]) # denormalize subject vehicle's velocity
leadH = tf.transpose(
tf.nn.embedding_lookup(tf.transpose(batchX_placeholder), [5])
) # extract leading vehicle's location at next time step with headway
leadH_min = tf.nn.embedding_lookup(tf.transpose(col_min_holder), [5])
leadH_max = tf.nn.embedding_lookup(tf.transpose(col_max_holder), [5])
leadH1 = tf.reshape(
tf.add(tf.multiply(leadH, tf.subtract(leadH_max, leadH_min)),
leadH_min), [batch_size, truncated_backprop_length, -1]
) # denormalize leading vehicle's location at next time step with headway
target_min = tf.nn.embedding_lookup(tf.transpose(col_min_holder), [6])
target_max = tf.nn.embedding_lookup(tf.transpose(col_max_holder), [6])
predict_accl = tf.add(
tf.multiply(predictions_series, tf.subtract(acel_max, acel_min)),
acel_min)
predict_head = tf.subtract(
leadH1,
tf.add(
location1,
tf.add(
tf.multiply(velo1, time_step),
tf.multiply(predict_accl, tf.multiply(time_step, time_step)))))
predict_head_N = tf.divide(tf.subtract(predict_head, target_min),
tf.subtract(target_max, target_min))
predict_head_N = tf.reshape(
predict_head_N, [truncated_backprop_length * batch_size, num_classes])
# Reshape training output
labels = tf.reshape(batchY_placeholder,
[batch_size * truncated_backprop_length, num_classes])
# Estimate error and run optimizer (gradient descent)
loss = tf.reduce_mean(
tf.divide(tf.abs(tf.subtract(predict_head_N, labels)), (labels)))
predictions_series = tf.unstack(tf.reshape(
predict_head_N, [batch_size, truncated_backprop_length, num_classes]),
axis=1)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(loss)
"""Start session and feed the network with data"""
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
plt.ion()
plt.figure()
plt.show()
ave_loss = 1
stop_training = 0
pervious_validation = 1
while stop_training < stop_training_error_time:
loss_list = []
_current_state = np.zeros((num_layers, 2, batch_size, state_size))
sum_loss = 0
for batch_idx in range(num_batches):
start_idx = batch_idx * truncated_backprop_length
end_idx = start_idx + truncated_backprop_length
batchX = x[:, start_idx:end_idx, :]
batchY = y[:, start_idx:end_idx]
_total_loss, _train_step, _current_state, _predictions_series, ttdoyou = sess.run(
[
loss, train_step, current_state, predictions_series,
predict_head_N
],
feed_dict={
batchX_placeholder: batchX,
batchY_placeholder: batchY,
init_state: _current_state,
keep_prob: keep_rate,
time_step: 0.1,
col_min_holder: col_min,
col_max_holder: col_max
})
loss_list.append(_total_loss)
sum_loss = sum_loss + _total_loss
#Plot training result
if (batch_idx == num_batches - 1):
plot(loss_list, _predictions_series, batchY)
print('This run s %headway error = %',
"%.7f" % (_total_loss * 100))
ave_loss = sum_loss / num_batches
print('Average training %headway error %',
"%.7f" % (ave_loss * 100))
""" Calculate accuracy of the model for validation dataset"""
Ave_loss_validation = 0
loss_listV = []
for batch_idx in range(
int(len(xv[0]) / truncated_backprop_length)):
start_idx = batch_idx * truncated_backprop_length
end_idx = start_idx + truncated_backprop_length
batchXV = xv[:, start_idx:end_idx, :]
batchYV = yv[:, start_idx:end_idx]
__current_state, __predictions_series = sess.run(
[current_state, predictions_series],
feed_dict={
batchX_placeholder: batchXV,
init_state: _current_state,
time_step: 0.1,
col_min_holder: col_min,
col_max_holder: col_max
})
if (batch_idx == int(
len(xv[0]) / truncated_backprop_length) - 2):
plottest(__predictions_series, batchYV)
# Calculate MSE for validation data
ttt = []
for batch_series_idx in range(batch_size):
one_hot_output_series = np.array(
__predictions_series)[:, batch_series_idx, :]
single_output_series = np.array([
(out[0]) for out in one_hot_output_series
])
ttt.extend(single_output_series)
ttt = np.array(ttt)
batchYV = batchYV.reshape(
(truncated_backprop_length * batch_size))
validation_loss = np.mean(
np.abs((batchYV - ttt) / batchYV))
loss_listV.append(validation_loss)
Ave_loss_validation = Ave_loss_validation + validation_loss
validation_loss1 = Ave_loss_validation / (int(
len(xv[0]) / truncated_backprop_length))
print('Validation average %headway error is %',
"%.7f" % (validation_loss * 100))
# Check if the model trained well enough
if (validation_loss1 < 0.001):
# First stop condition
if ((pervious_validation - validation_loss1) <
0.000001):
stop_training = stop_training + 1
pervious_validation = validation_loss1
lost_train.append(loss_list)
lost_validate.append(loss_listV)
"""Test the model"""
Ave_loss_test = 0
loss_listT = []
test_headway = []
for batch_idx in range(int(len(xt[0]) / truncated_backprop_length)):
start_idx = batch_idx * truncated_backprop_length
end_idx = start_idx + truncated_backprop_length
batchXT = xt[:, start_idx:end_idx, :]
batchYT = yt[:, start_idx:end_idx]
_current_state, _predictions_series = sess.run(
[current_state, predictions_series],
feed_dict={
batchX_placeholder: batchXT,
init_state: _current_state,
time_step: 0.1,
col_min_holder: col_min,
col_max_holder: col_max
})
""" Calculate MSE of the model for test dataset"""
ttt = []
for batch_series_idx in range(batch_size):
one_hot_output_series = np.array(
_predictions_series)[:, batch_series_idx, :]
single_output_series = np.array([
(out[0]) for out in one_hot_output_series
])
ttt.extend(single_output_series)
ttt = np.array(ttt)
batchYT1 = batchYT.reshape(
(truncated_backprop_length * batch_size))
test_loss = np.mean(np.abs((batchYT1 - ttt) / batchYT1))
loss_listT.append(test_loss)
Ave_loss_test = Ave_loss_test + test_loss
test_headway.append(ttt)
#Plot test data once in 20 batches
if (batch_idx % 50 == 1):
plottest(_predictions_series, batchYT)
print('This batch s %headway error = ',
"%.7f" % (test_loss * 100))
test_loss = Ave_loss_test / (int(
len(xt[0]) / truncated_backprop_length))
print('Test average %headway error = %', "%.7f" % (test_loss * 100))
lost_test.append(loss_listT)
plt.ioff()
plt.show()
return (test_headway, lost_train, lost_validate, lost_test)
def triangulate(startpoints, endpoints, weights, name="ray_triangulate"):
"""Triangulates 3d points by miminizing the sum of squared distances to rays.
The rays are defined by their start points and endpoints. At least two rays
are required to triangulate any given point. Contrary to the standard
reprojection-error metric, the sum of squared distances to rays can be
minimized in a closed form.
Note:
In the following, A1 to An are optional batch dimensions.
Args:
startpoints: A tensor of ray start points with shape `[A1, ..., An, V, 3]`,
the number of rays V around which the solution points live should be
greater or equal to 2, otherwise triangulation is impossible.
endpoints: A tensor of ray endpoints with shape `[A1, ..., An, V, 3]`, the
number of rays V around which the solution points live should be greater
or equal to 2, otherwise triangulation is impossible. The `endpoints`
tensor should have the same shape as the `startpoints` tensor.
weights: A tensor of ray weights (certainties) with shape `[A1, ..., An,
V]`. Weights should have all positive entries. Weight should have at least
two non-zero entries for each point (at least two rays should have
certainties > 0).
name: A name for this op. The default value of None means "ray_triangulate".
Returns:
A tensor of triangulated points with shape `[A1, ..., An, 3]`.
Raises:
ValueError: If the shape of the arguments is not supported.
"""
with tf.name_scope(name):
startpoints = tf.convert_to_tensor(value=startpoints)
endpoints = tf.convert_to_tensor(value=endpoints)
weights = tf.convert_to_tensor(value=weights)
shape.check_static(
tensor=startpoints,
tensor_name="startpoints",
has_rank_greater_than=1,
has_dim_equals=(-1, 3),
has_dim_greater_than=(-2, 1))
shape.check_static(
tensor=endpoints,
tensor_name="endpoints",
has_rank_greater_than=1,
has_dim_equals=(-1, 3),
has_dim_greater_than=(-2, 1))
shape.compare_batch_dimensions(
tensors=(startpoints, endpoints, weights),
last_axes=(-2, -2, -1),
broadcast_compatible=False)
weights = asserts.assert_all_above(weights, 0.0, open_bound=False)
weights = asserts.assert_at_least_k_non_zero_entries(weights, k=2)
left_hand_side_list = []
right_hand_side_list = []
# TODO(b/130892100): Replace the inefficient for loop and add comments here.
for ray_id in range(weights.shape[-1]):
weights_single_ray = weights[..., ray_id]
startpoints_single_ray = startpoints[..., ray_id, :]
endpoints_singleview = endpoints[..., ray_id, :]
ray = endpoints_singleview - startpoints_single_ray
ray = tf.nn.l2_normalize(ray, axis=-1)
ray_x, ray_y, ray_z = tf.unstack(ray, axis=-1)
zeros = tf.zeros_like(ray_x)
cross_product_matrix = tf.stack(
(zeros, -ray_z, ray_y, ray_z, zeros, -ray_x, -ray_y, ray_x, zeros),
axis=-1)
cross_product_matrix_shape = tf.concat(
(tf.shape(input=cross_product_matrix)[:-1], (3, 3)), axis=-1)
cross_product_matrix = tf.reshape(
cross_product_matrix, shape=cross_product_matrix_shape)
weights_single_ray = tf.expand_dims(weights_single_ray, axis=-1)
weights_single_ray = tf.expand_dims(weights_single_ray, axis=-1)
left_hand_side = weights_single_ray * cross_product_matrix
left_hand_side_list.append(left_hand_side)
dot_product = tf.matmul(cross_product_matrix,
tf.expand_dims(startpoints_single_ray, axis=-1))
right_hand_side = weights_single_ray * dot_product
right_hand_side_list.append(right_hand_side)
left_hand_side_multi_rays = tf.concat(left_hand_side_list, axis=-2)
right_hand_side_multi_rays = tf.concat(right_hand_side_list, axis=-2)
points = tf.linalg.lstsq(left_hand_side_multi_rays,
right_hand_side_multi_rays)
points = tf.squeeze(points, axis=-1)
return points
def unit(hidden_memory_tuple):
hidden_state = tf.unstack(hidden_memory_tuple)
logits = tf.matmul(hidden_state, self.Wo) + self.bo
return logits
def _add_seq2seq(self):
"""Add the whole sequence-to-sequence model to the graph."""
hps = self._hps
vsize = self._vocab.size() # size of the vocabulary
with tf.variable_scope('seq2seq'):
# Some initializers
self.rand_unif_init = tf.random_uniform_initializer(-hps.rand_unif_init_mag, hps.rand_unif_init_mag, seed=123)
self.trunc_norm_init = tf.truncated_normal_initializer(stddev=hps.trunc_norm_init_std)
# Add embedding matrix (shared by the encoder and decoder inputs)
with tf.variable_scope('embedding'):
embedding = tf.get_variable('embedding', [vsize, hps.emb_dim], dtype=tf.float32, initializer=self.trunc_norm_init)
if hps.mode=="train": self._add_emb_vis(embedding) # add to tensorboard
emb_enc_inputs = tf.nn.embedding_lookup(embedding, self._enc_batch) # tensor with shape (batch_size, max_enc_steps, emb_size)
emb_dec_inputs = [tf.nn.embedding_lookup(embedding, x) for x in tf.unstack(self._dec_batch, axis=1)] # list length max_dec_steps containing shape (batch_size, emb_size)
with tf.variable_scope('amr_embedding'):
amr_embedding = tf.get_variable('amr_embedding', [amr_vsize, hps.amr_emb_dim], dtype=tf.float32, initializer=self.trunc_norm_init)
amr_emb_enc_inputs = tf.nn.embedding_lookup(amr_embedding, self._amr_enc_batch) # tensor with shape (batch_size, max_enc_steps, emb_size)
# Add the encoder.
enc_outputs, fw_st, bw_st, amr_states = self._add_encoder(emb_enc_inputs, amr_emb_enc_inputs, self._enc_lens)
self._enc_states = enc_outputs
# Our encoder is bidirectional and our decoder is unidirectional so we need to reduce the final encoder hidden state to the right size to be the initial decoder hidden state
self._dec_in_state = self._reduce_states(fw_st, bw_st)
# Add the decoder.
with tf.variable_scope('decoder'):
decoder_outputs, self._dec_out_state, self.attn_dists, self.p_gens, self.coverage = self._add_decoder(emb_dec_inputs)
# Add the output projection to obtain the vocabulary distribution
with tf.variable_scope('output_projection'):
w = tf.get_variable('w', [hps.hidden_dim, vsize], dtype=tf.float32, initializer=self.trunc_norm_init)
w_t = tf.transpose(w)
v = tf.get_variable('v', [vsize], dtype=tf.float32, initializer=self.trunc_norm_init)
vocab_scores = [] # vocab_scores is the vocabulary distribution before applying softmax. Each entry on the list corresponds to one decoder step
for i,output in enumerate(decoder_outputs):
if i > 0:
tf.get_variable_scope().reuse_variables()
vocab_scores.append(tf.nn.xw_plus_b(output, w, v)) # apply the linear layer
vocab_dists = [tf.nn.softmax(s) for s in vocab_scores] # The vocabulary distributions. List length max_dec_steps of (batch_size, vsize) arrays. The words are in the order they appear in the vocabulary file.
# For pointer-generator model, calc final distribution from copy distribution and vocabulary distribution, then take log
if FLAGS.pointer_gen:
final_dists = self._calc_final_dist(vocab_dists, self.attn_dists)
# Take log of final distribution
log_dists = [tf.log(dist) for dist in final_dists]
else: # just take log of vocab_dists
log_dists = [tf.log(dist) for dist in vocab_dists]
if hps.mode in ['train', 'eval']:
# Calculate the loss
with tf.variable_scope('loss'):
if FLAGS.pointer_gen: # calculate loss from log_dists
# Calculate the loss per step
# This is fiddly; we use tf.gather_nd to pick out the log probabilities of the target words
loss_per_step = [] # will be list length max_dec_steps containing shape (batch_size)
batch_nums = tf.range(0, limit=hps.batch_size) # shape (batch_size)
for dec_step, log_dist in enumerate(log_dists):
targets = self._target_batch[:,dec_step] # The indices of the target words. shape (batch_size)
indices = tf.stack( (batch_nums, targets), axis=1) # shape (batch_size, 2)
losses = tf.gather_nd(-log_dist, indices) # shape (batch_size). loss on this step for each batch
loss_per_step.append(losses)
# Apply padding_mask mask and get loss
self._loss = _mask_and_avg(loss_per_step, self._padding_mask)
else: # baseline model
self._loss = tf.contrib.seq2seq.sequence_loss(tf.stack(vocab_scores, axis=1), self._target_batch, self._padding_mask) # this applies softmax internally
tf.summary.scalar('loss', self._loss)
# Calculate coverage loss from the attention distributions
if hps.coverage:
with tf.variable_scope('coverage_loss'):
self._coverage_loss = _coverage_loss(self.attn_dists, self._padding_mask)
tf.summary.scalar('coverage_loss', self._coverage_loss)
self._total_loss = self._loss + hps.cov_loss_wt * self._coverage_loss
tf.summary.scalar('total_loss', self._total_loss)
if hps.mode == "decode":
# We run decode beam search mode one decoder step at a time
assert len(log_dists)==1 # log_dists is a singleton list containing shape (batch_size, extended_vsize)
log_dists = log_dists[0]
self._topk_log_probs, self._topk_ids = tf.nn.top_k(log_dists, hps.batch_size*2) # note batch_size=beam_size in decode mode
####
x_hat_f = transform_output(x_hat_flat)
if False:
reshape_batch = 100
x_hat_var = tf.Variable(
tf.zeros((seq_size, batch_size, input_size), dtype=tf.float32))
x_hat_f = x_hat_var
for rbi in xrange(1, seq_size, reshape_batch):
fut_border = max(seq_size, rbi + reshape_batch) - seq_size
x_hat_slice = tf.slice(
x_hat, [rbi, 0, 0, 0],
[reshape_batch - fut_border, batch_size, filter_len, input_size])
for ti, xx in enumerate(tf.unstack(x_hat_slice)):
ti = rbi + ti
left_ti = max(0, ti - filter_len)
xx_sliced = tf.slice(xx, (0, 0, 0),
(batch_size, ti - left_ti, input_size))
xx_sliced = tf.transpose(xx_sliced, (1, 0, 2)) / (c.tau * 2)
x_hat_f = tf.scatter_add(x_hat_f, range(left_ti, ti), xx_sliced)
# gg = tf.gradients(x_hat_f, [net._cells[0].F_flat])
# x_hat_f, _ = tf.tuple(
# [tf.identity(x_hat_f), tf.variables_initializer([x_hat_var])]
# )
def __init__(self, config, device, loader, mode):
self.config = config
self.mode = mode
if mode == "Train":
self.is_training = False
self.batch_size = self.config.train_batch_size
self.maxstep_size = self.config.train_step_size
reuse = None
elif mode == "Valid":
self.is_training = False
self.batch_size = self.config.valid_batch_size
self.maxstep_size = self.config.valid_step_size
reuse = True
else:
self.is_training = False
self.batch_size = self.config.test_batch_size
self.maxstep_size = self.config.test_step_size
reuse = True
self.hidden_size = hidden_size = config.hidden_size
self.GNN_step = GNN_step = config.GNN_step
# self.learning_rate = learning_rate = config.learning_rate
opt = config.sgd_opt
beta = config.beta
batch_size = self.batch_size
hidden_stdv = np.sqrt(1. / (hidden_size))
# embedding initial
with tf.device(device), tf.name_scope(mode), tf.variable_scope(
"gnn", reuse=reuse):
feature_embedding_list = []
for idx, each_feature_num in enumerate(config.feature_num):
feature_embedding_list.append(
tf.get_variable(
name='feature_embedding_' + str(idx),
shape=[each_feature_num, hidden_size],
initializer=tf.random_normal_initializer(hidden_stdv)))
w_attention = tf.get_variable(
name='w_attetion',
shape=[
hidden_size * len(config.feature_num),
len(config.feature_num)
],
initializer=tf.random_normal_initializer(stddev=0.2))
w_score2 = tf.get_variable(
name='w_score_2',
shape=[hidden_size, 1],
initializer=tf.random_normal_initializer(hidden_stdv))
# #------------feed-----------------##
self.input_x = input_x = tf.placeholder(
tf.int32, [batch_size, len(config.feature_num)])
self.input_y = input_y = tf.placeholder(tf.int32, [batch_size, 1])
#
# # #--------init graph---------##
# self.graph = graph = init_graph(config, loader)
#
# input_x = input_x.transpose((1, 0))
input_x_unstack = tf.unstack(tf.transpose(input_x, (1, 0)), axis=0)
feature_embedding_input = []
# translate into embedding (lookup)
for idx in range(len(input_x_unstack)):
feature_embedding_input.append(
tf.nn.embedding_lookup(feature_embedding_list[idx],
input_x_unstack[idx]))
self.feature_input = tf.transpose(
tf.stack(feature_embedding_input, axis=0), (1, 0, 2))
with tf.device(device), tf.name_scope(mode), tf.variable_scope(
"gnn", reuse=reuse):
self.w_A = self.weights('a_value', hidden_size, 0)
self.b_A = self.biases('a_value', hidden_size, 0)
graph = self.init_graph(config, self.feature_input)
final_state, test1 = self.GNN(
self.feature_input, batch_size, hidden_size, GNN_step,
len(config.feature_num),
graph) # output: [batch_size, config.feature_num, hiddensize]
atten_pos = self.attention_layer(final_state, w_attention,
batch_size, hidden_size,
len(config.feature_num))
score_pos = tf.matmul(tf.reshape(final_state, [-1, hidden_size]),
w_score2)
score_pos = tf.maximum(0.01 * score_pos, score_pos)
score_pos = tf.reshape(
score_pos, [batch_size, len(config.feature_num)])
s_pos = tf.reshape(tf.reduce_sum(score_pos * atten_pos, axis=1),
[batch_size, 1])
# s_pos = self.dense_layer(final_state, batch_size, hidden_size, len(config.feature_num))
# s_pos = tf.reshape(s_pos, [batch_size, 1])
self.predict = predict = tf.sigmoid(s_pos)
# -------------evaluation--------------
self.auc_result, self.auc_opt = tf.metrics.auc(labels=self.input_y,
predictions=predict)
self.s_pos = s_pos
# -------------cost ---------------
cost_parameter = 0.
num_parameter = 0.
# for variable in tf.trainable_variables():
# print (variable)
# cost_parameter += tf.contrib.layers.l2_regularizer(beta)(variable)
# num_parameter += 1.
# cost_parameter /= num_parameter
# score = tf.nn.sigmoid(s_pos)
# score_mean = tf.reduce_mean(score)
score_mean = tf.losses.log_loss(labels=self.input_y,
predictions=predict)
self.cost = cost = score_mean + cost_parameter
# ---------------optimizer---------------#
self.no_opt = tf.no_op()
self.learning_rate = tf.Variable(config.learning_rate, trainable=False)
if mode == 'Train':
self.auc_opt = tf.no_op()
self.auc_result = tf.no_op()
if opt == 'Adam':
self.optimizer = tf.train.AdamOptimizer(
self.learning_rate).minimize(cost)
if opt == 'Momentum':
self.optimizer = tf.train.MomentumOptimizer(
self.learning_rate, 0.9).minimize(cost)
if opt == 'RMSProp':
self.optimizer = tf.train.RMSPropOptimizer(
self.learning_rate).minimize(cost)
if opt == 'Adadelta':
self.optimizer = tf.train.AdadeltaOptimizer(
self.learning_rate).minimize(cost)
else:
self.optimizer = tf.no_op()
def unstack(*args, **kwargs):
""" See https://www.tensorflow.org/api_docs/python/tf/unstack .
"""
return tensorflow.unstack(*args, **kwargs)
#stack
'''
将两个学校的数据进行拼接,拼接后仍然要对2学校进行区分
school1[classes,students,scores]
school2[classes,students,scores]
拼接后的shape：[schools,classes,students,scores]
'''
a=tf.ones([4,35,8])
b=tf.zeros([4,35,8])
c=tf.stack([a,b],axis=0)#在0之前创作一个新的维度
c.shape()#返回[2,4,35,8]
#所有维度都必须相等
#unstack，按维度数拆分
a=tf.ones([2,4,35,8])
aa,bb=tf.unstack(a,axis=0)#a[0].shape=2,就拆为2个，所以需要2个tensor来保存
aa.shape()#[4,35,8]
bb.shape()#[4,35,8]
cc=tf.unstack(a,axis=1)#拆为4个,保存在cc这个tensor中
#读取第一个就是cc[0],cc[0].shape->[2,35,8].第二个就是cc[1]...
#split 可以指定打散的数量和比例
a=tf.ones([2,4,35,8])
b=tf.split(a,axis=3,num_or_size_splits=2)
#拆分为两个，8/2=4，8拆分为2个4
c=tf.split(a,axis=3,num_or_size_splits=[2,2,4])
#拆分为3个tensor，c[0],c[1],c[2]
#8拆分为2、2、4
def main():
"""Create the model and start the evaluation process."""
# Create queue coordinator.
coord = tf.train.Coordinator()
# Load reader.
with tf.name_scope("create_inputs"):
reader = ImageReader(DATA_DIR, LIST_PATH, DATA_ID_LIST, None, False,
False, False, coord)
image, label, edge_gt = reader.image, reader.label, reader.edge
image_rev = tf.reverse(image, tf.stack([1]))
image_list = reader.image_list
image_batch = tf.stack([image, image_rev])
label_batch = tf.expand_dims(label, dim=0) # Add one batch dimension.
edge_gt_batch = tf.expand_dims(edge_gt, dim=0)
h_orig, w_orig = tf.to_float(tf.shape(image_batch)[1]), tf.to_float(
tf.shape(image_batch)[2])
image_batch050 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 0.50)),
tf.to_int32(tf.multiply(w_orig, 0.50))
]))
image_batch075 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 0.75)),
tf.to_int32(tf.multiply(w_orig, 0.75))
]))
image_batch125 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 1.25)),
tf.to_int32(tf.multiply(w_orig, 1.25))
]))
image_batch150 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 1.50)),
tf.to_int32(tf.multiply(w_orig, 1.50))
]))
image_batch175 = tf.image.resize_images(
image_batch,
tf.stack([
tf.to_int32(tf.multiply(h_orig, 1.75)),
tf.to_int32(tf.multiply(w_orig, 1.75))
]))
# Create network.
with tf.variable_scope('', reuse=False):
net_100 = PGNModel({'data': image_batch},
is_training=False,
n_classes=N_CLASSES)
with tf.variable_scope('', reuse=True):
net_050 = PGNModel({'data': image_batch050},
is_training=False,
n_classes=N_CLASSES)
with tf.variable_scope('', reuse=True):
net_075 = PGNModel({'data': image_batch075},
is_training=False,
n_classes=N_CLASSES)
with tf.variable_scope('', reuse=True):
net_125 = PGNModel({'data': image_batch125},
is_training=False,
n_classes=N_CLASSES)
with tf.variable_scope('', reuse=True):
net_150 = PGNModel({'data': image_batch150},
is_training=False,
n_classes=N_CLASSES)
with tf.variable_scope('', reuse=True):
net_175 = PGNModel({'data': image_batch175},
is_training=False,
n_classes=N_CLASSES)
# parsing net
parsing_out1_050 = net_050.layers['parsing_fc']
parsing_out1_075 = net_075.layers['parsing_fc']
parsing_out1_100 = net_100.layers['parsing_fc']
parsing_out1_125 = net_125.layers['parsing_fc']
parsing_out1_150 = net_150.layers['parsing_fc']
parsing_out1_175 = net_175.layers['parsing_fc']
parsing_out2_050 = net_050.layers['parsing_rf_fc']
parsing_out2_075 = net_075.layers['parsing_rf_fc']
parsing_out2_100 = net_100.layers['parsing_rf_fc']
parsing_out2_125 = net_125.layers['parsing_rf_fc']
parsing_out2_150 = net_150.layers['parsing_rf_fc']
parsing_out2_175 = net_175.layers['parsing_rf_fc']
# edge net
edge_out2_100 = net_100.layers['edge_rf_fc']
edge_out2_125 = net_125.layers['edge_rf_fc']
edge_out2_150 = net_150.layers['edge_rf_fc']
edge_out2_175 = net_175.layers['edge_rf_fc']
# combine resize
parsing_out1 = tf.reduce_mean(tf.stack([
tf.image.resize_images(parsing_out1_050,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out1_075,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out1_100,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out1_125,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out1_150,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out1_175,
tf.shape(image_batch)[1:3, ])
]),
axis=0)
parsing_out2 = tf.reduce_mean(tf.stack([
tf.image.resize_images(parsing_out2_050,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out2_075,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out2_100,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out2_125,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out2_150,
tf.shape(image_batch)[1:3, ]),
tf.image.resize_images(parsing_out2_175,
tf.shape(image_batch)[1:3, ])
]),
axis=0)
edge_out2_100 = tf.image.resize_images(edge_out2_100,
tf.shape(image_batch)[1:3, ])
edge_out2_125 = tf.image.resize_images(edge_out2_125,
tf.shape(image_batch)[1:3, ])
edge_out2_150 = tf.image.resize_images(edge_out2_150,
tf.shape(image_batch)[1:3, ])
edge_out2_175 = tf.image.resize_images(edge_out2_175,
tf.shape(image_batch)[1:3, ])
edge_out2 = tf.reduce_mean(tf.stack(
[edge_out2_100, edge_out2_125, edge_out2_150, edge_out2_175]),
axis=0)
raw_output = tf.reduce_mean(tf.stack([parsing_out1, parsing_out2]), axis=0)
head_output, tail_output = tf.unstack(raw_output, num=2, axis=0)
tail_list = tf.unstack(tail_output, num=20, axis=2)
tail_list_rev = [None] * 20
for xx in xrange(14):
tail_list_rev[xx] = tail_list[xx]
tail_list_rev[14] = tail_list[15]
tail_list_rev[15] = tail_list[14]
tail_list_rev[16] = tail_list[17]
tail_list_rev[17] = tail_list[16]
tail_list_rev[18] = tail_list[19]
tail_list_rev[19] = tail_list[18]
tail_output_rev = tf.stack(tail_list_rev, axis=2)
tail_output_rev = tf.reverse(tail_output_rev, tf.stack([1]))
raw_output_all = tf.reduce_mean(tf.stack([head_output, tail_output_rev]),
axis=0)
raw_output_all = tf.expand_dims(raw_output_all, dim=0)
pred_scores = tf.reduce_max(raw_output_all, axis=3)
raw_output_all = tf.argmax(raw_output_all, axis=3)
pred_all = tf.expand_dims(raw_output_all, dim=3) # Create 4-d tensor.
raw_edge = tf.reduce_mean(tf.stack([edge_out2]), axis=0)
head_output, tail_output = tf.unstack(raw_edge, num=2, axis=0)
tail_output_rev = tf.reverse(tail_output, tf.stack([1]))
raw_edge_all = tf.reduce_mean(tf.stack([head_output, tail_output_rev]),
axis=0)
raw_edge_all = tf.expand_dims(raw_edge_all, dim=0)
pred_edge = tf.sigmoid(raw_edge_all)
res_edge = tf.cast(tf.greater(pred_edge, 0.5), tf.int32)
# prepare ground truth
preds = tf.reshape(pred_all, [
-1,
])
gt = tf.reshape(label_batch, [
-1,
])
weights = tf.cast(
tf.less_equal(gt, N_CLASSES - 1),
tf.int32) # Ignoring all labels greater than or equal to n_classes.
mIoU, update_op_iou = tf.contrib.metrics.streaming_mean_iou(
preds, gt, num_classes=N_CLASSES, weights=weights)
macc, update_op_acc = tf.contrib.metrics.streaming_accuracy(
preds, gt, weights=weights)
# precision and recall
recall, update_op_recall = tf.contrib.metrics.streaming_recall(
res_edge, edge_gt_batch)
precision, update_op_precision = tf.contrib.metrics.streaming_precision(
res_edge, edge_gt_batch)
update_op = tf.group(update_op_iou, update_op_acc, update_op_recall,
update_op_precision)
# Which variables to load.
restore_var = tf.global_variables()
# Set up tf session and initialize variables.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
init = tf.global_variables_initializer()
sess.run(init)
sess.run(tf.local_variables_initializer())
# Load weights.
loader = tf.train.Saver(var_list=restore_var)
if RESTORE_FROM is not None:
if load(loader, sess, RESTORE_FROM):
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
# Start queue threads.
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
# evaluate prosessing
parsing_dir = './output/cihp_parsing_maps'
if not os.path.exists(parsing_dir):
os.makedirs(parsing_dir)
edge_dir = './output/cihp_edge_maps'
if not os.path.exists(edge_dir):
os.makedirs(edge_dir)
# Iterate over training steps.
for step in range(NUM_STEPS):
parsing_, scores, edge_, _ = sess.run(
[pred_all, pred_scores, pred_edge, update_op])
if step % 100 == 0:
print('step {:d}'.format(step))
print(image_list[step])
img_split = image_list[step].split('/')
img_id = img_split[-1][:-4]
msk = decode_labels(parsing_, num_classes=N_CLASSES)
parsing_im = Image.fromarray(msk[0])
parsing_im.save('{}/{}_vis.png'.format(parsing_dir, img_id))
cv2.imwrite('{}/{}.png'.format(parsing_dir, img_id), parsing_[0, :, :,
0])
sio.savemat('{}/{}.mat'.format(parsing_dir, img_id),
{'data': scores[0, :, :]})
cv2.imwrite('{}/{}.png'.format(edge_dir, img_id),
edge_[0, :, :, 0] * 255)
res_mIou = mIoU.eval(session=sess)
res_macc = macc.eval(session=sess)
res_recall = recall.eval(session=sess)
res_precision = precision.eval(session=sess)
f1 = 2 * res_precision * res_recall / (res_precision + res_recall)
print('Mean IoU: {:.4f}, Mean Acc: {:.4f}'.format(res_mIou, res_macc))
print('Recall: {:.4f}, Precision: {:.4f}, F1 score: {:.4f}'.format(
res_recall, res_precision, f1))
coord.request_stop()
coord.join(threads)
def from_rotation_matrix(rotation_matrix, name=None):
"""Converts a rotation matrix representation to a quaternion.
Warning:
This function is not smooth everywhere.
Note:
In the following, A1 to An are optional batch dimensions.
Args:
rotation_matrix: A tensor of shape `[A1, ..., An, 3, 3]`, where the last two
dimensions represent a rotation matrix.
name: A name for this op that defaults to "quaternion_from_rotation_matrix".
Returns:
A tensor of shape `[A1, ..., An, 4]`, where the last dimension represents
a normalized quaternion.
Raises:
ValueError: If the shape of `rotation_matrix` is not supported.
"""
with tf.compat.v1.name_scope(name, "quaternion_from_rotation_matrix",
[rotation_matrix]):
rotation_matrix = tf.convert_to_tensor(value=rotation_matrix)
shape.check_static(tensor=rotation_matrix,
tensor_name="rotation_matrix",
has_rank_greater_than=1,
has_dim_equals=((-1, 3), (-2, 3)))
rotation_matrix = rotation_matrix_3d.assert_rotation_matrix_normalized(
rotation_matrix)
trace = tf.linalg.trace(rotation_matrix)
eps_addition = asserts.select_eps_for_addition(rotation_matrix.dtype)
rows = tf.unstack(rotation_matrix, axis=-2)
entries = [tf.unstack(row, axis=-1) for row in rows]
def tr_positive():
sq = tf.sqrt(trace + 1.0) * 2. # sq = 4 * qw.
qw = 0.25 * sq
qx = safe_ops.safe_unsigned_div(entries[2][1] - entries[1][2], sq)
qy = safe_ops.safe_unsigned_div(entries[0][2] - entries[2][0], sq)
qz = safe_ops.safe_unsigned_div(entries[1][0] - entries[0][1], sq)
return tf.stack((qx, qy, qz, qw), axis=-1)
def cond_1():
sq = tf.sqrt(1.0 + entries[0][0] - entries[1][1] - entries[2][2] +
eps_addition) * 2. # sq = 4 * qx.
qw = safe_ops.safe_unsigned_div(entries[2][1] - entries[1][2], sq)
qx = 0.25 * sq
qy = safe_ops.safe_unsigned_div(entries[0][1] + entries[1][0], sq)
qz = safe_ops.safe_unsigned_div(entries[0][2] + entries[2][0], sq)
return tf.stack((qx, qy, qz, qw), axis=-1)
def cond_2():
sq = tf.sqrt(1.0 + entries[1][1] - entries[0][0] - entries[2][2] +
eps_addition) * 2. # sq = 4 * qy.
qw = safe_ops.safe_unsigned_div(entries[0][2] - entries[2][0], sq)
qx = safe_ops.safe_unsigned_div(entries[0][1] + entries[1][0], sq)
qy = 0.25 * sq
qz = safe_ops.safe_unsigned_div(entries[1][2] + entries[2][1], sq)
return tf.stack((qx, qy, qz, qw), axis=-1)
def cond_3():
sq = tf.sqrt(1.0 + entries[2][2] - entries[0][0] - entries[1][1] +
eps_addition) * 2. # sq = 4 * qz.
qw = safe_ops.safe_unsigned_div(entries[1][0] - entries[0][1], sq)
qx = safe_ops.safe_unsigned_div(entries[0][2] + entries[2][0], sq)
qy = safe_ops.safe_unsigned_div(entries[1][2] + entries[2][1], sq)
qz = 0.25 * sq
return tf.stack((qx, qy, qz, qw), axis=-1)
def cond_idx(cond):
cond = tf.expand_dims(cond, -1)
cond = tf.tile(cond, [1] * (rotation_matrix.shape.ndims - 2) + [4])
return cond
where_2 = tf.compat.v1.where(cond_idx(entries[1][1] > entries[2][2]),
cond_2(), cond_3())
where_1 = tf.compat.v1.where(
cond_idx((entries[0][0] > entries[1][1])
& (entries[0][0] > entries[2][2])), cond_1(), where_2)
quat = tf.compat.v1.where(cond_idx(trace > 0), tr_positive(), where_1)
return quat
def Lstm(self, previous_hidden_memory_tuple, x):
"""
This function takes previous hidden
state and memory tuple with input and
outputs current hidden state.
"""
previous_hidden_state_l1, c_prev_l1, previous_hidden_state_l2, c_prev_l2 = tf.unstack(
previous_hidden_memory_tuple)
# Input Gate
i_l1 = tf.sigmoid(
tf.matmul(x, self.Wi_l1) +
tf.matmul(previous_hidden_state_l1, self.Ui_l1) + self.bi_l1)
# Forget Gate
f_l1 = tf.sigmoid(
tf.matmul(x, self.Wf_l1) +
tf.matmul(previous_hidden_state_l1, self.Uf_l1) + self.bf_l1)
# Output Gate
o_l1 = tf.sigmoid(
tf.matmul(x, self.Wog_l1) +
tf.matmul(previous_hidden_state_l1, self.Uog_l1) + self.bog_l1)
# New Memory Cell
c__l1 = tf.nn.tanh(
tf.matmul(x, self.Wc_l1) +
tf.matmul(previous_hidden_state_l1, self.Uc_l1) + self.bc_l1)
# Final Memory cell
c_l1 = f_l1 * c_prev_l1 + i_l1 * c__l1
# Current Hidden state
current_hidden_state_l1 = o_l1 * tf.nn.tanh(c_l1)
# Input Gate for layer 2
i_l2 = tf.sigmoid(
tf.matmul(current_hidden_state_l1, self.Wi_l2) +
tf.matmul(previous_hidden_state_l2, self.Ui_l2) + self.bi_l2)
# Forget Gate for layer 2
f_l2 = tf.sigmoid(
tf.matmul(current_hidden_state_l1, self.Wf_l2) +
tf.matmul(previous_hidden_state_l2, self.Uf_l2) + self.bf_l2)
# Output Gate for layer 2
o_l2 = tf.sigmoid(
tf.matmul(current_hidden_state_l1, self.Wog_l2) +
tf.matmul(previous_hidden_state_l2, self.Uog_l2) + self.bog_l2)
# New Memory Cell for layer 2
c__l2 = tf.nn.tanh(
tf.matmul(current_hidden_state_l1, self.Wc_l2) +
tf.matmul(previous_hidden_state_l2, self.Uc_l2) + self.bc_l2)
# Final Memory cell for layer 2
c_l2 = f_l2 * c_prev_l2 + i_l2 * c__l2
# Current Hidden state
current_hidden_state_l2 = o_l2 * tf.nn.tanh(c_l2)
return tf.stack(
[current_hidden_state_l1, c_l1, current_hidden_state_l2, c_l2])
def loop_fn(time, previous_output, previous_state, previous_loop_state):
if previous_state is None: # time == 0
assert previous_output is None and previous_state is None
return loop_fn_initial()
else:
return loop_fn_transition(time, previous_output, previous_state,
previous_loop_state)
decoder_outputs_ta, decoder_final_state, _ = tf.nn.raw_rnn(
decoder_cell, loop_fn)
decoder_outputs = decoder_outputs_ta.stack()
print('1')
decoder_max_steps, decoder_batch_size, decoder_dim = tf.unstack(
tf.shape(decoder_outputs))
decoder_outputs_flat = tf.reshape(decoder_outputs, (-1, decoder_dim))
decoder_logits_flat = tf.add(tf.matmul(decoder_outputs_flat, W), b)
decoder_logits = tf.reshape(
decoder_logits_flat, (decoder_max_steps, decoder_batch_size, vocab_size))
decoder_prediction = tf.argmax(decoder_logits, 2)
stepwise_cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=tf.one_hot(decoder_targets, depth=vocab_size, dtype=tf.float32),
logits=decoder_logits,
)
loss = tf.reduce_mean(stepwise_cross_entropy)
train_op = tf.train.AdamOptimizer().minimize(loss)
def create_model(bert_config, is_training, input_ids, input_mask, segment_ids,
use_one_hot_embeddings):
"""Creates a classification model."""
model = modeling.BertModel(
config=bert_config,
is_training=is_training,
input_ids=input_ids,
input_mask=input_mask,
token_type_ids=segment_ids,
use_one_hot_embeddings=use_one_hot_embeddings)
final_hidden = model.get_sequence_output()
final_hidden_shape = modeling.get_shape_list(final_hidden, expected_rank=3)
batch_size = final_hidden_shape[0]
seq_length = final_hidden_shape[1]
hidden_size = final_hidden_shape[2]
use_cnn = True
use_attention = False
use_transformer = False
use_dense = False
if use_cnn:
nb_channels = 3
width = int(math.sqrt(hidden_size / nb_channels))
height = width
output_weights = tf.get_variable(
"cls/squad/output_weights", [2,width*height],
initializer=tf.truncated_normal_initializer(stddev=0.02))
filters = tf.get_variable(
"cls/squad/filters", [3,3,3,1],
initializer=tf.truncated_normal_initializer(stddev=0.02))
final_hidden_matrix = tf.reshape(final_hidden,
[batch_size * seq_length, width,width,3])
final_hidden_matrix = tf.nn.conv2d(input = final_hidden_matrix,filter=filters,strides=[1, 1, 1, 1],padding="SAME")
final_hidden_matrix = tf.reshape(final_hidden_matrix,[batch_size ,seq_length,width*height])
with tf.variable_scope("attention_after_conv2D"):
modeling.attention_layer(final_hidden_matrix,final_hidden_matrix)
final_hidden_matrix = tf.reshape(final_hidden_matrix,[batch_size *seq_length,width*height])
logits = tf.matmul(final_hidden_matrix, output_weights, transpose_b=True)
if use_attention:
with tf.variable_scope("layer_custom_%d" % 1):
modeling.attention_layer(final_hidden,final_hidden)
with tf.variable_scope("layer_custom_%d" % 2):
modeling.attention_layer(final_hidden,final_hidden)
with tf.variable_scope("layer_custom_%d" % 3):
modeling.attention_layer(final_hidden,final_hidden)
if use_transformer:
with tf.variable_scope("custom_transformer"):
final_hidden = modeling.transformer_model(final_hidden,num_hidden_layers=3,num_attention_heads=3)
if use_attention or use_transformer:
output_weights = tf.get_variable(
"cls/squad/output_weights", [2, hidden_size],
initializer=tf.truncated_normal_initializer(stddev=0.02))
final_hidden_matrix = tf.reshape(final_hidden,
[batch_size * seq_length, hidden_size])
logits = tf.matmul(final_hidden_matrix, output_weights, transpose_b=True)
output_bias = tf.get_variable(
"cls/squad/output_bias", [2], initializer=tf.zeros_initializer())
logits = tf.nn.bias_add(logits, output_bias)
logits = tf.reshape(logits, [batch_size, seq_length, 2])
logits = tf.transpose(logits, [2, 0, 1])
unstacked_logits = tf.unstack(logits, axis=0)
(start_logits, end_logits) = (unstacked_logits[0], unstacked_logits[1])
return (start_logits, end_logits)
def main(args):
network = importlib.import_module(args.model_def)
subdir = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')
log_dir = os.path.join(os.path.expanduser(args.logs_base_dir), subdir)
if not os.path.isdir(
log_dir): # Create the log directory if it doesn't exist
os.makedirs(log_dir)
model_dir = os.path.join(os.path.expanduser(args.models_base_dir), subdir)
if not os.path.isdir(
model_dir): # Create the model directory if it doesn't exist
os.makedirs(model_dir)
# Store some git revision info in a text file in the log directory
# src_path,_ = os.path.split(os.path.realpath(__file__))
# facenet.store_revision_info(src_path, log_dir, ' '.join(sys.argv))
np.random.seed(seed=args.seed)
random.seed(args.seed)
train_set = facenet.get_dataset(args.data_dir)
if args.filter_filename:
train_set = filter_dataset(train_set, args.filter_filename,
args.filter_percentile,
args.filter_min_nrof_images_per_class)
nrof_classes = len(train_set)
print('Model directory: %s' % model_dir)
print('Log directory: %s' % log_dir)
pretrained_model = None
if args.pretrained_model:
pretrained_model = os.path.expanduser(args.pretrained_model)
print('Pre-trained model: %s' % pretrained_model)
if args.lfw_dir:
print('LFW directory: %s' % args.lfw_dir)
# Read the file containing the pairs used for testing
pairs = lfw.read_pairs(os.path.expanduser(args.lfw_pairs))
# Get the paths for the corresponding images
lfw_paths, actual_issame = lfw.get_paths(
os.path.expanduser(args.lfw_dir), pairs, args.lfw_file_ext)
with tf.Graph().as_default():
tf.set_random_seed(args.seed)
global_step = tf.Variable(0, trainable=False)
# Get a list of image paths and their labels
image_list, label_list = facenet.get_image_paths_and_labels(train_set)
assert len(image_list) > 0, 'The dataset should not be empty'
# Create a queue that produces indices into the image_list and label_list
labels = ops.convert_to_tensor(label_list, dtype=tf.int32)
range_size = array_ops.shape(labels)[0]
index_queue = tf.train.range_input_producer(range_size,
num_epochs=None,
shuffle=True,
seed=None,
capacity=32)
index_dequeue_op = index_queue.dequeue_many(
args.batch_size * args.epoch_size, 'index_dequeue')
learning_rate_placeholder = tf.placeholder(tf.float32,
name='learning_rate')
batch_size_placeholder = tf.placeholder(tf.int32, name='batch_size')
phase_train_placeholder = tf.placeholder(tf.bool, name='phase_train')
image_paths_placeholder = tf.placeholder(tf.string,
shape=(None, 1),
name='image_paths')
labels_placeholder = tf.placeholder(tf.int64,
shape=(None, 1),
name='labels')
input_queue = data_flow_ops.FIFOQueue(capacity=100000,
dtypes=[tf.string, tf.int64],
shapes=[(1, ), (1, )],
shared_name=None,
name=None)
enqueue_op = input_queue.enqueue_many(
[image_paths_placeholder, labels_placeholder], name='enqueue_op')
nrof_preprocess_threads = 4
images_and_labels = []
for _ in range(nrof_preprocess_threads):
filenames, label = input_queue.dequeue()
images = []
for filename in tf.unstack(filenames):
file_contents = tf.read_file(filename)
image = tf.image.decode_png(file_contents)
if args.random_rotate:
image = tf.py_func(facenet.random_rotate_image, [image],
tf.uint8)
if args.random_crop:
image = tf.random_crop(
image, [args.image_size, args.image_size, 3])
else:
image = tf.image.resize_image_with_crop_or_pad(
image, args.image_size, args.image_size)
if args.random_flip:
image = tf.image.random_flip_left_right(image)
#pylint: disable=no-member
image.set_shape((args.image_size, args.image_size, 3))
images.append(tf.image.per_image_standardization(image))
images_and_labels.append([images, label])
image_batch, label_batch = tf.train.batch_join(
images_and_labels,
batch_size=batch_size_placeholder,
shapes=[(args.image_size, args.image_size, 3), ()],
enqueue_many=True,
capacity=4 * nrof_preprocess_threads * args.batch_size,
allow_smaller_final_batch=True)
image_batch = tf.identity(image_batch, 'image_batch')
image_batch = tf.identity(image_batch, 'input')
label_batch = tf.identity(label_batch, 'label_batch')
print('Total number of classes: %d' % nrof_classes)
print('Total number of examples: %d' % len(image_list))
print('Building training graph')
batch_norm_params = {
# Decay for the moving averages
'decay': 0.995,
# epsilon to prevent 0s in variance
'epsilon': 0.001,
# force in-place updates of mean and variance estimates
'updates_collections': None,
# Moving averages ends up in the trainable variables collection
'variables_collections': [tf.GraphKeys.TRAINABLE_VARIABLES],
# Only update statistics during training mode
'is_training': phase_train_placeholder
}
# Build the inference graph
# print('Build the inference graph')
prelogits, _ = network.inference(image_batch,
args.keep_probability,
phase_train=phase_train_placeholder,
weight_decay=args.weight_decay)
bottleneck = slim.fully_connected(
prelogits,
args.embedding_size,
activation_fn=None,
weights_initializer=tf.truncated_normal_initializer(stddev=0.1),
weights_regularizer=slim.l2_regularizer(args.weight_decay),
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params,
scope='Bottleneck',
reuse=False)
logits = slim.fully_connected(
bottleneck,
len(train_set),
activation_fn=None,
weights_initializer=tf.truncated_normal_initializer(stddev=0.1),
weights_regularizer=slim.l2_regularizer(args.weight_decay),
scope='Logits',
reuse=False)
embeddings = tf.nn.l2_normalize(bottleneck,
1,
1e-10,
name='embeddings')
# Add center loss
# print('Add center loss')
if args.center_loss_factor > 0.0:
prelogits_center_loss, _ = facenet.center_loss(
prelogits, label_batch, args.center_loss_alfa, nrof_classes)
tf.add_to_collection(
tf.GraphKeys.REGULARIZATION_LOSSES,
prelogits_center_loss * args.center_loss_factor)
learning_rate = tf.train.exponential_decay(
learning_rate_placeholder,
global_step,
args.learning_rate_decay_epochs * args.epoch_size,
args.learning_rate_decay_factor,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
# Calculate the average cross entropy loss across the batch
# print('Calculate the average cross entropy loss across the batch')
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=label_batch,
logits=logits,
name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy,
name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
# Calculate the total losses
# print('Calculate the total losses')
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
total_loss = tf.add_n([cross_entropy_mean] + regularization_losses,
name='total_loss')
# Build a Graph that trains the model with one batch of examples and updates the model parameters
# print('Build a Graph that trains the model with one batch of examples and updates the model parameters')
train_op = facenet.train(total_loss, global_step, args.optimizer,
learning_rate, args.moving_average_decay,
tf.global_variables(), args.log_histograms)
# Create a saver
saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=1)
# Build the summary operation based on the TF collection of Summaries.
# print('Build the summary operation based on the TF collection of Summaries.')
summary_op = tf.summary.merge_all()
# Start running operations on the Graph.
# print('Start running operations on the Graph.')
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=args.gpu_memory_fraction)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
log_device_placement=False))
#sess = tf.Session(config=tf.ConfigProto(inter_op_parallelism_threads=1, intra_op_parallelism_threads=1))
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
coord = tf.train.Coordinator()
tf.train.start_queue_runners(coord=coord, sess=sess)
best_accuracy = 0.0
with sess.as_default():
if pretrained_model:
print('Restoring pretrained model: %s' % pretrained_model)
saver.restore(sess, pretrained_model)
# Training and validation loop
print('Running training')
epoch = 0
while epoch < args.max_nrof_epochs:
step = sess.run(global_step, feed_dict=None)
epoch = step // args.epoch_size
# Train for one epoch
train(args, sess, epoch, image_list, label_list,
index_dequeue_op, enqueue_op, image_paths_placeholder,
labels_placeholder, learning_rate_placeholder,
phase_train_placeholder, batch_size_placeholder,
global_step, total_loss, train_op, summary_op,
summary_writer, regularization_losses,
args.learning_rate_schedule_file)
# Evaluate on LFW
if args.lfw_dir:
accuracy = evaluate(
sess, enqueue_op, image_paths_placeholder,
labels_placeholder, phase_train_placeholder,
batch_size_placeholder, embeddings, label_batch,
lfw_paths, actual_issame, args.lfw_batch_size,
args.lfw_nrof_folds, log_dir, step, summary_writer)
if accuracy > best_accuracy:
best_accuracy = accuracy
# Save best variables and the metagraph if it doesn't exist already
save_variables_and_metagraph(sess, saver,
summary_writer, model_dir,
subdir, step)
else:
save_variables_and_metagraph(sess, saver, summary_writer,
model_dir, subdir, step)
sess.close()
return model_dir
def BiRNN(batch_x, seq_length, dropout, reuse=False, batch_size=None, n_steps=-1, previous_state=None, tflite=False):
r'''
That done, we will define the learned variables, the weights and biases,
within the method ``BiRNN()`` which also constructs the neural network.
The variables named ``hn``, where ``n`` is an integer, hold the learned weight variables.
The variables named ``bn``, where ``n`` is an integer, hold the learned bias variables.
In particular, the first variable ``h1`` holds the learned weight matrix that
converts an input vector of dimension ``n_input + 2*n_input*n_context``
to a vector of dimension ``n_hidden_1``.
Similarly, the second variable ``h2`` holds the weight matrix converting
an input vector of dimension ``n_hidden_1`` to one of dimension ``n_hidden_2``.
The variables ``h3``, ``h5``, and ``h6`` are similar.
Likewise, the biases, ``b1``, ``b2``..., hold the biases for the various layers.
'''
layers = {}
# Input shape: [batch_size, n_steps, n_input + 2*n_input*n_context]
if not batch_size:
batch_size = tf.shape(batch_x)[0]
# Reshaping `batch_x` to a tensor with shape `[n_steps*batch_size, n_input + 2*n_input*n_context]`.
# This is done to prepare the batch for input into the first layer which expects a tensor of rank `2`.
# Permute n_steps and batch_size
batch_x = tf.transpose(batch_x, [1, 0, 2, 3])
# Reshape to prepare input for first layer
batch_x = tf.reshape(batch_x, [-1, Config.n_input + 2*Config.n_input*Config.n_context]) # (n_steps*batch_size, n_input + 2*n_input*n_context)
layers['input_reshaped'] = batch_x
# The next three blocks will pass `batch_x` through three hidden layers with
# clipped RELU activation and dropout.
# 1st layer
b1 = variable_on_worker_level('b1', [Config.n_hidden_1], tf.zeros_initializer())
h1 = variable_on_worker_level('h1', [Config.n_input + 2*Config.n_input*Config.n_context, Config.n_hidden_1], tf.contrib.layers.xavier_initializer())
layer_1 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(batch_x, h1), b1)), FLAGS.relu_clip)
layer_1 = tf.nn.dropout(layer_1, rate=dropout[0])
layers['layer_1'] = layer_1
# 2nd layer
b2 = variable_on_worker_level('b2', [Config.n_hidden_2], tf.zeros_initializer())
h2 = variable_on_worker_level('h2', [Config.n_hidden_1, Config.n_hidden_2], tf.contrib.layers.xavier_initializer())
layer_2 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_1, h2), b2)), FLAGS.relu_clip)
layer_2 = tf.nn.dropout(layer_2, rate=dropout[1])
layers['layer_2'] = layer_2
# 3rd layer
b3 = variable_on_worker_level('b3', [Config.n_hidden_3], tf.zeros_initializer())
h3 = variable_on_worker_level('h3', [Config.n_hidden_2, Config.n_hidden_3], tf.contrib.layers.xavier_initializer())
layer_3 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_2, h3), b3)), FLAGS.relu_clip)
layer_3 = tf.nn.dropout(layer_3, rate=dropout[2])
layers['layer_3'] = layer_3
# Now we create the forward and backward LSTM units.
# Both of which have inputs of length `n_cell_dim` and bias `1.0` for the forget gate of the LSTM.
# Forward direction cell:
if not tflite:
fw_cell = tf.contrib.rnn.LSTMBlockFusedCell(Config.n_cell_dim, reuse=reuse)
layers['fw_cell'] = fw_cell
else:
fw_cell = tf.nn.rnn_cell.LSTMCell(Config.n_cell_dim, reuse=reuse)
# `layer_3` is now reshaped into `[n_steps, batch_size, 2*n_cell_dim]`,
# as the LSTM RNN expects its input to be of shape `[max_time, batch_size, input_size]`.
layer_3 = tf.reshape(layer_3, [n_steps, batch_size, Config.n_hidden_3])
if tflite:
# Generated StridedSlice, not supported by NNAPI
#n_layer_3 = []
#for l in range(layer_3.shape[0]):
# n_layer_3.append(layer_3[l])
#layer_3 = n_layer_3
# Unstack/Unpack is not supported by NNAPI
layer_3 = tf.unstack(layer_3, n_steps)
# We parametrize the RNN implementation as the training and inference graph
# need to do different things here.
if not tflite:
output, output_state = fw_cell(inputs=layer_3, dtype=tf.float32, sequence_length=seq_length, initial_state=previous_state)
else:
output, output_state = tf.nn.static_rnn(fw_cell, layer_3, previous_state, tf.float32)
output = tf.concat(output, 0)
# Reshape output from a tensor of shape [n_steps, batch_size, n_cell_dim]
# to a tensor of shape [n_steps*batch_size, n_cell_dim]
output = tf.reshape(output, [-1, Config.n_cell_dim])
layers['rnn_output'] = output
layers['rnn_output_state'] = output_state
# Now we feed `output` to the fifth hidden layer with clipped RELU activation and dropout
b5 = variable_on_worker_level('b5', [Config.n_hidden_5], tf.zeros_initializer())
h5 = variable_on_worker_level('h5', [Config.n_cell_dim, Config.n_hidden_5], tf.contrib.layers.xavier_initializer())
layer_5 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(output, h5), b5)), FLAGS.relu_clip)
layer_5 = tf.nn.dropout(layer_5, rate=dropout[5])
layers['layer_5'] = layer_5
# Now we apply the weight matrix `h6` and bias `b6` to the output of `layer_5`
# creating `n_classes` dimensional vectors, the logits.
b6 = variable_on_worker_level('b6', [Config.n_hidden_6], tf.zeros_initializer())
h6 = variable_on_worker_level('h6', [Config.n_hidden_5, Config.n_hidden_6], tf.contrib.layers.xavier_initializer())
layer_6 = tf.add(tf.matmul(layer_5, h6), b6)
layers['layer_6'] = layer_6
# Finally we reshape layer_6 from a tensor of shape [n_steps*batch_size, n_hidden_6]
# to the slightly more useful shape [n_steps, batch_size, n_hidden_6].
# Note, that this differs from the input in that it is time-major.
layer_6 = tf.reshape(layer_6, [n_steps, batch_size, Config.n_hidden_6], name="raw_logits")
layers['raw_logits'] = layer_6
# Output shape: [n_steps, batch_size, n_hidden_6]
return layer_6, layers
def encode_all_anchors(self, labels, bboxes, all_anchors, all_num_anchors_depth, all_num_anchors_spatial,
debug=False):
# y, x, h, w are all in range [0, 1] relative to the original image size
# shape info:
# y_on_image, x_on_image: layers_shapes[0] * layers_shapes[1]
# h_on_image, w_on_image: num_anchors
assert (len(all_num_anchors_depth) == len(all_num_anchors_spatial)) and (
len(all_num_anchors_depth) == len(all_anchors)), 'inconsist num layers for anchors.'
with tf.name_scope('encode_all_anchors'):
num_layers = len(all_num_anchors_depth)
list_anchors_ymin = []
list_anchors_xmin = []
list_anchors_ymax = []
list_anchors_xmax = []
tiled_allowed_borders = []
for ind, anchor in enumerate(all_anchors):
anchors_ymin_, anchors_xmin_, anchors_ymax_, anchors_xmax_ = self.center2point(anchor[0], anchor[1],
anchor[2], anchor[3])
list_anchors_ymin.append(tf.reshape(anchors_ymin_, [-1]))
list_anchors_xmin.append(tf.reshape(anchors_xmin_, [-1]))
list_anchors_ymax.append(tf.reshape(anchors_ymax_, [-1]))
list_anchors_xmax.append(tf.reshape(anchors_xmax_, [-1]))
tiled_allowed_borders.extend(
[self._allowed_borders[ind]] * all_num_anchors_depth[ind] * all_num_anchors_spatial[ind])
anchors_ymin = tf.concat(list_anchors_ymin, 0, name='concat_ymin')
anchors_xmin = tf.concat(list_anchors_xmin, 0, name='concat_xmin')
anchors_ymax = tf.concat(list_anchors_ymax, 0, name='concat_ymax')
anchors_xmax = tf.concat(list_anchors_xmax, 0, name='concat_xmax')
if self._clip:
anchors_ymin = tf.clip_by_value(anchors_ymin, 0., 1.)
anchors_xmin = tf.clip_by_value(anchors_xmin, 0., 1.)
anchors_ymax = tf.clip_by_value(anchors_ymax, 0., 1.)
anchors_xmax = tf.clip_by_value(anchors_xmax, 0., 1.)
anchor_allowed_borders = tf.stack(tiled_allowed_borders, 0, name='concat_allowed_borders')
inside_mask = tf.logical_and(tf.logical_and(anchors_ymin > -anchor_allowed_borders * 1.,
anchors_xmin > -anchor_allowed_borders * 1.),
tf.logical_and(anchors_ymax < (1. + anchor_allowed_borders * 1.),
anchors_xmax < (1. + anchor_allowed_borders * 1.)))
anchors_point = tf.stack([anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax], axis=-1)
# save_anchors_op = tf.py_func(save_anchors,
# [bboxes,
# labels,
# anchors_point],
# tf.int64, stateful=True)
# with tf.control_dependencies([save_anchors_op]):
overlap_matrix = iou_matrix(bboxes, anchors_point) * tf.cast(tf.expand_dims(inside_mask, 0), tf.float32)
matched_gt, gt_scores = do_dual_max_match(overlap_matrix, self._ignore_threshold, self._positive_threshold)
# get all positive matching positions
matched_gt_mask = matched_gt > -1
matched_indices = tf.clip_by_value(matched_gt, 0, tf.int64.max)
# the labels here maybe chaos at those non-positive positions
gt_labels = tf.gather(labels, matched_indices)
# filter the invalid labels
gt_labels = gt_labels * tf.cast(matched_gt_mask, tf.int64)
# set those ignored positions to -1
gt_labels = gt_labels + (-1 * tf.cast(matched_gt < -1, tf.int64))
gt_ymin, gt_xmin, gt_ymax, gt_xmax = tf.unstack(tf.gather(bboxes, matched_indices), 4, axis=-1)
# transform to center / size.
gt_cy, gt_cx, gt_h, gt_w = self.point2center(gt_ymin, gt_xmin, gt_ymax, gt_xmax)
anchor_cy, anchor_cx, anchor_h, anchor_w = self.point2center(anchors_ymin, anchors_xmin, anchors_ymax,
anchors_xmax)
# encode features.
# the prior_scaling (in fact is 5 and 10) is use for balance the regression loss of center and with(or height)
gt_cy = (gt_cy - anchor_cy) / anchor_h / self._prior_scaling[0]
gt_cx = (gt_cx - anchor_cx) / anchor_w / self._prior_scaling[1]
gt_h = tf.log(gt_h / anchor_h) / self._prior_scaling[2]
gt_w = tf.log(gt_w / anchor_w) / self._prior_scaling[3]
# now gt_localizations is our regression object, but also maybe chaos at those non-positive positions
if debug:
gt_targets = tf.stack([anchors_ymin, anchors_xmin, anchors_ymax, anchors_xmax], axis=-1)
else:
gt_targets = tf.stack([gt_cy, gt_cx, gt_h, gt_w], axis=-1)
# set all targets of non-positive positions to 0
gt_targets = tf.expand_dims(tf.cast(matched_gt_mask, tf.float32), -1) * gt_targets
self._all_anchors = (anchor_cy, anchor_cx, anchor_h, anchor_w)
return gt_targets, gt_labels, gt_scores
def main(args):
network = importlib.import_module(args.model_def)
subdir = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')
log_dir = os.path.join(os.path.expanduser(args.logs_base_dir), subdir)
if not os.path.isdir(log_dir): # Create the log directory if it doesn't exist
os.makedirs(log_dir)
model_dir = os.path.join(os.path.expanduser(args.models_base_dir), subdir)
if not os.path.isdir(model_dir): # Create the model directory if it doesn't exist
os.makedirs(model_dir)
# Write arguments to a text file
facenet.write_arguments_to_file(args, os.path.join(log_dir, 'arguments.txt'))
# Store some git revision info in a text file in the log directory
src_path,_ = os.path.split(os.path.realpath(__file__))
facenet.store_revision_info(src_path, log_dir, ' '.join(sys.argv))
np.random.seed(seed=args.seed)
train_set = facenet.get_dataset(args.data_dir)
print('Model directory: %s' % model_dir)
print('Log directory: %s' % log_dir)
if args.pretrained_model:
print('Pre-trained model: %s' % os.path.expanduser(args.pretrained_model))
if args.lfw_dir:
print('LFW directory: %s' % args.lfw_dir)
# Read the file containing the pairs used for testing
pairs = lfw.read_pairs(os.path.expanduser(args.lfw_pairs))
# Get the paths for the corresponding images
lfw_paths, actual_issame = lfw.get_paths(os.path.expanduser(args.lfw_dir), pairs)
with tf.Graph().as_default():
tf.set_random_seed(args.seed)
global_step = tf.Variable(0, trainable=False)
# Placeholder for the learning rate
learning_rate_placeholder = tf.placeholder(tf.float32, name='learning_rate')
batch_size_placeholder = tf.placeholder(tf.int32, name='batch_size')
phase_train_placeholder = tf.placeholder(tf.bool, name='phase_train')
image_paths_placeholder = tf.placeholder(tf.string, shape=(None,3), name='image_paths')
labels_placeholder = tf.placeholder(tf.int64, shape=(None,3), name='labels')
input_queue = data_flow_ops.FIFOQueue(capacity=100000,
dtypes=[tf.string, tf.int64],
shapes=[(3,), (3,)],
shared_name=None, name=None)
enqueue_op = input_queue.enqueue_many([image_paths_placeholder, labels_placeholder])
nrof_preprocess_threads = 4
images_and_labels = []
for _ in range(nrof_preprocess_threads):
filenames, label = input_queue.dequeue()
images = []
for filename in tf.unstack(filenames):
file_contents = tf.read_file(filename)
image = tf.image.decode_image(file_contents, channels=3)
if args.random_crop:
image = tf.random_crop(image, [args.image_size, args.image_size, 3])
else:
image = tf.image.resize_image_with_crop_or_pad(image, args.image_size, args.image_size)
if args.random_flip:
image = tf.image.random_flip_left_right(image)
#pylint: disable=no-member
image.set_shape((args.image_size, args.image_size, 3))
images.append(tf.image.per_image_standardization(image))
images_and_labels.append([images, label])
image_batch, labels_batch = tf.train.batch_join(
images_and_labels, batch_size=batch_size_placeholder,
shapes=[(args.image_size, args.image_size, 3), ()], enqueue_many=True,
capacity=4 * nrof_preprocess_threads * args.batch_size,
allow_smaller_final_batch=True)
image_batch = tf.identity(image_batch, 'image_batch')
image_batch = tf.identity(image_batch, 'input')
labels_batch = tf.identity(labels_batch, 'label_batch')
# Build the inference graph
prelogits, _ = network.inference(image_batch, args.keep_probability,
phase_train=phase_train_placeholder, bottleneck_layer_size=args.embedding_size,
weight_decay=args.weight_decay)
embeddings = tf.nn.l2_normalize(prelogits, 1, 1e-10, name='embeddings')
# Split embeddings into anchor, positive and negative and calculate triplet loss
anchor, positive, negative = tf.unstack(tf.reshape(embeddings, [-1,3,args.embedding_size]), 3, 1)
if args.gor_alfa:
triplet_loss = facenet.triplet_loss_gor(anchor,
positive, negative, args.alpha,
args.gor_alfa, args.embedding_size)
elif args.kld_alfa:
triplet_loss = facenet.triplet_loss_kld(anchor,
positive, negative, args.alpha,
args.kld_alfa, args.embedding_size)
elif args.soft_alfa:
triplet_loss = facenet.triplet_loss_soft(anchor, positive,
negative, args.soft_alfa)
else:
triplet_loss = facenet.triplet_loss(anchor, positive, negative, args.alpha)
learning_rate = tf.train.exponential_decay(learning_rate_placeholder, global_step,
args.learning_rate_decay_epochs*args.epoch_size, args.learning_rate_decay_factor, staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
# Calculate the total losses
regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
total_loss = tf.add_n([triplet_loss] + regularization_losses, name='total_loss')
# Build a Graph that trains the model with one batch of examples and updates the model parameters
train_op = facenet.train(total_loss, global_step, args.optimizer,
learning_rate, args.moving_average_decay, tf.global_variables())
# Create a saver
saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=3)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
# Start running operations on the Graph.
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
# Initialize variables
sess.run(tf.global_variables_initializer(), feed_dict={phase_train_placeholder:True})
sess.run(tf.local_variables_initializer(), feed_dict={phase_train_placeholder:True})
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
coord = tf.train.Coordinator()
tf.train.start_queue_runners(coord=coord, sess=sess)
with sess.as_default():
if args.pretrained_model:
print('Restoring pretrained model: %s' % args.pretrained_model)
saver.restore(sess, os.path.expanduser(args.pretrained_model))
# Training and validation loop
epoch = 0
while epoch < args.max_nrof_epochs:
step = sess.run(global_step, feed_dict=None)
epoch = step // args.epoch_size
# Train for one epoch
train(args, sess, train_set, epoch, image_paths_placeholder, labels_placeholder, labels_batch,
batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, input_queue, global_step,
embeddings, total_loss, train_op, summary_op, summary_writer, args.learning_rate_schedule_file,
args.embedding_size, anchor, positive, negative, triplet_loss)
# Save variables and the metagraph if it doesn't exist already
save_variables_and_metagraph(sess, saver, summary_writer, model_dir, subdir, step)
# Evaluate on LFW
if args.lfw_dir:
evaluate(sess, lfw_paths, embeddings, labels_batch, image_paths_placeholder, labels_placeholder,
batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, actual_issame, args.batch_size,
args.lfw_nrof_folds, log_dir, step, summary_writer, args.embedding_size)
return model_dir
def build_model(self):
net = tl.layers.InputLayer(self.input, name='input_layer')
with tf.variable_scope('fc1'):
net = tl.layers.TimeDistributedLayer(
net,
layer_class=tl.layers.DenseLayer,
args={
'n_units': 64,
'act': tf.nn.elu,
'W_init': tf.contrib.layers.variance_scaling_initializer(),
'W_init_args': {
'regularizer':
tf.contrib.layers.l2_regularizer(
self.args.weight_decay)
},
'name': 'fc1_'
},
name='time_dense_fc1')
# net = tl.layers.DropoutLayer(net, keep=self.args.keep_prob, name='fc1_drop')
with tf.variable_scope('highway'):
num_highway = 3
for idx in xrange(num_highway):
highway_args = {
'n_units': 64,
'act': tf.nn.elu,
'W_init': tf.contrib.layers.variance_scaling_initializer(),
'b_init': tf.constant_initializer(value=0.0),
'W_init_args': {
'regularizer':
tf.contrib.layers.l2_regularizer(
self.args.weight_decay)
},
'name': 'highway_%03d_' % idx
}
net = tl.layers.TimeDistributedLayer(
net,
layer_class=utility.Highway,
args=highway_args,
name='time_dense_highway_%d' % idx)
# if idx % 8 == 0:
# net = tl.layers.DropoutLayer(net, keep=self.args.keep_prob, name='highway_drop_%d' % idx)
# net = tl.layers.DropoutLayer(net, keep=self.args.keep_prob, name='highway_drop')
with tf.variable_scope('fc2'):
net = tl.layers.TimeDistributedLayer(
net,
layer_class=tl.layers.DenseLayer,
args={
'n_units': 64,
'act': tf.nn.elu,
'W_init': tf.contrib.layers.variance_scaling_initializer(),
'W_init_args': {
'regularizer':
tf.contrib.layers.l2_regularizer(
self.args.weight_decay)
},
'name': 'highway_to_fc_'
},
name='time_dense_highway_to_fc')
net = tl.layers.DropoutLayer(net,
keep=self.args.keep_prob,
name='hw_to_fc_drop')
with tf.variable_scope('RNN'):
if self.args.rnn_cell == 'lstm':
rnn_cell_fn = tf.contrib.rnn.BasicLSTMCell
elif self.args.rnn_cell == 'gru':
rnn_cell_fn = tf.contrib.rnn.GRUCell
else:
raise ValueError(
'Unimplemented RNN Cell, should be \'lstm\' or \'gru\'')
self.rnn_keep_prob = tf.placeholder(tf.float32)
rnn_layer_name = 'DRNN_layer'
net = tl.layers.DynamicRNNLayer(layer=net,
cell_fn=rnn_cell_fn,
n_hidden=128,
dropout=(1.0, self.rnn_keep_prob),
n_layer=self.args.num_cells,
return_last=True,
name=rnn_layer_name)
rnn_weights_params = [
var for var in net.all_params
if rnn_layer_name in var.name and 'weights' in var.name
]
self.add_regularization_loss(rnn_weights_params)
# net = tl.layers.DenseLayer(net,
# n_units=50,
# act=tf.nn.elu,
# W_init=tf.contrib.layers.variance_scaling_initializer(),
# name='fc1')
# with tf.variable_scope('Highway'):
# num_highway = 15
# for idx in xrange(num_highway - 1):
# net = utility.Highway(net,
# n_units=64,
# act=tf.nn.elu,
# W_init=tf.contrib.layers.variance_scaling_initializer(),
# b_init=tf.constant_initializer(value=0.0),
# W_init_args={'regularizer': tf.contrib.layers.l2_regularizer(self.args.weight_decay)},
# reuse=False,
# name='highway_%d'%idx)
net = tl.layers.DenseLayer(
net,
n_units=64,
act=tf.nn.elu,
W_init=tf.contrib.layers.variance_scaling_initializer(),
W_init_args={
'regularizer':
tf.contrib.layers.l2_regularizer(self.args.weight_decay)
},
name='fc_3')
mus_num = self.args.num_mixtures * self.args.gaussian_dim
sigmas_num = self.args.num_mixtures * self.args.gaussian_dim
weights_num = self.args.num_mixtures
num_output = mus_num + sigmas_num + weights_num
self.net = tl.layers.DenseLayer(
net,
n_units=num_output,
act=tf.identity,
W_init=tf.contrib.layers.variance_scaling_initializer(),
W_init_args={
'regularizer':
tf.contrib.layers.l2_regularizer(self.args.weight_decay)
},
name='nn_output')
output = self.net.outputs
with tf.variable_scope('MDN'):
mus = output[:, :mus_num]
sigmas = tf.exp(output[:, mus_num:mus_num + sigmas_num])
self.weight_logits = output[:, mus_num + sigmas_num:]
self.mus = tf.reshape(
mus, (-1, self.args.num_mixtures, self.args.gaussian_dim))
self.sigmas = tf.reshape(
sigmas, (-1, self.args.num_mixtures, self.args.gaussian_dim))
self.weights = tf.nn.softmax(self.weight_logits)
cat = Categorical(logits=self.weight_logits)
components = [
MultivariateNormalDiag(mu=mu, diag_stdev=sigma)
for mu, sigma in zip(
tf.unstack(tf.transpose(self.mus, (
1, 0,
2))), tf.unstack(tf.transpose(self.sigmas, (1, 0, 2))))
]
self.y_mix = Mixture(cat=cat, components=components)
self.loss = self.get_loss()
def roll_attention_decoder(decoder_inputs,
initial_state,
encoder_states,
enc_padding_mask,
cell,
initial_state_attention=False,
pointer_gen=True):
"""
Args:
decoder_inputs: A list of 2D Tensors [batch_size x input_size].
initial_state: 2D Tensor [batch_size x cell.state_size].
encoder_states: 3D Tensor [batch_size x attn_length x attn_size].
enc_padding_mask: 2D Tensor [batch_size x attn_length] containing 1s and 0s; indicates which of the encoder locations are padding (0) or a real token (1).
cell: rnn_cell.RNNCell defining the cell function and size.
initial_state_attention:
Note that this attention decoder passes each decoder input through a linear layer with the previous step's context vector to get a modified version of the input. If initial_state_attention is False, on the first decoder step the "previous context vector" is just a zero vector. If initial_state_attention is True, we use initial_state to (re)calculate the previous step's context vector. We set this to False for train/eval mode (because we call attention_decoder once for all decoder steps) and True for decode mode (because we call attention_decoder once for each decoder step).
pointer_gen: boolean. If True, calculate the generation probability p_gen for each decoder step.
Returns:
outputs: A list of the same length as decoder_inputs of 2D Tensors of
shape [batch_size x cell.output_size]. The output vectors.
state: The final state of the decoder. A tensor shape [batch_size x cell.state_size].
attn_dists: A list containing tensors of shape (batch_size,attn_length).
The attention distributions for each decoder step.
p_gens: List of scalars. The values of p_gen for each decoder step. Empty list if pointer_gen=False.
"""
with variable_scope.variable_scope("attention_decoder") as scope:
batch_size = encoder_states.get_shape(
)[0].value # if this line fails, it's because the batch size isn't defined
attn_size = encoder_states.get_shape(
)[2].value # if this line fails, it's because the attention length isn't defined
# Reshape encoder_states (need to insert a dim)
encoder_states = tf.expand_dims(
encoder_states,
axis=2) # now is shape (batch_size, attn_len, 1, attn_size)
# To calculate attention, we calculate
# v^T tanh(W_h h_i + W_s s_t + b_attn)
# where h_i is an encoder state, and s_t a decoder state.
# attn_vec_size is the length of the vectors v, b_attn, (W_h h_i) and (W_s s_t).
# We set it to be equal to the size of the encoder states.
attention_vec_size = attn_size
# Get the weight matrix W_h and apply it to each encoder state to get (W_h h_i), the encoder features
W_h = variable_scope.get_variable(
"W_h", [1, 1, attn_size, attention_vec_size])
encoder_features = nn_ops.conv2d(
encoder_states, W_h, [1, 1, 1, 1],
"SAME") # shape (batch_size,attn_length,1,attention_vec_size)
# Get the weight vectors v and w_c (w_c is for coverage)
v = variable_scope.get_variable("v", [attention_vec_size])
def attention(decoder_state):
"""Calculate the context vector and attention distribution from the decoder state.
Args:
decoder_state: state of the decoder
Returns:
context_vector: weighted sum of encoder_states
attn_dist: attention distribution
"""
with variable_scope.variable_scope("Attention"):
# Pass the decoder state through a linear layer (this is W_s s_t + b_attn in the paper)
decoder_features = linear(
decoder_state, attention_vec_size,
True) # shape (batch_size, attention_vec_size)
decoder_features = tf.expand_dims(
tf.expand_dims(decoder_features, 1),
1) # reshape to (batch_size, 1, 1, attention_vec_size)
def masked_attention(e):
"""Take softmax of e then apply enc_padding_mask and re-normalize"""
attn_dist = nn_ops.softmax(
e) # take softmax. shape (batch_size, attn_length)
attn_dist *= enc_padding_mask # apply mask
masked_sums = tf.reduce_sum(attn_dist,
axis=1) # shape (batch_size)
return attn_dist / tf.reshape(masked_sums,
[-1, 1]) # re-normalize
# Calculate v^T tanh(W_h h_i + W_s s_t + b_attn)
e = math_ops.reduce_sum(
v * math_ops.tanh(encoder_features + decoder_features),
[2, 3]) # calculate e
# Calculate attention distribution
attn_dist = masked_attention(e)
# Calculate the context vector from attn_dist and encoder_states
context_vector = math_ops.reduce_sum(
array_ops.reshape(attn_dist, [batch_size, -1, 1, 1]) *
encoder_states, [1, 2]) # shape (batch_size, attn_size).
context_vector = array_ops.reshape(context_vector,
[-1, attn_size])
return context_vector, attn_dist
def run(ind):
i = 0
inp = tf.gather(decoder_inputs, ind)
ith_state = tf.unstack(tf.gather(initial_state, ind))
if initial_state_attention: # true in decode mode
# Re-calculate the context vector from the previous step so that we can pass it through a linear layer with this step's input to get a modified version of the input
context_vector, _ = attention(ith_state)
context_vector = array_ops.zeros([batch_size, attn_size])
context_vector.set_shape([
None, attn_size
]) # Ensure the second shape of attention vectors is set.
# tf.logging.info("Adding attention_decoder timestep %i of %i", i, len(decoder_inputs))
if i > 0:
variable_scope.get_variable_scope().reuse_variables()
# Merge input and previous attentions into one vector x of the same size as inp
input_size = inp.get_shape().with_rank(2)[1]
if input_size.value is None:
raise ValueError("Could not infer input size from input: %s" %
inp.name)
x = linear([inp] + [context_vector], input_size, True)
# Run the decoder RNN cell. cell_output = decoder state
cell_output, state = cell(x, ith_state)
# state_list.append(state)
# Run the attention mechanism.
#if initial_state_attention: # always true in decode mode
with variable_scope.variable_scope(
variable_scope.get_variable_scope(), reuse=True
): # you need this because you've already run the initial attention(...) call
context_vector, attn_dist = attention(state)
# Calculate p_gen
if pointer_gen:
with tf.variable_scope('calculate_pgen'):
p_gen = linear([context_vector, state.c, state.h, x], 1,
True) # a scalar
p_gen = tf.sigmoid(p_gen)
# p_gens.append(p_gen)
# Concatenate the cell_output (= decoder state) and the context vector, and pass them through a linear layer
# This is V[s_t, h*_t] + b in the paper
with variable_scope.variable_scope("AttnOutputProjection"):
output = linear([cell_output] + [context_vector],
cell.output_size, True)
# outputs.append(output)
return tf.stack(state), attn_dist, p_gen, output
states, attn_dists, p_gens, outputs = tf.map_fn(
run,
tf.range(len(decoder_inputs)),
dtype=(tf.float32, tf.float32, tf.float32, tf.float32))
# for i, inp in enumerate(decoder_inputs):
# state, attn_dist, p_gen, output = run(i, inp)
# state_list.append(state)
# attn_dists.append(attn_dist)
# p_gens.append(p_gen)
# outputs.append(output)
# state = state_list
return tf.unstack(outputs), [
tf.contrib.rnn.LSTMStateTuple(elem[0], elem[1])
for elem in tf.unstack(states)
], tf.unstack(attn_dists), tf.unstack(p_gens)
def conv_latent_tower(images,
time_axis,
latent_channels=1,
min_logvar=-5,
is_training=False,
random_latent=False,
tiny_mode=False,
small_mode=False):
"""Builds convolutional latent tower for stochastic model.
At training time this tower generates a latent distribution (mean and std)
conditioned on the entire video. This latent variable will be fed to the
main tower as an extra variable to be used for future frames prediction.
At inference time, the tower is disabled and only returns latents sampled
from N(0,1).
If the multi_latent flag is on, a different latent for every timestep would
be generated.
Args:
images: tensor of ground truth image sequences
time_axis: the time axis in images tensor
latent_channels: number of latent channels
min_logvar: minimum value for log_var
is_training: whether or not it is training mode
random_latent: whether or not generate random latents
tiny_mode: whether or not it is tiny_mode. tiny_mode sets the number
of conv channels to 1 at each layer. useful for testing the
integration tests.
small_mode: whether or not it is small_mode. small mode is the same model
with less conv and lstm layers and also lower number of channels.
suitable for videos with less complexity and testing.
Returns:
latent_mean: predicted latent mean
latent_logvar: predicted latent log variance
"""
conv_size = tinyify([32, 64, 64], tiny_mode, small_mode)
with tf.variable_scope("latent", reuse=tf.AUTO_REUSE):
images = tf.to_float(images)
images = tf.unstack(images, axis=time_axis)
images = tf.concat(images, axis=3)
x = images
x = common_layers.make_even_size(x)
x = tfl.conv2d(x,
conv_size[0], [3, 3],
strides=(2, 2),
padding="SAME",
activation=tf.nn.relu,
name="latent_conv1")
x = contrib.layers().layer_norm(x)
if not small_mode:
x = tfl.conv2d(x,
conv_size[1], [3, 3],
strides=(2, 2),
padding="SAME",
activation=tf.nn.relu,
name="latent_conv2")
x = contrib.layers().layer_norm(x)
x = tfl.conv2d(x,
conv_size[2], [3, 3],
strides=(1, 1),
padding="SAME",
activation=tf.nn.relu,
name="latent_conv3")
x = contrib.layers().layer_norm(x)
nc = latent_channels
mean = tfl.conv2d(x,
nc, [3, 3],
strides=(2, 2),
padding="SAME",
activation=None,
name="latent_mean")
logv = tfl.conv2d(x,
nc, [3, 3],
strides=(2, 2),
padding="SAME",
activation=tf.nn.relu,
name="latent_std")
logvar = logv + min_logvar
# No latent tower at inference time, just standard gaussian.
if not is_training:
return tf.zeros_like(mean), tf.zeros_like(logvar)
# No latent in the first phase
ret_mean, ret_logvar = tf.cond(
random_latent, lambda:
(tf.zeros_like(mean), tf.zeros_like(logvar)), lambda:
(mean, logvar))
return ret_mean, ret_logvar
def _add_encoder(self, encoder_inputs, amr_encoder_inputs, seq_len):
"""Add a single-layer bidirectional LSTM encoder to the graph.
Args:
encoder_inputs: A tensor of shape [batch_size, <=max_enc_steps, emb_size].
seq_len: Lengths of encoder_inputs (before padding). A tensor of shape [batch_size].
Returns:
encoder_outputs:
A tensor of shape [batch_size, <=max_enc_steps, 2*hidden_dim]. It's 2*hidden_dim because it's the concatenation of the forwards and backwards states.
fw_state, bw_state:
Each are LSTMStateTuples of shape ([batch_size,hidden_dim],[batch_size,hidden_dim])
"""
with tf.variable_scope('encoder'):
cell_fw = tf.contrib.rnn.LSTMCell(self._hps.hidden_dim, initializer=self.rand_unif_init, state_is_tuple=True)
cell_bw = tf.contrib.rnn.LSTMCell(self._hps.hidden_dim, initializer=self.rand_unif_init, state_is_tuple=True)
(encoder_outputs, (fw_st, bw_st)) = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, encoder_inputs, dtype=tf.float32, sequence_length=seq_len, swap_memory=True)
encoder_outputs = tf.concat(axis=2, values=encoder_outputs) # concatenate the forwards and backwards states
# amr encoder
with tf.variable_scope('amr_encoder'):
(self.num_documents,
self.num_sentences,
self.num_words) = tf.unstack(tf.shape(self.amr_inputs))
word_level_inputs = tf.reshape(self.amr_encoder_inputs, [
self.num_documents * self.num_sentences,
self.num_words,
hps.amr_emb_dim
])
word_level_lengths = tf.reshape(
self.amr_word_lengths, [self.num_documents * self.num_sentences])
with tf.variable_scope('word') as scope:
word_cell = tf.contrib.rnn.LSTMCell(self._hps.amr_hidden_dim, initializer=self.rand_unif_init, state_is_tuple=True)
word_cell = tf.contrib.rnn.DropoutWrapper(word_cell, output_keep_prob=0.5)
word_encoder_output, _ = tf.nn.dynamic_rnn(
word_cell,
word_level_inputs, word_level_lengths,
scope=scope)
with tf.variable_scope('attention') as scope:
word_level_output = task_specific_attention(
word_encoder_output,
self.word_output_size, # fix
scope=scope, seq_len=self.num_words)
# sentence_level
sentence_inputs = tf.reshape(
word_level_output, [self.num_documents, self.num_sentences, self._hps.amr_hidden_dim])
self.sentence_inputs = sentence_inputs
with tf.variable_scope('sentence') as scope:
sentence_cell = tf.contrib.rnn.LSTMCell(self._hps.amr_hidden_dim, initializer=self.rand_unif_init, state_is_tuple=True)
sentence_cell = tf.contrib.rnn.DropoutWrapper(sentence_cell, output_keep_prob=0.5)
sentence_encoder_output, sentence_encoder_state = tf.nn.dynamic_rnn(
sentence_cell, sentence_inputs, self.amr_sentence_lengths, scope=scope)
with tf.variable_scope('attention') as scope:
sentence_level_output = task_specific_attention(
sentence_encoder_output, self.sentence_output_size, scope=scope, seq_len=self.num_sentences) # fix
amr_outputs = tf.reshape(
sentence_level_output, [self.num_documents, self.sentence_output_size])
return encoder_outputs, fw_st, bw_st, amr_outputs
def compute_range_image_cartesian(range_image_polar,
extrinsic,
pixel_pose=None,
frame_pose=None,
dtype=tf.float64,
scope=None):
"""Computes range image cartesian coordinates from polar ones.
Args:
range_image_polar: [B, H, W, 3] float tensor. Lidar range image in polar
coordinate in sensor frame.
extrinsic: [B, 4, 4] float tensor. Lidar extrinsic.
pixel_pose: [B, H, W, 4, 4] float tensor. If not None, it sets pose for each
range image pixel.
frame_pose: [B, 4, 4] float tensor. This must be set when pixel_pose is set.
It decides the vehicle frame at which the cartesian points are computed.
dtype: float type to use internally. This is needed as extrinsic and
inclination sometimes have higher resolution than range_image.
scope: the name scope.
Returns:
range_image_cartesian: [B, H, W, 3] cartesian coordinates.
"""
range_image_polar_dtype = range_image_polar.dtype
range_image_polar = tf.cast(range_image_polar, dtype)
extrinsic = tf.cast(extrinsic, dtype)
if pixel_pose is not None:
pixel_pose = tf.cast(pixel_pose, dtype)
if frame_pose is not None:
frame_pose = tf.cast(frame_pose, dtype)
with tf.name_scope(scope, 'ComputeRangeImageCartesian',
[range_image_polar, extrinsic, pixel_pose, frame_pose]):
azimuth, inclination, range_image_range = tf.unstack(
range_image_polar, axis=-1)
cos_azimuth = tf.cos(azimuth)
sin_azimuth = tf.sin(azimuth)
cos_incl = tf.cos(inclination)
sin_incl = tf.sin(inclination)
# [B, H, W].
x = cos_azimuth * cos_incl * range_image_range
y = sin_azimuth * cos_incl * range_image_range
z = sin_incl * range_image_range
# [B, H, W, 3]
range_image_points = tf.stack([x, y, z], -1)
# [B, 3, 3]
rotation = extrinsic[..., 0:3, 0:3]
# translation [B, 1, 3]
translation = tf.expand_dims(tf.expand_dims(extrinsic[..., 0:3, 3], 1), 1)
# To vehicle frame.
# [B, H, W, 3]
range_image_points = tf.einsum('bkr,bijr->bijk', rotation,
range_image_points) + translation
if pixel_pose is not None:
# To global frame.
# [B, H, W, 3, 3]
pixel_pose_rotation = pixel_pose[..., 0:3, 0:3]
# [B, H, W, 3]
pixel_pose_translation = pixel_pose[..., 0:3, 3]
# [B, H, W, 3]
range_image_points = tf.einsum(
'bhwij,bhwj->bhwi', pixel_pose_rotation,
range_image_points) + pixel_pose_translation
if frame_pose is None:
raise ValueError('frame_pose must be set when pixel_pose is set.')
# To vehicle frame corresponding to the given frame_pose
# [B, 4, 4]
world_to_vehicle = tf.matrix_inverse(frame_pose)
world_to_vehicle_rotation = world_to_vehicle[:, 0:3, 0:3]
world_to_vehicle_translation = world_to_vehicle[:, 0:3, 3]
# [B, H, W, 3]
range_image_points = tf.einsum(
'bij,bhwj->bhwi', world_to_vehicle_rotation,
range_image_points) + world_to_vehicle_translation[:, tf.newaxis,
tf.newaxis, :]
range_image_points = tf.cast(
range_image_points, dtype=range_image_polar_dtype)
return range_image_points
def make_net(labels, input_shape, loss_name='loss_total', mode='train'):
names = dict()
raw = tf.placeholder(tf.float32, shape=input_shape)
names['raw'] = raw.name
raw_bc = tf.reshape(raw, (
1,
1,
) + input_shape)
last_fmap, fov, anisotropy = unet.unet(
raw_bc, [12, 12 * 6, 12 * 6 * 6, 12 * 6 * 6 * 6],
[48, 12 * 6, 12 * 6 * 6, 12 * 6 * 6 * 6],
[[3, 3, 3], [3, 3, 3], [3, 3, 3]],
[[(3, 3, 3),
(3, 3, 3)], [(3, 3, 3),
(3, 3, 3)], [(3, 3, 3),
(3, 3, 3)], [(3, 3, 3), (3, 3, 3)]],
[[(3, 3, 3),
(3, 3, 3)], [(3, 3, 3),
(3, 3, 3)], [(3, 3, 3),
(3, 3, 3)], [(3, 3, 3), (3, 3, 3)]],
voxel_size=(1, 1, 1),
fov=(1, 1, 1))
dist_bc, fov = ops3d.conv_pass(last_fmap,
kernel_size=[[1, 1, 1]],
num_fmaps=len(labels),
activation=None,
fov=fov,
voxel_size=anisotropy)
output_shape_bc = dist_bc.get_shape().as_list()
output_shape_c = output_shape_bc[1:] # strip the batch dimension
output_shape = output_shape_c[1:]
dist_c = tf.reshape(dist_bc, output_shape_c)
names['dist'] = dist_c.name
network_outputs = tf.unstack(dist_c, len(labels), axis=0)
if mode.lower() == 'train' or mode.lower() == 'training':
# mask = tf.placeholder(tf.float32, shape=output_shape)
# ribo_mask = tf.placeholder(tf.float32, shape=output_shape)
gt = []
w = []
cw = []
masks = []
for l in labels:
masks.append(tf.placeholder(tf.float32, shape=output_shape))
gt.append(tf.placeholder(tf.float32, shape=output_shape))
w.append(tf.placeholder(tf.float32, shape=output_shape))
cw.append(l.class_weight)
lb = []
lub = []
for output_it, gt_it, w_it, m_it, label in zip(network_outputs, gt, w,
masks, labels):
lb.append(
tf.losses.mean_squared_error(gt_it, output_it, w_it * m_it))
lub.append(tf.losses.mean_squared_error(gt_it, output_it, m_it))
names[label.labelname] = output_it.name
names['gt_' + label.labelname] = gt_it.name
names['w_' + label.labelname] = w_it.name
names['mask_' + label.labelname] = m_it.name
for label, lb_it, lub_it in zip(labels, lb, lub):
tf.summary.scalar('lb_' + label.labelname, lb_it)
tf.summary.scalar('lub_' + label.labelname, lub_it)
names['lb_' + label.labelname] = lb_it.name
names['lb_' + label.labelname] = lub_it.name
loss_total = tf.add_n(lb)
loss_total_unbalanced = tf.add_n(lub)
loss_total_classweighted = tf.tensordot(lb, cw, axes=1)
loss_total_unbalanced_classweighted = tf.tensordot(lub, cw, axes=1)
tf.summary.scalar('loss_total', loss_total)
names['loss_total'] = loss_total.name
tf.summary.scalar('loss_total_unbalanced', loss_total_unbalanced)
names['loss_total_unbalanced'] = loss_total_unbalanced.name
tf.summary.scalar('loss_total_classweighted', loss_total_classweighted)
names['loss_total_classweighted'] = loss_total_classweighted.name
tf.summary.scalar('loss_total_unbalanced_classweighted',
loss_total_unbalanced_classweighted)
names[
'loss_total_unbalanced_classweighted'] = loss_total_unbalanced_classweighted.name
opt = tf.train.AdamOptimizer(learning_rate=0.5e-4,
beta1=0.95,
beta2=0.999,
epsilon=1e-8)
if loss_name == 'loss_total':
optimizer = opt.minimize(loss_total)
elif loss_name == 'loss_total_unbalanced':
optimizer = opt.minimize(loss_total_unbalanced)
elif loss_name == 'loss_total_unbalanced_classweighted':
optimizer = opt.minimize(loss_total_unbalanced_classweighted)
elif loss_name == 'loss_total_classweighted':
optimizer = opt.minimize(loss_total_classweighted)
else:
raise ValueError(loss_name + " not defined")
names['optimizer'] = optimizer.name
merged = tf.summary.merge_all()
names['summary'] = merged.name
with open('net_io_names.json', 'w') as f:
json.dump(names, f)
elif mode.lower() == 'inference' or mode.lower(
) == 'prediction' or mode.lower == 'pred':
pass
else:
raise ValueError(
"unknown mode for network construction {0:}".format(mode))
net_name = 'unet_' + mode
tf.train.export_meta_graph(filename=net_name + '.meta')
return net_name, output_shape
# In[78]:
with tf.Session() as sess:
x_v, x_list_v, y_v, y_list_v, x_r325_v, x_hs_v, x_hs_235_v = sess.run(
[x, x_list, y, y_list, x_r325, x_hs, x_hs_235])
print('x shape:', x_v.shape, 'x:\n', x_v)
#print('x list:\n', x_list_v)
#print('y shape:', y_v.shape, 'y:\n', y_v)
#print('y list:\n', y_list_v)
print('x_r325:\n', x_r325_v)
print('x_hs:\n', x_hs_v)
print('x_hs_235:\n', x_hs_235_v)
# In[107]:
x_us1_list = tf.unstack(x, axis=1)
x_us1_array = tf.stack(x_us1, axis=0)
x_us1_soft_list = [tf.nn.softmax(x_us1_list[ui]) for ui in range(2)]
# In[108]:
with tf.Session() as sess:
x_us1_list_v, x_us1_array_v, x_us1_soft_list_v = sess.run(
[x_us1_list, x_us1_array, x_us1_soft_list])
#print('x_us0:\n',x_us0_v)
print('x_us1_list type:', type(x_us1_list_v), 'x_us1_list[0] type:',
x_us1_list_v[0].shape, 'x_us1:\n', x_us1_list_v)
print('x_us1_array type:', type(x_us1_array_v), 'x_us1s shape:',
x_us1_array_v.shape, '\n', x_us1_array_v)
print('x_us1_soft_list type:', type(x_us1_soft_list_v),
def model(X,
params,
labels=None,
past=None,
scope='model',
reuse=False,
train=False):
with tf.variable_scope(scope, reuse=reuse):
results = {}
batch, sequence = shape_list(X)
if params["precision"] == "bfloat16":
wpe = tf.get_variable(
'wpe',
[params["n_ctx"], params["n_embd"]], # Position encoding
initializer=tf.random_normal_initializer(stddev=0.01,
dtype=tf.bfloat16),
dtype=tf.bfloat16)
wte = tf.get_variable(
'wte',
[params["n_vocab"], params["n_embd"]], # Text encoding
initializer=tf.random_normal_initializer(stddev=0.02,
dtype=tf.bfloat16),
dtype=tf.bfloat16)
else:
wpe = tf.get_variable(
'wpe',
[params["n_ctx"], params["n_embd"]], # Position encoding
initializer=tf.random_normal_initializer(stddev=0.01))
wte = tf.get_variable(
'wte',
[params["n_vocab"], params["n_embd"]], # Text encoding
initializer=tf.random_normal_initializer(stddev=0.02))
past_length = 0 if past is None else tf.shape(past)[-2]
wpe = dropout(wpe, params["embed_dropout"], train)
wte = dropout(wte, params["embed_dropout"], train)
h = tf.gather(wte, X) + tf.gather(wpe, positions_for(X, past_length))
# Transformer
presents = []
pasts = tf.unstack(
past, axis=1) if past is not None else [None] * params["n_layer"]
assert len(pasts) == params["n_layer"]
for layer, past in enumerate(pasts):
h, present = block(h,
'h%d' % layer,
past=past,
params=params,
train=train)
presents.append(present)
results['present'] = tf.stack(presents, axis=1)
h = norm(h, 'ln_f', params=params)
h_flat = tf.reshape(h, [batch * sequence, params["n_embd"]])
logits = tf.matmul(h_flat, wte, transpose_b=True)
logits = tf.reshape(logits, [batch, sequence, params["n_vocab"]])
results['logits'] = logits
return results
def unstack(input_tensor):
input_ = tf.expand_dims(input_tensor, axis=2)
return tf.unstack(input_, input_.shape[1], 1)
def postprocess_fastrcnn_h(self, rois, bbox_ppred, scores, img_shape):
'''
:param rois:[-1, 4]
:param bbox_ppred: [-1, (cfgs.Class_num+1) * 4]
:param scores: [-1, cfgs.Class_num + 1]
:return:
'''
with tf.name_scope('postprocess_fastrcnn_h'):
rois = tf.stop_gradient(rois)
scores = tf.stop_gradient(scores)
bbox_ppred = tf.reshape(bbox_ppred, [-1, cfgs.CLASS_NUM + 1, 4])
bbox_ppred = tf.stop_gradient(bbox_ppred)
bbox_pred_list = tf.unstack(bbox_ppred, axis=1)
score_list = tf.unstack(scores, axis=1)
allclasses_boxes = []
allclasses_scores = []
categories = []
for i in range(1, cfgs.CLASS_NUM + 1):
# 1. decode boxes in each class
tmp_encoded_box = bbox_pred_list[i]
tmp_score = score_list[i]
tmp_decoded_boxes = encode_and_decode.decode_boxes(
encode_boxes=tmp_encoded_box,
reference_boxes=rois,
scale_factors=cfgs.ROI_SCALE_FACTORS)
# tmp_decoded_boxes = encode_and_decode.decode_boxes(boxes=rois,
# deltas=tmp_encoded_box,
# scale_factor=cfgs.ROI_SCALE_FACTORS)
# 2. clip to img boundaries
tmp_decoded_boxes = boxes_utils.clip_boxes_to_img_boundaries(
decode_boxes=tmp_decoded_boxes, img_shape=img_shape)
# 3. NMS
keep = tf.image.non_max_suppression(
boxes=tmp_decoded_boxes,
scores=tmp_score,
max_output_size=cfgs.FAST_RCNN_NMS_MAX_BOXES_PER_CLASS,
iou_threshold=cfgs.FAST_RCNN_NMS_IOU_THRESHOLD)
perclass_boxes = tf.gather(tmp_decoded_boxes, keep)
perclass_scores = tf.gather(tmp_score, keep)
allclasses_boxes.append(perclass_boxes)
allclasses_scores.append(perclass_scores)
categories.append(tf.ones_like(perclass_scores) * i)
final_boxes = tf.concat(allclasses_boxes, axis=0)
final_scores = tf.concat(allclasses_scores, axis=0)
final_category = tf.concat(categories, axis=0)
# if self.is_training:
'''
in training. We should show the detecitons in the tensorboard. So we add this.
'''
kept_indices = tf.reshape(
tf.where(
tf.greater_equal(final_scores, cfgs.SHOW_SCORE_THRSHOLD)),
[-1])
final_boxes = tf.gather(final_boxes, kept_indices)
final_scores = tf.gather(final_scores, kept_indices)
final_category = tf.gather(final_category, kept_indices)
return final_boxes, final_scores, final_category
def adc_function_to_graph(adc, **kwargs):
channels = tf.unstack(adc, axis=-1)
channels = list(
map((lambda channel: vector_to_graph(channel, **kwargs)), channels))
return tf.stack(channels, axis=-1)
def _add_interpretation_graph(self):
"""Interpret NN output."""
mc = self.mc
with tf.variable_scope('interpret_output') as scope:
preds = self.preds
# probability
num_class_probs = mc.ANCHOR_PER_GRID * mc.CLASSES
self.pred_class_probs = tf.reshape(
tf.nn.softmax(
tf.reshape(preds[:, :, :, :num_class_probs],
[-1, mc.CLASSES])),
[mc.BATCH_SIZE, mc.ANCHORS, mc.CLASSES],
name='pred_class_probs')
print("pred_class_probs shape: ",
self.pred_class_probs.get_shape())
# confidence
num_confidence_scores = mc.ANCHOR_PER_GRID + num_class_probs
self.pred_conf = tf.sigmoid(tf.reshape(
preds[:, :, :, num_class_probs:num_confidence_scores],
[mc.BATCH_SIZE, mc.ANCHORS]),
name='pred_confidence_score')
print("pred_confidence_score: ", self.pred_conf.get_shape())
# bbox_delta
self.pred_box_delta = tf.reshape(preds[:, :, :,
num_confidence_scores:],
[mc.BATCH_SIZE, mc.ANCHORS, 4],
name='bbox_delta')
print("bbox_delta: ", self.pred_box_delta.get_shape())
# number of object. Used to normalize bbox and classification loss
self.num_objects = tf.reduce_sum(self.input_mask,
name='num_objects')
with tf.variable_scope('bbox') as scope:
with tf.variable_scope('stretching'):
delta_x, delta_y, delta_w, delta_h = tf.unstack(
self.pred_box_delta, axis=2)
anchor_x = mc.ANCHOR_BOX[:, 0]
anchor_y = mc.ANCHOR_BOX[:, 1]
anchor_w = mc.ANCHOR_BOX[:, 2]
anchor_h = mc.ANCHOR_BOX[:, 3]
box_center_x = tf.identity(anchor_x + delta_x * anchor_w,
name='bbox_cx')
box_center_y = tf.identity(anchor_y + delta_y * anchor_h,
name='bbox_cy')
box_width = tf.identity(anchor_w *
util.safe_exp(delta_w, mc.EXP_THRESH),
name='bbox_width')
box_height = tf.identity(anchor_h *
util.safe_exp(delta_h, mc.EXP_THRESH),
name='bbox_height')
self._activation_summary(delta_x, 'delta_x')
self._activation_summary(delta_y, 'delta_y')
self._activation_summary(delta_w, 'delta_w')
self._activation_summary(delta_h, 'delta_h')
self._activation_summary(box_center_x, 'bbox_cx')
self._activation_summary(box_center_y, 'bbox_cy')
self._activation_summary(box_width, 'bbox_width')
self._activation_summary(box_height, 'bbox_height')
with tf.variable_scope('trimming'):
xmins, ymins, xmaxs, ymaxs = util.bbox_transform(
[box_center_x, box_center_y, box_width, box_height])
# The max x position is mc.IMAGE_WIDTH - 1 since we use zero-based
# pixels. Same for y.
xmins = tf.minimum(tf.maximum(0.0, xmins),
mc.IMAGE_WIDTH - 1.0,
name='bbox_xmin')
self._activation_summary(xmins, 'box_xmin')
ymins = tf.minimum(tf.maximum(0.0, ymins),
mc.IMAGE_HEIGHT - 1.0,
name='bbox_ymin')
self._activation_summary(ymins, 'box_ymin')
xmaxs = tf.maximum(tf.minimum(mc.IMAGE_WIDTH - 1.0, xmaxs),
0.0,
name='bbox_xmax')
self._activation_summary(xmaxs, 'box_xmax')
ymaxs = tf.maximum(tf.minimum(mc.IMAGE_HEIGHT - 1.0, ymaxs),
0.0,
name='bbox_ymax')
self._activation_summary(ymaxs, 'box_ymax')
self.det_boxes = tf.transpose(tf.stack(
util.bbox_transform_inv([xmins, ymins, xmaxs, ymaxs])),
(1, 2, 0),
name='bbox')
with tf.variable_scope('IOU'):
def _tensor_iou(box1, box2):
with tf.variable_scope('intersection'):
xmin = tf.maximum(box1[0], box2[0], name='xmin')
ymin = tf.maximum(box1[1], box2[1], name='ymin')
xmax = tf.minimum(box1[2], box2[2], name='xmax')
ymax = tf.minimum(box1[3], box2[3], name='ymax')
w = tf.maximum(0.0, xmax - xmin, name='inter_w')
h = tf.maximum(0.0, ymax - ymin, name='inter_h')
intersection = tf.multiply(w, h, name='intersection')
with tf.variable_scope('union'):
w1 = tf.subtract(box1[2], box1[0], name='w1')
h1 = tf.subtract(box1[3], box1[1], name='h1')
w2 = tf.subtract(box2[2], box2[0], name='w2')
h2 = tf.subtract(box2[3], box2[1], name='h2')
union = w1 * h1 + w2 * h2 - intersection
return intersection/(union+mc.EPSILON) \
* tf.reshape(self.input_mask, [mc.BATCH_SIZE, mc.ANCHORS])
self.ious = self.ious.assign(
_tensor_iou(
util.bbox_transform(tf.unstack(self.det_boxes, axis=2)),
util.bbox_transform(tf.unstack(self.box_input, axis=2))))
self._activation_summary(self.ious, 'conf_score')
with tf.variable_scope('probability') as scope:
self._activation_summary(self.pred_class_probs, 'class_probs')
probs = tf.multiply(self.pred_class_probs,
tf.reshape(self.pred_conf,
[mc.BATCH_SIZE, mc.ANCHORS, 1]),
name='final_class_prob')
self._activation_summary(probs, 'final_class_prob')
self.det_probs = tf.reduce_max(probs, 2, name='score')
self.det_class = tf.argmax(probs, 2, name='class_idx')
def __init__(self):
# # Server Settings
BATCH_SIZE = 1
NUM_STEPS = 30
NUM_UNITS = 650
NUM_LAYERS = 2
KEEP_PROB = 0.35
MAX_GRAD_NORM = 5
LEARNING_RATE = 0.005
NUM_ITERATIONS = 2
sentences_gutenberg = gutenberg.sents()
guten_sents_count = len(sentences_gutenberg)
train_data_size = int(0.4 * guten_sents_count)
validation_data_size = int(0.1 * guten_sents_count)
test_data_size = int(0.1 * guten_sents_count)
# print(guten_sents_count)
# print(train_data_size)
# print(validation_data_size)
# print(test_data_size)
train_data = sentences_gutenberg[0:train_data_size]
validation_data = sentences_gutenberg[train_data_size:train_data_size +
validation_data_size]
test_data = sentences_gutenberg[train_data_size +
validation_data_size:train_data_size +
validation_data_size + test_data_size]
# print(len(train_data))
# print(len(validation_data))
# print(len(test_data))
# Local Settings
# BATCH_SIZE = 20
# NUM_STEPS = 10
# NUM_UNITS = 200
# NUM_LAYERS = 2
# KEEP_PROB = 0.8
# MAX_GRAD_NORM = 1
# LEARNING_RATE = 0.001
# NUM_ITERATIONS = 100
#
# sentences_gutenberg = gutenberg.sents()
# # print(len(sentences_gutenberg))
#
# train_data = sentences_gutenberg[0:1000]
# test_data = sentences_gutenberg[100:150]
# validation_data = sentences_gutenberg[150:200]
# print(len(train_data))
# print(train_data[3])
word_list_train, word_count_train = _read_words(train_data)
word_to_id_train, id_to_word_train = _build_vocab(train_data)
vocab_size = len(word_to_id_train)
# print('Vocab size :: ' + str(vocab_size))
# print(word_to_id_train['<eos>'])
train_data_id = _file_to_word_ids(train_data, word_to_id_train)
# print(len(train_data_id))
# print(word_to_id_train)
# print(id_to_word_train)
# Start: Code for training
# train_data_tensor = tf.convert_to_tensor(train_data_id, name="train_data_tensor", dtype=tf.int64)
# train_data_len = tf.size(train_data_tensor)
# batch_len = train_data_len // BATCH_SIZE
# data = tf.reshape(train_data_tensor[0:BATCH_SIZE * batch_len], [BATCH_SIZE, batch_len])
# epoch_size = (batch_len - 1) // NUM_STEPS
# assertion = tf.assert_positive(
# epoch_size,
# message="epoch_size == 0, decrease batch_size or num_steps")
#
# with tf.control_dependencies([assertion]):
# epoch_size = tf.identity(epoch_size, name="epoch_size")
#
# i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
# x = tf.strided_slice(data, [0, i * NUM_STEPS],
# [BATCH_SIZE, (i + 1) * NUM_STEPS])
# x.set_shape([BATCH_SIZE, NUM_STEPS])
# y = tf.strided_slice(data, [0, i * NUM_STEPS + 1],
# [BATCH_SIZE, (i + 1) * NUM_STEPS + 1])
# y.set_shape([BATCH_SIZE, NUM_STEPS])
# End: code for training
# # Start : Code for validation
# validation_data_id = _file_to_word_ids(validation_data, word_to_id_train)
#
# # print(validation_data_id)
#
# validation_data_tensor = tf.convert_to_tensor(validation_data_id, name="train_data_tensor", dtype=tf.int64)
# validation_data_len = tf.size(validation_data_tensor)
# batch_len = validation_data_len // BATCH_SIZE
# data = tf.reshape(validation_data_tensor[0:BATCH_SIZE * batch_len], [BATCH_SIZE, batch_len])
# epoch_size = (batch_len - 1) // NUM_STEPS
# assertion = tf.assert_positive(
# epoch_size,
# message="epoch_size == 0, decrease batch_size or num_steps")
#
# with tf.control_dependencies([assertion]):
# epoch_size = tf.identity(epoch_size, name="epoch_size")
#
# i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
# x = tf.strided_slice(data, [0, i * NUM_STEPS],
# [BATCH_SIZE, (i + 1) * NUM_STEPS])
# x.set_shape([BATCH_SIZE, NUM_STEPS])
# y = tf.strided_slice(data, [0, i * NUM_STEPS + 1],
# [BATCH_SIZE, (i + 1) * NUM_STEPS + 1])
# y.set_shape([BATCH_SIZE, NUM_STEPS])
# # end: Code for validation
# # Start : Code for test
# test_data_id = _file_to_word_ids(test_data, word_to_id_train)
#
# # print(validation_data_id)
#
# test_data_tensor = tf.convert_to_tensor(test_data_id, name="train_data_tensor", dtype=tf.int64)
# test_data_len = tf.size(test_data_tensor)
# batch_len = test_data_len // BATCH_SIZE
# data = tf.reshape(test_data_tensor[0:BATCH_SIZE * batch_len], [BATCH_SIZE, batch_len])
# epoch_size = (batch_len - 1) // NUM_STEPS
# assertion = tf.assert_positive(
# epoch_size,
# message="epoch_size == 0, decrease batch_size or num_steps")
#
# with tf.control_dependencies([assertion]):
# epoch_size = tf.identity(epoch_size, name="epoch_size")
#
# i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
# x = tf.strided_slice(data, [0, i * NUM_STEPS],
# [BATCH_SIZE, (i + 1) * NUM_STEPS])
# x.set_shape([BATCH_SIZE, NUM_STEPS])
# y = tf.strided_slice(data, [0, i * NUM_STEPS + 1],
# [BATCH_SIZE, (i + 1) * NUM_STEPS + 1])
# y.set_shape([BATCH_SIZE, NUM_STEPS])
# # end: Code for test
# # Start: code for sentence generation
SENTENCE_LENGTH = 15
seed_sentence = [word_to_id_train['<eos>']] * SENTENCE_LENGTH
x = tf.placeholder(dtype=tf.int64, shape=(None, None))
# # End: Code for sentence generation
# print(x.shape, y.shape)
# print(x[0])
# print(y[0])
# print(x[1])
# print(y[1])
embedding = tf.get_variable("embedding", [vocab_size, NUM_UNITS])
inputs = tf.nn.embedding_lookup(embedding, x)
enc_cell = tf.contrib.rnn.MultiRNNCell(
[_get_lstm_cell(NUM_UNITS, KEEP_PROB) for _ in range(NUM_LAYERS)],
state_is_tuple=True)
initial_state = enc_cell.zero_state(BATCH_SIZE, dtype=tf.float32)
# state = initial_state
# outputs = []
# with tf.variable_scope("RNN"):
# for time_step in range(NUM_STEPS):
# if time_step > 0: tf.get_variable_scope().reuse_variables()
# (cell_output, state) = enc_cell(inputs[:, time_step, :], state)
# outputs.append(cell_output)
# output = tf.reshape(tf.concat(outputs, 1), [-1, NUM_UNITS])
inputs = tf.unstack(inputs, num=NUM_STEPS, axis=1)
outputs, state = tf.nn.static_rnn(enc_cell,
inputs,
initial_state=initial_state)
output = tf.reshape(tf.concat(outputs, 1), [-1, NUM_UNITS])
softmax_w = tf.get_variable("softmax_w", [NUM_UNITS, vocab_size])
softmax_b = tf.get_variable("softmax_b", [vocab_size])
logits = tf.nn.xw_plus_b(output, softmax_w, softmax_b)
# Reshape logits to be a 3-D tensor for sequence loss
logits = tf.reshape(logits, [BATCH_SIZE, NUM_STEPS, vocab_size])
prediction = tf.nn.softmax(logits)
word_index = tf.argmax(prediction, 2)
# correct_pred = tf.equal(tf.argmax(prediction, 2), y)
# accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
#
# # Use the contrib sequence loss and average over the batches
# loss = tf.contrib.seq2seq.sequence_loss(
# logits,
# y,
# tf.ones([BATCH_SIZE, NUM_STEPS]),
# average_across_timesteps=False,
# average_across_batch=True)
#
# # Update the cost
# cost = tf.reduce_sum(loss)
# final_state = state
# if not is_training:
# return
# _lr = tf.Variable(0.0, trainable=False)
# tvars = tf.trainable_variables()
# grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), MAX_GRAD_NORM)
# optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
# _train_op = optimizer.apply_gradients(
# zip(grads, tvars),
# global_step=tf.train.get_or_create_global_step())
#
# _new_lr = tf.placeholder(
# tf.float32, shape=[], name="new_learning_rate")
# _lr_update = tf.assign(_lr, _new_lr)
#
# params = tf.trainable_variables()
# gradients = tf.gradients(loss, params)
# clipped_gradients, _ = tf.clip_by_global_norm(gradients, MAX_GRAD_NORM)
# #
# # # Optimization
# optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
# update_step = optimizer.apply_gradients(zip(clipped_gradients, params))
#
# sess = tf.Session()
# sess.run(tf.global_variables_initializer())
# tf.train.start_queue_runners(sess=sess)
# # # train_data_len, batch_len, epoch_size, i_, x, y = sess.run([train_data_len, batch_len, epoch_size, i, x, y])
# # # print(train_data_len, batch_len, epoch_size, i_)
# # # inputs = sess.run(inputs)
# # # print(inputs.shape)
# # # output = sess.run(output)
# # # print(output.shape)
# # # logits = sess.run(logits)
# # # print(logits.shape)
# # # cost = sess.run(cost)
# # # print(cost)
# #
# for epoch in range(NUM_ITERATIONS):
# loss_history = []
#
# _, train_loss, i_, x_, accuracy_ = sess.run((update_step, cost, i, x, accuracy))
# loss_history.append(train_loss)
#
# # print(i_)
# # print(x_[0])
# if (epoch + 1) % 2 == 0:
# print('Epoch: ' + str(epoch + 1), ' Loss: ' + str(np.mean(loss_history)))
# print(accuracy_)
# Save model
relative_path_parent_dir = os.path.dirname(__file__)
# print(relative_path_parent_dir)
# if os.path.isdir(os.path.join(relative_path_parent_dir, 'saved_model_data')):
# shutil.rmtree(os.path.join(relative_path_parent_dir, 'saved_model_data'))
# export_dir = os.path.join(relative_path_parent_dir, 'saved_model_data')
# saver = tf.train.Saver()
# saver.save(sess=sess, save_path=export_dir + '/session', write_meta_graph=False)
# Restore Model
sess = tf.Session()
tf.train.start_queue_runners(sess=sess)
export_dir = os.path.join(relative_path_parent_dir, 'word_model_data')
# tf.saved_model.loader.load(sess, ['train_model'], export_dir)
tf.train.Saver().restore(sess, export_dir + '/session')
# mean_loss = 0
# accuracy_mean = 0
#
# for epoch in range(NUM_ITERATIONS):
# loss_history = []
# accuracy_history = []
#
# validation_loss, i_, x_, pred_, accuracy_ = sess.run((cost, i, x, prediction, accuracy))
# loss_history.append(validation_loss)
# accuracy_history.append(accuracy_)
#
# # print(i_)
# # print(x_[0])
# # print(pred_.shape)
# print(accuracy_)
# if (epoch + 1) % 2 == 0:
# print('Epoch: ' + str(epoch + 1), ' Loss: ' + str(np.mean(loss_history)))
# print('Epoch: ' + str(epoch + 1), ' Accuracy: ' + str(np.mean(accuracy_history)))
#
# mean_loss = np.mean(loss_history)
# accuracy_mean = np.mean(accuracy_history)
#
# print('Mean Loss :: ', str(mean_loss))
# print('Mean Accuracy :: ', str(accuracy_mean))
#
# perplexity = _calculate_perplexity(mean_loss, NUM_STEPS)
# print('Perplexity :: ', str(perplexity))
# # Code for sentence generation
x_input = np.zeros((BATCH_SIZE, NUM_STEPS))
words_generated = []
for i in range(SENTENCE_LENGTH):
for t, word in enumerate(seed_sentence):
x_input[0, NUM_STEPS - SENTENCE_LENGTH + t] = word
word_index_ = sess.run(word_index, {x: x_input})
predicted_word = id_to_word_train[word_index_[0, -1]]
if predicted_word != '<eos>':
words_generated.append(predicted_word)
seed_sentence = seed_sentence[1:] + list([word_index_[0, -1]])
if predicted_word == '<eos>' and len(words_generated) > 10:
break
# print(words_generated)
sentence_generated = ' '.join(words_generated)
print(sentence_generated)
