def unpool_layer2x2_batch(self, bottom, argmax):
bottom_shape = tf.shape(bottom)
top_shape = [bottom_shape[0], bottom_shape[1] * 2, bottom_shape[2] * 2, bottom_shape[3]]
batch_size = top_shape[0]
height = top_shape[1]
width = top_shape[2]
channels = top_shape[3]
argmax_shape = tf.to_int64([batch_size, height, width, channels])
argmax = self.unravel_argmax(argmax, argmax_shape)
t1 = tf.to_int64(tf.range(channels))
t1 = tf.tile(t1, [batch_size * (width // 2) * (height // 2)])
t1 = tf.reshape(t1, [-1, channels])
t1 = tf.transpose(t1, perm=[1, 0])
t1 = tf.reshape(t1, [channels, batch_size, height // 2, width // 2, 1])
t1 = tf.transpose(t1, perm=[1, 0, 2, 3, 4])
t2 = tf.to_int64(tf.range(batch_size))
t2 = tf.tile(t2, [channels * (width // 2) * (height // 2)])
t2 = tf.reshape(t2, [-1, batch_size])
t2 = tf.transpose(t2, perm=[1, 0])
t2 = tf.reshape(t2, [batch_size, channels, height // 2, width // 2, 1])
t3 = tf.transpose(argmax, perm=[1, 4, 2, 3, 0])
t = tf.concat(4, [t2, t3, t1])
indices = tf.reshape(t, [(height // 2) * (width // 2) * channels * batch_size, 4])
x1 = tf.transpose(bottom, perm=[0, 3, 1, 2])
values = tf.reshape(x1, [-1])
return tf.scatter_nd(indices, values, tf.to_int64(top_shape))
def unpool_layer2x2(self, x, raveled_argmax, out_shape):
argmax = self.unravel_argmax(raveled_argmax, tf.to_int64(out_shape))
output = tf.zeros([out_shape[1], out_shape[2], out_shape[3]])
height = tf.shape(output)[0]
width = tf.shape(output)[1]
channels = tf.shape(output)[2]
t1 = tf.to_int64(tf.range(channels))
t1 = tf.tile(t1, [((width + 1) // 2) * ((height + 1) // 2)])
t1 = tf.reshape(t1, [-1, channels])
t1 = tf.transpose(t1, perm=[1, 0])
t1 = tf.reshape(t1, [channels, (height + 1) // 2, (width + 1) // 2, 1])
t2 = tf.squeeze(argmax)
t2 = tf.pack((t2[0], t2[1]), axis=0)
t2 = tf.transpose(t2, perm=[3, 1, 2, 0])
t = tf.concat(3, [t2, t1])
indices = tf.reshape(t, [((height + 1) // 2) * ((width + 1) // 2) * channels, 3])
x1 = tf.squeeze(x)
x1 = tf.reshape(x1, [-1, channels])
x1 = tf.transpose(x1, perm=[1, 0])
values = tf.reshape(x1, [-1])
delta = tf.SparseTensor(indices, values, tf.to_int64(tf.shape(output)))
return tf.expand_dims(tf.sparse_tensor_to_dense(tf.sparse_reorder(delta)), 0)
def one_hot_matrix(tensor_in, num_classes, on_value=1.0, off_value=0.0):
"""Encodes indices from given tensor as one-hot tensor.
TODO(ilblackdragon): Ideally implementation should be
part of TensorFlow with Eigen-native operation.
Args:
tensor_in: Input tensor of shape [N1, N2].
num_classes: Number of classes to expand index into.
on_value: Tensor or float, value to fill-in given index.
off_value: Tensor or float, value to fill-in everything else.
Returns:
Tensor of shape [N1, N2, num_classes] with 1.0 for each id in original
tensor.
"""
tensor_in = tf.convert_to_tensor(tensor_in)
sparse_values = tf.to_int64(tf.reshape(tensor_in, [-1, 1]))
size = tf.shape(sparse_values)[0]
dims = tf.shape(tensor_in)
indices = tf.to_int64(tf.reshape(tf.range(0, size), [-1, 1]))
indices_values = tf.concat(1, [indices, sparse_values])
outshape = tf.to_int64(expand_concat(0, [size, num_classes]))
one_hot_vector = tf.sparse_to_dense(indices_values, outshape, on_value, off_value)
ret = tf.reshape(one_hot_vector, tf.concat(0, [dims, [num_classes]]))
ret.set_shape(tensor_in.get_shape().concatenate(num_classes))
return ret
def _build_once(self, dataset, feature_transformer):
with tf.device(self._local_device):
tr_batch = dataset()
te_batch = dataset()
num_classes = tr_batch.label_onehot.shape.as_list()[1]
all_batch = utils.structure_map_multi(lambda x: tf.concat(x, 0),
[tr_batch, te_batch])
features = feature_transformer(all_batch)
trX, teX = utils.structure_map_split(lambda x: tf.split(x, 2, axis=0),
features)
trY = tf.to_int64(tr_batch.label)
trY_onehot = tf.to_int32(tr_batch.label_onehot)
teY = tf.to_int64(te_batch.label)
teY_shape = teY.shape.as_list()
def blackbox((trX, trY, teX, teY)):
trY = tf.to_int32(tf.rint(trY))
teY = tf.to_int32(tf.rint(teY))
tf_fn = build_fit(
self._local_device,
self._get_model,
num_classes=num_classes,
probs=self.probs)
if self.probs:
trP, teP, teP_probs = tf_fn(trX, trY, teX)
else:
trP, teP = tf_fn(trX, trY, teX)
teY.set_shape(teY_shape)
if self.probs:
onehot = tf.one_hot(teY, num_classes)
crossent = -tf.reduce_sum(onehot * teP_probs, [1])
return tf.reduce_mean(crossent)
else:
# use error rate as the loss if no surrogate is avalible.
return 1 - tf.reduce_mean(
tf.to_float(tf.equal(teY, tf.to_int32(teP))))
test_loss = blackbox((trX, tf.to_float(trY), teX, tf.to_float(teY)))
stats = {}
tf_fn = build_fit(
self._local_device,
self._get_model,
num_classes=num_classes,
probs=self.probs)
if self.probs:
trP, teP, teP_probs = tf_fn(trX, trY, teX)
else:
trP, teP = tf_fn(trX, trY, teX)
stats["%s/accuracy_train" % self.name] = tf.reduce_mean(
tf.to_float(tf.equal(tf.to_int32(trY), tf.to_int32(trP))))
stats["%s/accuracy_test" % self.name] = tf.reduce_mean(
tf.to_float(tf.equal(tf.to_int32(teY), tf.to_int32(teP))))
stats["%s/test_loss" % self.name] = test_loss
return test_loss, stats
def build(self):
print('Building model')
self.x_embeddings = tf.Variable(
tf.random_normal([self.alphabet_src_size, self.embedd_dims],
stddev=0.1), name='x_embeddings')
self.t_embeddings = tf.Variable(
tf.random_normal([self.alphabet_tar_size, self.embedd_dims],
stddev=0.1), name='t_embeddings')
X_embedded = tf.gather(self.x_embeddings, self.Xs, name='embed_X')
t_embedded = tf.gather(self.t_embeddings, self.ts_go, name='embed_t')
with tf.variable_scope('dense_out'):
W_out = tf.get_variable('W_out', [self.word_encoder_units*2, self.alphabet_tar_size])
b_out = tf.get_variable('b_out', [self.alphabet_tar_size])
# forward encoding
char_enc_state, char_enc_out = encoder(X_embedded, self.X_len, 'char_encoder', self.char_encoder_units)
char2word = _grid_gather(char_enc_out, self.X_spaces)
char2word.set_shape([None, None, self.char_encoder_units])
word_enc_state, word_enc_out = encoder(char2word, self.X_spaces_len, 'word_encoder', self.word_encoder_units)
# backward encoding words
char2word = tf.reverse_sequence(char2word, tf.to_int64(self.X_spaces_len), 1)
char2word.set_shape([None, None, self.char_encoder_units])
word_enc_state_bck, word_enc_out_bck = encoder(char2word, self.X_spaces_len, 'word_encoder_backwards', self.word_encoder_units)
word_enc_out_bck = tf.reverse_sequence(word_enc_out_bck, tf.to_int64(self.X_spaces_len), 1)
word_enc_state = tf.concat(1, [word_enc_state, word_enc_state_bck])
word_enc_out = tf.concat(2, [word_enc_out, word_enc_out_bck])
# decoding
dec_state, dec_out, valid_dec_out, valid_attention_tracker = (
attention_decoder(word_enc_out, self.X_spaces_len, word_enc_state,
t_embedded, self.t_len, self.attn_units,
self.t_embeddings, W_out, b_out))
out_tensor = tf.reshape(dec_out, [-1, self.word_encoder_units*2])
out_tensor = tf.matmul(out_tensor, W_out) + b_out
out_shape = tf.concat(0, [tf.expand_dims(tf.shape(self.X_len)[0], 0),
tf.expand_dims(tf.shape(t_embedded)[1], 0),
tf.expand_dims(tf.constant(self.alphabet_tar_size), 0)])
self.valid_attention_tracker = valid_attention_tracker.pack()
self.out_tensor = tf.reshape(out_tensor, out_shape)
self.out_tensor.set_shape([None, None, self.alphabet_tar_size])
valid_out_tensor = tf.reshape(valid_dec_out, [-1, self.word_encoder_units*2])
valid_out_tensor = tf.matmul(valid_out_tensor, W_out) + b_out
self.valid_out_tensor = tf.reshape(valid_out_tensor, out_shape)
self.out = None
# add TensorBoard summaries for all variables
tf.contrib.layers.summarize_variables()
def preprocess_example(self, example, mode, unused_hparams):
# Just resize with area.
if self._was_reversed:
example["inputs"] = tf.to_int64(
tf.image.resize_images(example["inputs"], [32, 32],
tf.image.ResizeMethod.AREA))
else:
example = imagenet_preprocess_example(example, mode)
example["inputs"] = tf.to_int64(
tf.image.resize_images(example["inputs"], [32, 32]))
return example
def preprocess_example(self, example, mode, _):
# Just resize with area.
if self._was_reversed:
example["inputs"] = tf.to_int64(
tf.image.resize_images(example["inputs"], self.rescale_size,
tf.image.ResizeMethod.AREA))
else:
example = imagenet_preprocess_example(example, mode)
example["inputs"] = tf.to_int64(
tf.image.resize_images(example["inputs"], self.rescale_size))
return example
def model_single(input_dims, output_dims, scale_frac, scales, nkNN):
"""
Forms the knn model.
Arguments:
input_dim -- the dimension of the input data
output_dim -- the number of classes
scale_frac -- the fraction of events to use for finding widths
scales -- list of distribution widths for each dimension
nkNN -- the number of nearest neighbours to find
Returns:
A tensor with the number of neighbours in each class.
"""
training = tf.placeholder(tf.float32, shape=(None, input_dims))
one_hot = tf.placeholder(tf.float32, shape=(None, output_dims))
test = tf.placeholder(tf.float32, shape=(1, input_dims))
distances = metric_single(training, test, scale_frac, scales)
remaining_training = tf.identity(training)
remaining_one_hot = tf.identity(one_hot)
remaining_distances = tf.identity(distances)
for i in range(nkNN):
# Gets the location of training entry currently closest to the test
# entry.
min_slice = tf.to_int64(tf.concat(0, [tf.argmin(remaining_distances, 0), [-1]]))
# Cuts the nearest neighbour out of the training set.
start = tf.slice(remaining_training, tf.to_int64([0, 0]), min_slice)
end = tf.slice(remaining_training, min_slice + [1, 1], [-1, -1])
remaining_training = tf.concat(0, [start, end])
# Cuts the nearest neighbour out of the distances set.
start = tf.slice(remaining_distances, tf.to_int64([0, 0]), min_slice)
end = tf.slice(remaining_distances, min_slice + [1, 1], [-1, -1])
remaining_distances = tf.concat(0, [start, end])
# Cuts the nearest neighbour's class and records it.
start = tf.slice(remaining_one_hot, tf.to_int64([0, 0]), min_slice)
end = tf.slice(remaining_one_hot, min_slice + [1, 1], [-1, -1])
class_slice = tf.slice(remaining_one_hot, min_slice + [0, 1], [1, -1])
remaining_one_hot = tf.concat(0, [start, end])
if i == 0:
neighbour_one_hot = class_slice
else:
neighbour_one_hot = tf.concat(0, [neighbour_one_hot, class_slice])
return training, one_hot, test, tf.reduce_sum(neighbour_one_hot, reduction_indices=0)
def _define_distance_to_clusters(self, data):
"""Defines the Mahalanobis distance to the assigned Gaussian."""
# TODO(xavigonzalvo): reuse (input - mean) * cov^-1 * (input -
# mean) from log probability function.
self._all_scores = []
for shard in data:
all_scores = []
shard = tf.expand_dims(shard, 0)
for c in xrange(self._num_classes):
if self._covariance_type == FULL_COVARIANCE:
cov = self._covs[c, :, :]
elif self._covariance_type == DIAG_COVARIANCE:
cov = tf.diag(self._covs[c, :])
inverse = tf.matrix_inverse(cov + self._min_var)
inv_cov = tf.tile(
tf.expand_dims(inverse, 0),
tf.pack([self._num_examples, 1, 1]))
diff = tf.transpose(shard - self._means[c, :, :], perm=[1, 0, 2])
m_left = tf.batch_matmul(diff, inv_cov)
all_scores.append(tf.sqrt(tf.batch_matmul(
m_left, tf.transpose(diff, perm=[0, 2, 1])
)))
self._all_scores.append(tf.reshape(
tf.concat(1, all_scores),
tf.pack([self._num_examples, self._num_classes])))
# Distance to the associated class.
self._all_scores = tf.concat(0, self._all_scores)
assignments = tf.concat(0, self.assignments())
rows = tf.to_int64(tf.range(0, self._num_examples))
indices = tf.concat(1, [tf.expand_dims(rows, 1),
tf.expand_dims(assignments, 1)])
self._scores = tf.gather_nd(self._all_scores, indices)
def generalised_dice_loss(prediction,
ground_truth,
weight_map=None,
type_weight='Square'):
"""
Function to calculate the Generalised Dice Loss defined in
Sudre, C. et. al. (2017) Generalised Dice overlap as a deep learning
loss function for highly unbalanced segmentations. DLMIA 2017
:param prediction: the logits
:param ground_truth: the segmentation ground truth
:param weight_map:
:param type_weight: type of weighting allowed between labels (choice
between Square (square of inverse of volume),
Simple (inverse of volume) and Uniform (no weighting))
:return: the loss
"""
ground_truth = tf.to_int64(ground_truth)
n_voxels = ground_truth.shape[0].value
n_classes = prediction.shape[1].value
ids = tf.constant(np.arange(n_voxels), dtype=tf.int64)
ids = tf.stack([ids, ground_truth], axis=1)
one_hot = tf.SparseTensor(indices=ids,
values=tf.ones([n_voxels], dtype=tf.float32),
dense_shape=[n_voxels, n_classes])
if weight_map is not None:
weight_map_nclasses = tf.reshape(
tf.tile(weight_map, [n_classes]), prediction.get_shape())
ref_vol = tf.sparse_reduce_sum(
weight_map_nclasses * one_hot, reduction_axes=[0])
intersect = tf.sparse_reduce_sum(
weight_map_nclasses * one_hot * prediction, reduction_axes=[0])
seg_vol = tf.reduce_sum(
tf.multiply(weight_map_nclasses, prediction), 0)
else:
ref_vol = tf.sparse_reduce_sum(one_hot, reduction_axes=[0])
intersect = tf.sparse_reduce_sum(one_hot * prediction,
reduction_axes=[0])
seg_vol = tf.reduce_sum(prediction, 0)
if type_weight == 'Square':
weights = tf.reciprocal(tf.square(ref_vol))
elif type_weight == 'Simple':
weights = tf.reciprocal(ref_vol)
elif type_weight == 'Uniform':
weights = tf.ones_like(ref_vol)
else:
raise ValueError("The variable type_weight \"{}\""
"is not defined.".format(type_weight))
new_weights = tf.where(tf.is_inf(weights), tf.zeros_like(weights), weights)
weights = tf.where(tf.is_inf(weights), tf.ones_like(weights) *
tf.reduce_max(new_weights), weights)
generalised_dice_numerator = \
2 * tf.reduce_sum(tf.multiply(weights, intersect))
generalised_dice_denominator = \
tf.reduce_sum(tf.multiply(weights, seg_vol + ref_vol))
generalised_dice_score = \
generalised_dice_numerator / generalised_dice_denominator
return 1 - generalised_dice_score
def loss(logits, labels):
"""Calculates the loss from the logits and the labels."""
labels = tf.to_int64(labels)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits, labels, name='xentropy')
loss = tf.reduce_mean(cross_entropy, name='xentropy_mean')
return loss
def make_multiscale(image, resolutions,
resize_method=tf.image.ResizeMethod.BICUBIC,
num_channels=3):
"""Returns list of scaled images, one for each resolution.
Args:
image: Tensor of shape [height, height, num_channels].
resolutions: List of heights that image's height is resized to.
resize_method: tf.image.ResizeMethod.
num_channels: Number of channels in image.
Returns:
List of Tensors, one for each resolution with shape given by
[resolutions[i], resolutions[i], num_channels].
"""
scaled_images = []
for height in resolutions:
scaled_image = tf.image.resize_images(
image,
size=[height, height], # assuming that height = width
method=resize_method)
scaled_image = tf.to_int64(scaled_image)
scaled_image.set_shape([height, height, num_channels])
scaled_images.append(scaled_image)
return scaled_images
def gather(self, indices, keys=None):
"""Returns elements at the specified indices from the buffer.
Args:
indices: (list of integers or rank 1 int Tensor) indices in the buffer to
retrieve elements from.
keys: List of keys of tensors to retrieve. If None retrieve all.
Returns:
A list of tensors, where each element in the list is obtained by indexing
one of the tensors in the buffer.
Raises:
ValueError: If gather is called before calling the add function.
tf.errors.InvalidArgumentError: If indices are bigger than the number of
items in the buffer.
"""
if not self._tensors:
raise ValueError('The add function must be called before calling gather.')
if keys is None:
keys = self._tensors.keys()
with tf.name_scope('Gather'):
index_bound_assert = tf.Assert(
tf.less(
tf.to_int64(tf.reduce_max(indices)),
tf.minimum(self.get_num_adds(), self._buffer_size)),
['Index out of bounds.'])
with tf.control_dependencies([index_bound_assert]):
indices = tf.convert_to_tensor(indices)
batch = []
for key in keys:
batch.append(tf.gather(self._tensors[key], indices, name=key))
return batch
def build_predict(self,Xnew,task_ind):
"""
We need to assume the task_ind starts from 0
"""
Fmean,Fvar = 0,0
for i in np.arange(self.rank):
for j in np.arange(self.num_latent_list[i]):
lat_id = np.sum(self.num_latent_list[:i],dtype = np.int64) + j
if self.whiten_list[lat_id]: # need to compute fmean and fvar by the weights
fmean, fvar = conditionals.gaussian_gp_predict_whitened(Xnew, self.Z[lat_id],
self.kern_list[i], self.q_mu_list[lat_id],
self.q_sqrt_list[lat_id], 1)
else:
fmean, fvar = conditionals.gaussian_gp_predict(Xnew, self.Z[lat_id],
self.kern_list[i], self.q_mu_list[lat_id],
self.q_sqrt_list[lat_id],1)
W_ij = tf.gather(self.W,task_ind)[lat_id]
Fmean += (fmean + self.mean_function_list[lat_id](Xnew))*W_ij
Fvar += fvar * tf.square(W_ij)
if self.tsk:
for i in np.arange(self.num_tasks):
lat_id = np.sum(self.num_latent_list,dtype = np.int64) + i
if self.whiten_list[lat_id]: # need to compute fmean and fvar by the weights
fmean, fvar = conditionals.gaussian_gp_predict_whitened(Xnew, self.Z[lat_id],
self.tskern_list[i], self.q_mu_list[lat_id],
self.q_sqrt_list[lat_id], 1)
else:
fmean, fvar = conditionals.gaussian_gp_predict(Xnew, self.Z[lat_id],
self.tskern_list[i], self.q_mu_list[lat_id],
self.q_sqrt_list[lat_id], 1)
switch = tf.cast(tf.equal(tf.to_int64(i), task_ind),tf.float64)
W_ij = tf.gather(self.Kappa,i)[0]*switch
Fmean += (fmean + self.mean_function_list[lat_id](Xnew))*W_ij
Fvar += fvar * tf.square(W_ij)
return Fmean, Fvar
def loss(logits, labels):
"""Add L2Loss to all the trainable variables.
Add summary for for "Loss" and "Loss/avg".
Args:
logits: Logits from inference().
labels: Labels from distorted_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
# Reshape the labels into a dense Tensor of
# shape [batch_size, NUM_CLASSES].
sparse_labels = tf.reshape(labels, [FLAGS.batch_size, 1])
indices = tf.reshape(tf.range(0, FLAGS.batch_size), [FLAGS.batch_size, 1])
concated = tf.concat(1, [tf.to_int64(indices, name='ToInt64'), sparse_labels])
dense_labels = tf.sparse_to_dense(concated,
[FLAGS.batch_size, FLAGS.num_classes],
1.0, 0.0)
# Calculate the average cross entropy loss across the batch.
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
logits, dense_labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
return tf.add_n(tf.get_collection('losses'), name='total_loss')
def infer(self, features, *args, **kwargs):
"""Produce predictions from the model by running it."""
del args, kwargs
if "targets" not in features:
if "infer_targets" in features:
targets_shape = common_layers.shape_list(features["infer_targets"])
elif "inputs" in features:
targets_shape = common_layers.shape_list(features["inputs"])
targets_shape[1] = self.hparams.video_num_target_frames
else:
raise ValueError("no inputs are given.")
features["targets"] = tf.zeros(targets_shape, dtype=tf.float32)
output, _ = self(features) # pylint: disable=not-callable
if not isinstance(output, dict):
output = {"targets": output}
x = output["targets"]
if self.is_per_pixel_softmax:
x_shape = common_layers.shape_list(x)
x = tf.reshape(x, [-1, x_shape[-1]])
x = tf.argmax(x, axis=-1)
x = tf.reshape(x, x_shape[:-1])
else:
x = tf.squeeze(x, axis=-1)
x = tf.to_int64(tf.round(x))
output["targets"] = x
if self.hparams.reward_prediction:
output["target_reward"] = tf.argmax(output["target_reward"], axis=-1)
# only required for decoding.
output["outputs"] = output["targets"]
output["scores"] = output["targets"]
return output
def _create_predictions(self, decoder_output, features, labels, losses=None):
"""Creates the dictionary of predictions that is returned by the model.
"""
predictions = {}
# Add features and, if available, labels to predictions
predictions.update(_flatten_dict({"features": features}))
if labels is not None:
predictions.update(_flatten_dict({"labels": labels}))
if losses is not None:
predictions["losses"] = _transpose_batch_time(losses)
# Decoders returns output in time-major form [T, B, ...]
# Here we transpose everything back to batch-major for the user
output_dict = collections.OrderedDict(
zip(decoder_output._fields, decoder_output))
decoder_output_flat = _flatten_dict(output_dict)
decoder_output_flat = {
k: _transpose_batch_time(v)
for k, v in decoder_output_flat.items()
}
predictions.update(decoder_output_flat)
# If we predict the ids also map them back into the vocab and process them
if "predicted_ids" in predictions.keys():
vocab_tables = graph_utils.get_dict_from_collection("vocab_tables")
target_id_to_vocab = vocab_tables["target_id_to_vocab"]
predicted_tokens = target_id_to_vocab.lookup(
tf.to_int64(predictions["predicted_ids"]))
# Raw predicted tokens
predictions["predicted_tokens"] = predicted_tokens
return predictions
def preprocess_example(self, example, mode, unused_hparams):
example["inputs"].set_shape([_CIFAR10_IMAGE_SIZE, _CIFAR10_IMAGE_SIZE, 3])
example["inputs"] = tf.to_int64(example["inputs"])
if mode == tf.estimator.ModeKeys.TRAIN:
example["inputs"] = image_utils.random_shift(
example["inputs"], wsr=0.1, hsr=0.1)
return example
def mnist_training(logits, labels, learning_rate):
"""Build the training graph.
Args:
logits: Logits tensor, float - [BATCH_SIZE, NUM_CLASSES].
labels: Labels tensor, int32 - [BATCH_SIZE], with values in the
range [0, NUM_CLASSES).
learning_rate: The learning rate to use for gradient descent.
Returns:
train_op: The Op for training.
loss: The Op for calculating loss.
"""
# Create an operation that calculates loss.
labels = tf.to_int64(labels)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits, labels, name='xentropy')
loss = tf.reduce_mean(cross_entropy, name='xentropy_mean')
# Create the gradient descent optimizer with the given learning rate.
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
# Create a variable to track the global step.
global_step = tf.Variable(0, name='global_step', trainable=False)
# Use the optimizer to apply the gradients that minimize the loss
# (and also increment the global step counter) as a single training step.
train_op = optimizer.minimize(loss, global_step=global_step)
# Uncomment the following line to see what we have constructed.
# tf.train.write_graph(tf.get_default_graph().as_graph_def(),
# "/tmp", "train.pbtxt", as_text=True)
return train_op, loss
def _compute_sparse_average_correct(input_, labels, per_example_weights, topk=1):
"""Returns the numerator and denominator of classifier accuracy."""
labels = tf.to_int64(labels)
labels.get_shape().assert_is_compatible_with([input_.get_shape()[0], None])
if topk == 1:
predictions = tf.reshape(tf.argmax(input_, 1), [-1, 1])
in_topk = tf.reduce_any(tf.equal(labels, predictions), reduction_indices=[1])
else:
# Use broadcasting to check if ANY of the predictions are in the top k.
# TODO(eiderman): For a multi-label top k, what does accuracy mean?
predictions = tf.reshape(tf.nn.top_k(input_, topk)[1], [-1, 1, topk])
labels = tf.expand_dims(labels, [-1])
in_topk = tf.reduce_any(tf.equal(tf.cast(labels, predictions.dtype), predictions), reduction_indices=[1, 2])
correct_predictions = tf.to_float(in_topk)
# If individual examples are weighted, then we want to normalize by that.
if per_example_weights is not None:
per_example_weights = _convert_and_assert_per_example_weights_compatible(
input_, per_example_weights, dtype=None
)
float_weights = tf.to_float(per_example_weights)
# TODO(eiderman): This should use an op that doesn't support broadcasting.
correct_predictions *= float_weights
num_examples = tf.reduce_sum(float_weights)
else:
# shape only holds ints, but we want to always return the same type
# for num_examples to make everything compatible.
num_examples = tf.to_float(tf.gather(tf.shape(input_), 0))
return tf.reduce_sum(correct_predictions), num_examples
def main(args):
# load the dataset
dataset = mnist.get_split('test', FLAGS.data_dir)
# load batch
images, labels = load_batch(
dataset,
FLAGS.batch_size,
is_training=False)
# get the model prediction
predictions = lenet(images)
# convert prediction values for each class into single class prediction
predictions = tf.to_int64(tf.argmax(predictions, 1))
# streaming metrics to evaluate
metrics_to_values, metrics_to_updates = metrics.aggregate_metric_map({
'mse': metrics.streaming_mean_squared_error(predictions, labels),
'accuracy': metrics.streaming_accuracy(predictions, labels),
})
# write the metrics as summaries
for metric_name, metric_value in metrics_to_values.iteritems():
tf.summary.scalar(metric_name, metric_value)
# evaluate on the model saved at the checkpoint directory
# evaluate every eval_interval_secs
slim.evaluation.evaluation_loop(
'',
FLAGS.checkpoint_dir,
FLAGS.log_dir,
num_evals=FLAGS.num_evals,
eval_op=metrics_to_updates.values(),
eval_interval_secs=FLAGS.eval_interval_secs)
def preprocess_example(self, example, mode, unused_hparams):
example["inputs"].set_shape([_CIFAR10_IMAGE_SIZE, _CIFAR10_IMAGE_SIZE, 3])
if mode == tf.estimator.ModeKeys.TRAIN:
example["inputs"] = common_layers.cifar_image_augmentation(
example["inputs"])
example["inputs"] = tf.to_int64(example["inputs"])
return example
def tensors_to_item(self, keys_to_tensors):
"""Maps the given dictionary of tensors to a concatenated list of bboxes.
Args:
keys_to_tensors: a mapping of TF-Example keys to parsed tensors.
Returns:
[time, num_boxes, 4] tensor of bounding box coordinates, in order
[y_min, x_min, y_max, x_max]. Whether the tensor is a SparseTensor
or a dense Tensor is determined by the return_dense parameter. Empty
positions in the sparse tensor are filled with -1.0 values.
"""
sides = []
for key in self._full_keys:
value = keys_to_tensors[key]
expanded_dims = tf.concat(
[tf.to_int64(tf.shape(value)),
tf.constant([1], dtype=tf.int64)], 0)
side = tf.sparse_reshape(value, expanded_dims)
sides.append(side)
bounding_boxes = tf.sparse_concat(2, sides)
if self._return_dense:
bounding_boxes = tf.sparse_tensor_to_dense(
bounding_boxes, default_value=self._default_value)
return bounding_boxes
def ndlstm_base_dynamic(inputs, noutput, scope=None, reverse=False):
"""Run an LSTM, either forward or backward.
This is a 1D LSTM implementation using dynamic_rnn and
the TensorFlow LSTM op.
Args:
inputs: input sequence (length, batch_size, ninput)
noutput: depth of output
scope: optional scope name
reverse: run LSTM in reverse
Returns:
Output sequence (length, batch_size, noutput)
"""
with tf.variable_scope(scope, "SeqLstm", [inputs]):
# TODO(tmb) make batch size, sequence_length dynamic
# example: sequence_length = tf.shape(inputs)[0]
_, batch_size, _ = _shape(inputs)
lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(noutput, state_is_tuple=False)
state = tf.zeros([batch_size, lstm_cell.state_size])
sequence_length = int(inputs.get_shape()[0])
sequence_lengths = tf.to_int64(tf.fill([batch_size], sequence_length))
if reverse:
inputs = tf.reverse(inputs, [True, False, False])
outputs, _ = tf.nn.dynamic_rnn(lstm_cell,
inputs,
sequence_lengths,
state,
time_major=True)
if reverse:
outputs = tf.reverse(outputs, [True, False, False])
return outputs
def infer_step(recent_output, recent_logits, unused_loss):
"""Inference step."""
if not context.in_eager_mode():
recent_output.set_shape([None, None, None, 1])
padded = tf.pad(recent_output, [[0, 0], [0, 1], [0, 0], [0, 0]])
features["targets"] = padded
# This is inefficient in that it generates samples at all timesteps,
# not just the last one, except if target_modality is pointwise.
samples, logits, losses = self.sample(features)
# Concatenate the already-generated recent_output with last timestep
# of the newly-generated samples.
if target_modality.top_is_pointwise:
cur_sample = samples[:, -1, :, :]
else:
cur_sample = samples[:,
common_layers.shape_list(recent_output)[1], :, :]
cur_sample = tf.to_int64(tf.expand_dims(cur_sample, axis=1))
samples = tf.concat([recent_output, cur_sample], axis=1)
if not context.in_eager_mode():
samples.set_shape([None, None, None, 1])
# Assuming we have one shard for logits.
logits = tf.concat([recent_logits, logits[:, -1:]], 1)
loss = sum([l for l in losses.values() if l is not None])
return samples, logits, loss
def generate_top_k_scores_and_ids(logits, top_k):
"""This function computes top K ids and scores from logits tensor.
Args:
logits: logit tensor computed in the serving graph.
top_k: number of top K elements to rank.
Returns:
predictions: scores of top K items.
output_alternatives: ids of the top K items.
"""
probabilities = tf.nn.softmax(
logits, name=tf.contrib.learn.PredictionKey.PROBABILITIES)
top_k_scores, top_k_ids = tf.nn.top_k(
input=probabilities, k=top_k)
top_k_ids = tf.contrib.lookup.index_to_string(
tf.to_int64(top_k_ids),
mapping=tf.constant([str(i) for i in xrange(MOVIE_VOCAB_SIZE)]))
predictions = {
# served as "scores" by Servo in the ClassificationResult
tf.contrib.learn.PredictionKey.PROBABILITIES:
top_k_scores,
# served as "classes" by Servo in the ClassificationResult
tf.contrib.learn.PredictionKey.CLASSES:
top_k_ids
}
output_alternatives = {DEFAULT_OUTPUT_ALTERNATIVE: (
tf.contrib.learn.ProblemType.CLASSIFICATION,
predictions)}
return predictions, output_alternatives
def make_multiscale_dilated(image, resolutions, num_channels=3):
"""Returns list of scaled images, one for each resolution.
Resizes by skipping every nth pixel.
Args:
image: Tensor of shape [height, height, num_channels].
resolutions: List of heights that image's height is resized to. The function
assumes VALID padding, so the original image's height must be divisible
by each resolution's height to return the exact resolution size.
num_channels: Number of channels in image.
Returns:
List of Tensors, one for each resolution with shape given by
[resolutions[i], resolutions[i], num_channels] if resolutions properly
divide the original image's height; otherwise shape height and width is up
to valid skips.
"""
image_height = common_layers.shape_list(image)[0]
scaled_images = []
for height in resolutions:
dilation_rate = image_height // height # assuming height = width
scaled_image = image[::dilation_rate, ::dilation_rate]
scaled_image = tf.to_int64(scaled_image)
scaled_image.set_shape([None, None, num_channels])
scaled_images.append(scaled_image)
return scaled_images
def key_func(src, tgt):
src_len = src[-1]
tgt_len = tgt[-1]
num_buckets = 4
bucket_width = 5
bucket_id = tf.maximum(src_len // bucket_width, tgt_len // bucket_width)
return tf.to_int64(tf.minimum(num_buckets, bucket_id))
def generate_single_output(encoder_state, attention_states, sequence_length,
targets, num_classes, buckets,
use_mean_attention=False,
softmax_loss_function=None, per_example_loss=False,
name=None, use_attention=False):
all_inputs = targets
with tf.name_scope(name, "model_with_buckets", all_inputs):
with tf.variable_scope(tf.get_variable_scope(),
reuse=None):
single_outputs = attention_single_output_decoder(encoder_state,
attention_states,
output_size=num_classes,
num_heads=1,
sequence_length=sequence_length,
use_attention=use_attention)
_, _, _, bucket_outputs = single_outputs
if softmax_loss_function is None:
assert len(bucket_outputs) == len(targets) == 1
# We need to make target and int64-tensor and set its shape.
bucket_target = tf.reshape(tf.to_int64(targets[0]), [-1])
crossent = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=bucket_outputs[0], labels=bucket_target)
else:
assert len(bucket_outputs) == len(targets) == 1
crossent = softmax_loss_function(bucket_outputs[0], targets[0])
batch_size = tf.shape(targets[0])[0]
loss = tf.reduce_sum(crossent) / tf.cast(batch_size, tf.float32)
return bucket_outputs, loss
def resize_video_frames(images, size):
resized_images = []
for image in images:
resized_images.append(
tf.to_int64(tf.image.resize_images(
image, [size, size], tf.image.ResizeMethod.BILINEAR)))
return resized_images
def result_dict_for_batched_example(images,
keys,
detections,
groundtruth=None,
class_agnostic=False,
scale_to_absolute=False,
original_image_spatial_shapes=None,
true_image_shapes=None,
max_gt_boxes=None):
"""Merges all detection and groundtruth information for a single example.
Note that evaluation tools require classes that are 1-indexed, and so this
function performs the offset. If `class_agnostic` is True, all output classes
have label 1.
Args:
images: A single 4D uint8 image tensor of shape [batch_size, H, W, C].
keys: A [batch_size] string tensor with image identifier.
detections: A dictionary of detections, returned from
DetectionModel.postprocess().
groundtruth: (Optional) Dictionary of groundtruth items, with fields:
'groundtruth_boxes': [batch_size, max_number_of_boxes, 4] float32 tensor
of boxes, in normalized coordinates.
'groundtruth_classes': [batch_size, max_number_of_boxes] int64 tensor of
1-indexed classes.
'groundtruth_area': [batch_size, max_number_of_boxes] float32 tensor of
bbox area. (Optional)
'groundtruth_is_crowd':[batch_size, max_number_of_boxes] int64
tensor. (Optional)
'groundtruth_difficult': [batch_size, max_number_of_boxes] int64
tensor. (Optional)
'groundtruth_group_of': [batch_size, max_number_of_boxes] int64
tensor. (Optional)
'groundtruth_instance_masks': 4D int64 tensor of instance
masks (Optional).
class_agnostic: Boolean indicating whether the detections are class-agnostic
(i.e. binary). Default False.
scale_to_absolute: Boolean indicating whether boxes and keypoints should be
scaled to absolute coordinates. Note that for IoU based evaluations, it
does not matter whether boxes are expressed in absolute or relative
coordinates. Default False.
original_image_spatial_shapes: A 2D int32 tensor of shape [batch_size, 2]
used to resize the image. When set to None, the image size is retained.
true_image_shapes: A 2D int32 tensor of shape [batch_size, 3]
containing the size of the unpadded original_image.
max_gt_boxes: [batch_size] tensor representing the maximum number of
groundtruth boxes to pad.
Returns:
A dictionary with:
'original_image': A [batch_size, H, W, C] uint8 image tensor.
'original_image_spatial_shape': A [batch_size, 2] tensor containing the
original image sizes.
'true_image_shape': A [batch_size, 3] tensor containing the size of
the unpadded original_image.
'key': A [batch_size] string tensor with image identifier.
'detection_boxes': [batch_size, max_detections, 4] float32 tensor of boxes,
in normalized or absolute coordinates, depending on the value of
`scale_to_absolute`.
'detection_scores': [batch_size, max_detections] float32 tensor of scores.
'detection_classes': [batch_size, max_detections] int64 tensor of 1-indexed
classes.
'detection_masks': [batch_size, max_detections, H, W] float32 tensor of
binarized masks, reframed to full image masks.
'num_detections': [batch_size] int64 tensor containing number of valid
detections.
'groundtruth_boxes': [batch_size, num_boxes, 4] float32 tensor of boxes, in
normalized or absolute coordinates, depending on the value of
`scale_to_absolute`. (Optional)
'groundtruth_classes': [batch_size, num_boxes] int64 tensor of 1-indexed
classes. (Optional)
'groundtruth_area': [batch_size, num_boxes] float32 tensor of bbox
area. (Optional)
'groundtruth_is_crowd': [batch_size, num_boxes] int64 tensor. (Optional)
'groundtruth_difficult': [batch_size, num_boxes] int64 tensor. (Optional)
'groundtruth_group_of': [batch_size, num_boxes] int64 tensor. (Optional)
'groundtruth_instance_masks': 4D int64 tensor of instance masks
(Optional).
'num_groundtruth_boxes': [batch_size] tensor containing the maximum number
of groundtruth boxes per image.
Raises:
ValueError: if original_image_spatial_shape is not 2D int32 tensor of shape
[2].
ValueError: if true_image_shapes is not 2D int32 tensor of shape
[3].
"""
label_id_offset = 1 # Applying label id offset (b/63711816)
input_data_fields = fields.InputDataFields
if original_image_spatial_shapes is None:
original_image_spatial_shapes = tf.tile(
tf.expand_dims(tf.shape(images)[1:3], axis=0),
multiples=[tf.shape(images)[0], 1])
else:
if (len(original_image_spatial_shapes.shape) != 2
and original_image_spatial_shapes.shape[1] != 2):
raise ValueError(
'`original_image_spatial_shape` should be a 2D tensor of shape '
'[batch_size, 2].')
if true_image_shapes is None:
true_image_shapes = tf.tile(tf.expand_dims(tf.shape(images)[1:4],
axis=0),
multiples=[tf.shape(images)[0], 1])
else:
if (len(true_image_shapes.shape) != 2
and true_image_shapes.shape[1] != 3):
raise ValueError('`true_image_shapes` should be a 2D tensor of '
'shape [batch_size, 3].')
output_dict = {
input_data_fields.original_image:
images,
input_data_fields.key:
keys,
input_data_fields.original_image_spatial_shape:
(original_image_spatial_shapes),
input_data_fields.true_image_shape:
true_image_shapes
}
detection_fields = fields.DetectionResultFields
detection_boxes = detections[detection_fields.detection_boxes]
detection_scores = detections[detection_fields.detection_scores]
num_detections = tf.to_int32(detections[detection_fields.num_detections])
if class_agnostic:
detection_classes = tf.ones_like(detection_scores, dtype=tf.int64)
else:
detection_classes = (
tf.to_int64(detections[detection_fields.detection_classes]) +
label_id_offset)
if scale_to_absolute:
output_dict[detection_fields.detection_boxes] = (
shape_utils.static_or_dynamic_map_fn(
_scale_box_to_absolute,
elems=[detection_boxes, original_image_spatial_shapes],
dtype=tf.float32))
else:
output_dict[detection_fields.detection_boxes] = detection_boxes
output_dict[detection_fields.detection_classes] = detection_classes
output_dict[detection_fields.detection_scores] = detection_scores
output_dict[detection_fields.num_detections] = num_detections
if detection_fields.detection_masks in detections:
detection_masks = detections[detection_fields.detection_masks]
# TODO(rathodv): This should be done in model's postprocess
# function ideally.
output_dict[detection_fields.detection_masks] = (
shape_utils.static_or_dynamic_map_fn(
_resize_detection_masks,
elems=[
detection_boxes, detection_masks,
original_image_spatial_shapes
],
dtype=tf.uint8))
if detection_fields.detection_keypoints in detections:
detection_keypoints = detections[detection_fields.detection_keypoints]
output_dict[detection_fields.detection_keypoints] = detection_keypoints
if scale_to_absolute:
output_dict[detection_fields.detection_keypoints] = (
shape_utils.static_or_dynamic_map_fn(
_scale_keypoint_to_absolute,
elems=[detection_keypoints, original_image_spatial_shapes],
dtype=tf.float32))
if groundtruth:
if max_gt_boxes is None:
if input_data_fields.num_groundtruth_boxes in groundtruth:
max_gt_boxes = groundtruth[
input_data_fields.num_groundtruth_boxes]
else:
raise ValueError(
'max_gt_boxes must be provided when processing batched examples.'
)
if input_data_fields.groundtruth_instance_masks in groundtruth:
masks = groundtruth[input_data_fields.groundtruth_instance_masks]
groundtruth[input_data_fields.groundtruth_instance_masks] = (
shape_utils.static_or_dynamic_map_fn(
_resize_groundtruth_masks,
elems=[masks, original_image_spatial_shapes],
dtype=tf.uint8))
output_dict.update(groundtruth)
if scale_to_absolute:
groundtruth_boxes = groundtruth[
input_data_fields.groundtruth_boxes]
output_dict[input_data_fields.groundtruth_boxes] = (
shape_utils.static_or_dynamic_map_fn(
_scale_box_to_absolute,
elems=[groundtruth_boxes, original_image_spatial_shapes],
dtype=tf.float32))
# For class-agnostic models, groundtruth classes all become 1.
if class_agnostic:
groundtruth_classes = groundtruth[
input_data_fields.groundtruth_classes]
groundtruth_classes = tf.ones_like(groundtruth_classes,
dtype=tf.int64)
output_dict[
input_data_fields.groundtruth_classes] = groundtruth_classes
output_dict[input_data_fields.num_groundtruth_boxes] = max_gt_boxes
return output_dict
def to_sparse(tensor, lengths, max_length):
mask = tf.sequence_mask(lengths, max_length)
indices = tf.to_int64(tf.where(tf.equal(mask, True)))
values = tf.to_int32(tf.boolean_mask(tensor, mask))
shape = tf.to_int64(tf.shape(tensor))
return tf.SparseTensor(indices, values, shape)
def main(args):
tf.reset_default_graph()
train_or_test = args.train_or_test
num_epoches = 10
face_feature_len = 4096
attention_feature_len = 2048
scene_feature_len = 1024
num_class = 3
max_attention_nodes = 16
if train_or_test == 'train':
max_face_nodes = 16
else:
max_face_nodes = 48
X1 = tf.placeholder("float", [None, face_feature_len])
X2 = tf.placeholder("float", [None, attention_feature_len])
X3 = tf.placeholder("float", [None, scene_feature_len])
Y = tf.placeholder("float", [None])
dropout_flag = tf.placeholder_with_default(0.5, shape=())
node_len = tf.placeholder("int32", shape=())
net= GNN()
with tf.name_scope("my_model"):
net.build(
features1=X1,
features2=X2,
features3=X3,
face_feature_size=face_feature_len,
attention_feature_size=attention_feature_len,
scene_feature_size=scene_feature_len,
hidden_size=128,
edge_features_length=128,
layer_num=1,
num_face_nodes=node_len,
num_attention_nodes=max_attention_nodes,
use_bias=False,
keep_prob=dropout_flag,
num_classes=num_class,
num_steps=4,
)
print '\ntrainable variables'
for v in tf.trainable_variables():
print v.name, v.get_shape().as_list()
learning_rate = 0.0001
corr = tf.equal(tf.argmax(net.probs, 1), tf.to_int64(Y))
accuracy = tf.reduce_mean(tf.cast(corr, tf.float32))
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=net.logits, labels=tf.to_int64(Y)))
train_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)
init = tf.global_variables_initializer()
saver = tf.train.Saver(max_to_keep=20)
with tf.Session() as sess:
print("start")
sess.run(init)
org_path = args.org_path
face_path = args.face_feature_path
attention_path = args.object_feature_path
scene_path = args.scene_feature_path
(
face_whole_list,
attention_whole_list,
scene_whole_list,
label_whole_list,
) = read_data(
org_path,
face_path,
attention_path,
scene_path
)
num_samples = len(face_whole_list)
#if train_or_test in {'test', 'val'}:
# np.save(os.path.join(args.model_path, train_or_test + 'labels.npy', label_list))
for epoch in range(num_epoches):
count=0
if train_or_test == 'train':
(
face_whole_list,
attention_whole_list,
scene_whole_list,
label_whole_list,
) = shuffle_list(
face_whole_list,
attention_whole_list,
scene_whole_list,
label_whole_list
)
if not os.path.exists(args.model_path):
os.makedirs(args.model_path)
else:
saver.restore(sess, os.path.join(args.model_path, "model_epoch"+str(epoch+1)+".ckpt"))
probs_whole = []
while count<num_samples:
#face
batch_face_list= face_whole_list[count]
if batch_face_list!='':
face_list = os.listdir(batch_face_list)
face_nodes_len = len(face_list)
batch_face_x = []
if train_or_test == 'train' and face_nodes_len> max_face_nodes:
shuffle(face_list)
face_nodes_len = face_nodes_len if face_nodes_len<max_face_nodes else max_face_nodes
for j in range(face_nodes_len):
batch_face_x.append(
np.reshape(
np.load(os.path.join(batch_face_list, face_list[j])),
[face_feature_len,]
)
)
else:
face_nodes_len=0
batch_face_x = np.zeros((1, face_feature_len))
#attention
batch_attention_list = attention_whole_list[count]
attention_list = os.listdir(batch_attention_list)
assert len(attention_list) == max_attention_nodes
attention_nodes_len = max_attention_nodes
batch_attention_x = []
for j in range(attention_nodes_len):
batch_attention_x.append(
np.reshape(
np.load(os.path.join(batch_attention_list, attention_list[j])),
[attention_feature_len,]
)
)
batch_scene_list = scene_whole_list[count]
batch_scene_x = np.reshape(np.load(batch_scene_list+'.npy'), [1, scene_feature_len])
batch_y = np.repeat(label_whole_list[count], face_nodes_len+attention_nodes_len+1, axis=0)
if train_or_test == 'train':
sess.run(train_op,
feed_dict = {
X1:batch_face_x,
X2:batch_attention_x,
X3:batch_scene_x,
Y:batch_y,
dropout_flag:0.5,
node_len:face_nodes_len,
}
)
else:
probs_out = sess.run(net.probs,
feed_dict = {
X1:batch_face_x,
X2:batch_attention_x,
X3:batch_scene_x,
Y:batch_y,
dropout_flag:1.0,
node_len:face_nodes_len,
}
)
probs = np.mean(probs_out, axis=0)
probs_whole.append(probs)
count+=1
if train_or_test == 'train' and count % (500)==0:
train_accuracy = sess.run(
accuracy,
feed_dict = {
X1:batch_face_x,
X2:batch_attention_x,
X3:batch_scene_x,
Y:batch_y,
dropout_flag:1.0,
node_len:face_nodes_len,
}
)
print(" Step %d, training accuracy %f" % (count, train_accuracy))
if train_or_test == 'train':
model_name = os.path.join(args.model_path, "model_epoch"+str(epoch+1)+".ckpt")
saver.save(sess, model_name)
else:
assert len(probs_whole)==num_samples
#probs_whole=probs_whole.reshape((len(probs_whole),3))
predicts = np.argmax(probs_whole, 1)
#np.save(os.path.join(args.model_path, "model_epoch"+str(epoch+1)+'.npy'), probs_whole)
print "Eopch " + str(epoch+1) + " accuracy is %f" % accuracy_score(label_whole_list, predicts)
def train():
with tf.Graph().as_default():
with tf.device('/gpu:'+str(GPU_INDEX)):
pointclouds_pl, labels_pl = MODEL.placeholder_inputs(BATCH_SIZE, NUM_POINT)
is_training_pl = tf.placeholder(tf.bool, shape=())
# Note the global_step=batch parameter to minimize.
# That tells the optimizer to helpfully increment the 'batch' parameter for you every time it trains.
batch = tf.Variable(0)
bn_decay = get_bn_decay(batch)
tf.summary.scalar('bn_decay', bn_decay)
# Get model and loss
pred, end_points = MODEL.get_model(pointclouds_pl, is_training_pl, bn_decay=bn_decay)
loss = MODEL.get_loss(pred, labels_pl, end_points)
tf.summary.scalar('loss', loss)
correct = tf.equal(tf.argmax(pred, 1), tf.to_int64(labels_pl))
accuracy = tf.reduce_sum(tf.cast(correct, tf.float32)) / float(BATCH_SIZE)
tf.summary.scalar('accuracy', accuracy)
# Get training operator
learning_rate = get_learning_rate(batch)
tf.summary.scalar('learning_rate', learning_rate)
if OPTIMIZER == 'momentum':
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=MOMENTUM)
elif OPTIMIZER == 'adam':
optimizer = tf.train.AdamOptimizer(learning_rate)
train_op = optimizer.minimize(loss, global_step=batch)
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess = tf.Session(config=config)
# Add summary writers
#merged = tf.merge_all_summaries()
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(os.path.join(LOG_DIR, 'train'),
sess.graph)
test_writer = tf.summary.FileWriter(os.path.join(LOG_DIR, 'test'))
# Init variables
init = tf.global_variables_initializer()
# To fix the bug introduced in TF 0.12.1 as in
# http://stackoverflow.com/questions/41543774/invalidargumenterror-for-tensor-bool-tensorflow-0-12-1
#sess.run(init)
sess.run(init, {is_training_pl: True})
ops = {'pointclouds_pl': pointclouds_pl,
'labels_pl': labels_pl,
'is_training_pl': is_training_pl,
'pred': pred,
'loss': loss,
'train_op': train_op,
'merged': merged,
'step': batch}
for epoch in range(MAX_EPOCH):
log_string('**** EPOCH %03d ****' % (epoch))
sys.stdout.flush()
train_one_epoch(sess, ops, train_writer)
eval_one_epoch(sess, ops, test_writer)
# Save the variables to disk.
if epoch % 10 == 0:
save_path = saver.save(sess, os.path.join(LOG_DIR, "model.ckpt"))
log_string("Model saved in file: %s" % save_path)
def main(_):
if len(sys.argv) < 2 or sys.argv[-1].startswith('-'):
print('Usage: mnist_saved_model.py [--training_iteration=x] '
'[--model_version=y] export_dir')
sys.exit(-1)
if FLAGS.training_iteration <= 0:
print('Please specify a positive value for training iteration.')
sys.exit(-1)
if FLAGS.model_version <= 0:
print('Please specify a positive value for version number.')
sys.exit(-1)
# Train model
print('Training model...')
mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True)
sess = tf.InteractiveSession()
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
x = tf.identity(tf_example['x'],
name='x') # use tf.identity() to assign name
y_ = tf.placeholder('float', shape=[None, 10])
w = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
sess.run(tf.global_variables_initializer())
y = tf.nn.softmax(tf.matmul(x, w) + b, name='y')
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(
cross_entropy)
values, indices = tf.nn.top_k(y, 10)
table = tf.contrib.lookup.index_to_string_table_from_tensor(
tf.constant([str(i) for i in range(10)]))
prediction_classes = table.lookup(tf.to_int64(indices))
for _ in range(FLAGS.training_iteration):
batch = mnist.train.next_batch(50)
train_step.run(feed_dict={x: batch[0], y_: batch[1]})
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
print('training accuracy %g' % sess.run(accuracy,
feed_dict={
x: mnist.test.images,
y_: mnist.test.labels
}))
print('Done training!')
# Export model
# WARNING(break-tutorial-inline-code): The following code snippet is
# in-lined in tutorials, please update tutorial documents accordingly
# whenever code changes.
export_path_base = sys.argv[-1]
export_path = os.path.join(tf.compat.as_bytes(export_path_base),
tf.compat.as_bytes(str(FLAGS.model_version)))
print('Exporting trained model to', export_path)
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
# Build the signature_def_map.
classification_inputs = tf.saved_model.utils.build_tensor_info(
serialized_tf_example)
classification_outputs_classes = tf.saved_model.utils.build_tensor_info(
prediction_classes)
classification_outputs_scores = tf.saved_model.utils.build_tensor_info(
values)
classification_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
tf.saved_model.signature_constants.CLASSIFY_INPUTS:
classification_inputs
},
outputs={
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES:
classification_outputs_classes,
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_SCORES:
classification_outputs_scores
},
method_name=tf.saved_model.signature_constants.CLASSIFY_METHOD_NAME
))
tensor_info_x = tf.saved_model.utils.build_tensor_info(x)
tensor_info_y = tf.saved_model.utils.build_tensor_info(y)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'images': tensor_info_x},
outputs={'scores': tensor_info_y},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)
)
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'predict_images':
prediction_signature,
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
classification_signature,
},
main_op=tf.tables_initializer(),
strip_default_attrs=True)
builder.save()
print('Done exporting!')
def export():
# Create index->synset mapping
synsets = []
with open(SYNSET_FILE) as f:
synsets = f.read().splitlines()
# Create synset->metadata mapping
texts = {}
with open(METADATA_FILE) as f:
for line in f.read().splitlines():
parts = line.split('\t')
assert len(parts) == 2
texts[parts[0]] = parts[1]
with tf.Graph().as_default():
# Build inference model.
# Please refer to Tensorflow inception model for details.
# Input transformation.
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
'image/encoded': tf.FixedLenFeature(
shape=[], dtype=tf.string),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
jpegs = tf_example['image/encoded']
images = tf.map_fn(preprocess_image, jpegs, dtype=tf.float32)
# Run inference.
logits, _ = inception_model.inference(images, NUM_CLASSES + 1)
# Transform output to topK result.
values, indices = tf.nn.top_k(logits, NUM_TOP_CLASSES)
# Create a constant string Tensor where the i'th element is
# the human readable class description for the i'th index.
# Note that the 0th index is an unused background class
# (see inception model definition code).
class_descriptions = ['unused background']
for s in synsets:
class_descriptions.append(texts[s])
class_tensor = tf.constant(class_descriptions)
table = tf.contrib.lookup.index_to_string_table_from_tensor(class_tensor)
classes = table.lookup(tf.to_int64(indices))
# Restore variables from training checkpoint.
variable_averages = tf.train.ExponentialMovingAverage(
inception_model.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
with tf.Session() as sess:
# Restore variables from training checkpoints.
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/imagenet_train/model.ckpt-0,
# extract global_step from it.
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
print 'Successfully loaded model from %s at step=%s.' % (
ckpt.model_checkpoint_path, global_step)
else:
print 'No checkpoint file found at %s' % FLAGS.checkpoint_dir
return
# Export inference model.
output_path = os.path.join(
tf.compat.as_bytes(FLAGS.output_dir),
tf.compat.as_bytes(str(FLAGS.model_version)))
print 'Exporting trained model to', output_path
builder = tf.saved_model.builder.SavedModelBuilder(output_path)
# Build the signature_def_map.
classify_inputs_tensor_info = tf.saved_model.utils.build_tensor_info(
serialized_tf_example)
classes_output_tensor_info = tf.saved_model.utils.build_tensor_info(
classes)
scores_output_tensor_info = tf.saved_model.utils.build_tensor_info(values)
classification_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
tf.saved_model.signature_constants.CLASSIFY_INPUTS:
classify_inputs_tensor_info
},
outputs={
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES:
classes_output_tensor_info,
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_SCORES:
scores_output_tensor_info
},
method_name=tf.saved_model.signature_constants.
CLASSIFY_METHOD_NAME))
predict_inputs_tensor_info = tf.saved_model.utils.build_tensor_info(jpegs)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'images': predict_inputs_tensor_info},
outputs={
'classes': classes_output_tensor_info,
'scores': scores_output_tensor_info
},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME
))
legacy_init_op = tf.group(
tf.tables_initializer(), name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'predict_images':
prediction_signature,
tf.saved_model.signature_constants.
DEFAULT_SERVING_SIGNATURE_DEF_KEY:
classification_signature,
},
legacy_init_op=legacy_init_op)
builder.save()
print 'Successfully exported model to %s' % FLAGS.output_dir
def __init__(self,
hparams,
mode,
iterator,
source_vocab_table,
target_vocab_table,
reverse_target_vocab_table=None,
scope=None,
extra_args=None):
"""Create the model.
Args:
hparams: Hyperparameter configurations.
mode: TRAIN | EVAL | INFER
iterator: Dataset Iterator that feeds data.
source_vocab_table: Lookup table mapping source words to ids.
target_vocab_table: Lookup table mapping target words to ids.
reverse_target_vocab_table: Lookup table mapping ids to target words. Only
required in INFER mode. Defaults to None.
scope: scope of the model.
extra_args: model_helper.ExtraArgs, for passing customizable functions.
"""
assert isinstance(iterator, iterator_utils.BatchedInput)
self.iterator = iterator
self.mode = mode
self.src_vocab_table = source_vocab_table
self.tgt_vocab_table = target_vocab_table
self.src_vocab_size = hparams.src_vocab_size
self.tgt_vocab_size = hparams.tgt_vocab_size
self.num_layers = hparams.num_layers
self.num_gpus = hparams.num_gpus
self.time_major = hparams.time_major
# extra_args: to make it flexible for adding external customizable code
self.single_cell_fn = None
if extra_args:
self.single_cell_fn = extra_args.single_cell_fn
# Initializer
initializer = model_helper.get_initializer(
hparams.init_op, hparams.random_seed, hparams.init_weight)
tf.get_variable_scope().set_initializer(initializer)
# Embeddings
# TODO(ebrevdo): Only do this if the mode is TRAIN?
self.init_embeddings(hparams, scope)
self.batch_size = tf.size(self.iterator.source_sequence_length)
# Projection
with tf.variable_scope(scope or "build_network"):
with tf.variable_scope("decoder/output_projection"):
self.output_layer = layers_core.Dense(
hparams.tgt_vocab_size, use_bias=False, name="output_projection")
## Train graph
res = self.build_graph(hparams, scope=scope)
if self.mode == tf.contrib.learn.ModeKeys.TRAIN:
self.train_loss = res[1]
self.word_count = tf.reduce_sum(
self.iterator.source_sequence_length) + tf.reduce_sum(
self.iterator.target_sequence_length)
elif self.mode == tf.contrib.learn.ModeKeys.EVAL:
self.eval_loss = res[1]
elif self.mode == tf.contrib.learn.ModeKeys.INFER:
self.infer_logits, _, self.final_context_state, self.sample_id = res
self.sample_words = reverse_target_vocab_table.lookup(
tf.to_int64(self.sample_id))
if self.mode != tf.contrib.learn.ModeKeys.INFER:
## Count the number of predicted words for compute ppl.
self.predict_count = tf.reduce_sum(
self.iterator.target_sequence_length)
## Learning rate
warmup_steps = hparams.learning_rate_warmup_steps
warmup_factor = hparams.learning_rate_warmup_factor
print(" start_decay_step=%d, learning_rate=%g, decay_steps %d, "
"decay_factor %g, learning_rate_warmup_steps=%d, "
"learning_rate_warmup_factor=%g, starting_learning_rate=%g" %
(hparams.start_decay_step, hparams.learning_rate, hparams.decay_steps,
hparams.decay_factor, warmup_steps, warmup_factor,
(hparams.learning_rate * warmup_factor ** warmup_steps)))
self.global_step = tf.Variable(0, trainable=False)
params = tf.trainable_variables()
# Gradients and SGD update operation for training the model.
# Arrage for the embedding vars to appear at the beginning.
if self.mode == tf.contrib.learn.ModeKeys.TRAIN:
self.learning_rate = tf.constant(hparams.learning_rate)
# Apply inverse decay if global steps less than warmup steps.
# Inspired by https://arxiv.org/pdf/1706.03762.pdf (Section 5.3)
# When step < warmup_steps,
# learing_rate *= warmup_factor ** (warmup_steps - step)
inv_decay = warmup_factor**(
tf.to_float(warmup_steps - self.global_step))
self.learning_rate = tf.cond(
self.global_step < hparams.learning_rate_warmup_steps,
lambda: inv_decay * self.learning_rate,
lambda: self.learning_rate,
name="learning_rate_decay_warump_cond")
if hparams.optimizer == "sgd":
self.learning_rate = tf.cond(
self.global_step < hparams.start_decay_step,
lambda: self.learning_rate,
lambda: tf.train.exponential_decay(
self.learning_rate,
(self.global_step - hparams.start_decay_step),
hparams.decay_steps,
hparams.decay_factor,
staircase=True),
name="learning_rate")
opt = tf.train.GradientDescentOptimizer(self.learning_rate)
tf.summary.scalar("lr", self.learning_rate)
elif hparams.optimizer == "adam":
assert float(
hparams.learning_rate
) <= 0.001, "! High Adam learning rate %g" % hparams.learning_rate
opt = tf.train.AdamOptimizer(self.learning_rate)
gradients = tf.gradients(
self.train_loss,
params,
colocate_gradients_with_ops=hparams.colocate_gradients_with_ops)
clipped_gradients, gradient_norm_summary = model_helper.gradient_clip(
gradients, max_gradient_norm=hparams.max_gradient_norm)
self.update = opt.apply_gradients(
zip(clipped_gradients, params), global_step=self.global_step)
# Summary
self.train_summary = tf.summary.merge([
tf.summary.scalar("lr", self.learning_rate),
tf.summary.scalar("train_loss", self.train_loss),
] + gradient_norm_summary)
if self.mode == tf.contrib.learn.ModeKeys.INFER:
self.infer_summary = self._get_infer_summary(hparams)
# Saver
self.saver = tf.train.Saver(tf.global_variables())
# Print trainable variables
utils.print_out("# Trainable variables")
for param in params:
utils.print_out(" %s, %s, %s" % (param.name, str(param.get_shape()),
param.op.device))
def accurracy(labels, logits):
prediction = tf.to_int64(tf.argmax(logits, 1))
correct_prediction = tf.equal(prediction, tf.argmax(labels, 1))
accurracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
return accurracy
def attention_decoder(decoder_inputs, initial_state, attention_states, encoders, decoder, encoder_input_length,
feed_previous=0.0, align_encoder_id=0, feed_argmax=True, sim_score=0.0, **kwargs):
"""
:param decoder_inputs: int32 tensor of shape (batch_size, output_length)
:param initial_state: initial state of the decoder (usually the final state of the encoder),
as a float32 tensor of shape (batch_size, initial_state_size). This state is mapped to the
correct state size for the decoder.
:param attention_states: list of tensors of shape (batch_size, input_length, encoder_cell_size),
the hidden states of the encoder(s) (one tensor for each encoder).
:param encoders: configuration of the encoders
:param decoder: configuration of the decoder
:param encoder_input_length: list of int32 tensors of shape (batch_size,), tells for each encoder,
the true length of each sequence in the batch (sequences in the same batch are padded to all have the same
length).
:param feed_previous: scalar tensor corresponding to the probability to use previous decoder output
instead of the ground truth as input for the decoder (1 when decoding, between 0 and 1 when training)
:param feed_argmax: boolean tensor, when True the greedy decoder outputs the word with the highest
probability (argmax). When False, it samples a word from the probability distribution (softmax).
:param align_encoder_id: outputs attention weights for this encoder. Also used when predicting edit operations
(pred_edits), to specifify which encoder reads the sequence to post-edit (MT).
:return:
outputs of the decoder as a tensor of shape (batch_size, output_length, decoder_cell_size)
attention weights as a tensor of shape (output_length, encoders, batch_size, input_length)
"""
assert not decoder.pred_maxout_layer or decoder.cell_size % 2 == 0, 'cell size must be a multiple of 2'
if decoder.use_lstm is False:
decoder.cell_type = 'GRU'
embedding_shape = [decoder.vocab_size, decoder.embedding_size]
if decoder.embedding_initializer == 'sqrt3':
initializer = tf.random_uniform_initializer(-math.sqrt(3), math.sqrt(3))
else:
initializer = None
device = '/cpu:0' if decoder.embeddings_on_cpu else None
if decoder.share_emb is None:
with tf.device(device):
embedding = get_variable('embedding_{}'.format(decoder.name), shape=embedding_shape, initializer=initializer)
else:
with tf.device(device):
embedding = get_variable('embedding_{}'.format(decoder.share_emb), shape=embedding_shape, initializer=initializer)
input_shape = tf.shape(decoder_inputs)
batch_size = input_shape[0]
time_steps = input_shape[1]
scope_name = 'decoder_{}'.format(decoder.name)
scope_name += '/' + '_'.join(encoder.name for encoder in encoders)
def embed(input_):
embedded_input = tf.nn.embedding_lookup(embedding, input_)
input_shape = tf.shape(embedded_input)
batch_size = input_shape[0]
if decoder.use_dropout and decoder.word_keep_prob is not None:
noise_shape = [1, 1] if decoder.pervasive_dropout else [batch_size, 1]
embedded_input = tf.nn.dropout(embedded_input, keep_prob=decoder.word_keep_prob, noise_shape=noise_shape)
if decoder.use_dropout and decoder.embedding_keep_prob is not None:
size = tf.shape(embedded_input)[1]
noise_shape = [1, size] if decoder.pervasive_dropout else [batch_size, size]
embedded_input = tf.nn.dropout(embedded_input, keep_prob=decoder.embedding_keep_prob,
noise_shape=noise_shape)
return embedded_input
def get_cell(input_size=None, reuse=False):
cells = []
for j in range(decoder.layers):
input_size_ = input_size if j == 0 else decoder.cell_size
if decoder.cell_type.lower() == 'lstm':
cell = CellWrapper(BasicLSTMCell(decoder.cell_size, reuse=reuse))
elif decoder.cell_type.lower() == 'dropoutgru':
cell = DropoutGRUCell(decoder.cell_size, reuse=reuse, layer_norm=decoder.layer_norm,
input_size=input_size_, input_keep_prob=decoder.rnn_input_keep_prob,
state_keep_prob=decoder.rnn_state_keep_prob)
else:
cell = GRUCell(decoder.cell_size, reuse=reuse, layer_norm=decoder.layer_norm)
if decoder.use_dropout and decoder.cell_type.lower() != 'dropoutgru':
cell = DropoutWrapper(cell, input_keep_prob=decoder.rnn_input_keep_prob,
output_keep_prob=decoder.rnn_output_keep_prob,
state_keep_prob=decoder.rnn_state_keep_prob,
variational_recurrent=decoder.pervasive_dropout,
dtype=tf.float32, input_size=input_size_)
cells.append(cell)
if len(cells) == 1:
return cells[0]
else:
return CellWrapper(MultiRNNCell(cells))
def look(state, input_, prev_weights=None, pos=None):
prev_weights_ = [prev_weights if i == align_encoder_id else None for i in range(len(encoders))]
pos_ = None
if decoder.pred_edits:
pos_ = [pos if i == align_encoder_id else None for i in range(len(encoders))]
if decoder.attn_prev_word:
state = tf.concat([state, input_], axis=1)
parameters = dict(hidden_states=attention_states, encoder_input_length=encoder_input_length,
encoders=encoders, aggregation_method=decoder.aggregation_method, sim_score=sim_score)
context, new_weights = multi_attention(state, pos=pos_, prev_weights=prev_weights_, **parameters)
if decoder.context_mapping:
with tf.variable_scope(scope_name):
activation = tf.nn.tanh if decoder.context_mapping_activation == 'tanh' else None
use_bias = not decoder.context_mapping_no_bias
context = dense(context, decoder.context_mapping, use_bias=use_bias, activation=activation,
name='context_mapping')
return context, new_weights[align_encoder_id]
def update(state, input_, context=None, symbol=None):
if context is not None and decoder.rnn_feed_attn:
input_ = tf.concat([input_, context], axis=1)
input_size = input_.get_shape()[1].value
initializer = CellInitializer(decoder.cell_size) if decoder.orthogonal_init else None
with tf.variable_scope(tf.get_variable_scope(), initializer=initializer):
try:
output, new_state = get_cell(input_size)(input_, state)
except ValueError: # auto_reuse doesn't work with LSTM cells
output, new_state = get_cell(input_size, reuse=True)(input_, state)
if decoder.skip_update and decoder.pred_edits and symbol is not None:
is_del = tf.equal(symbol, utils.DEL_ID)
new_state = tf.where(is_del, state, new_state)
if decoder.cell_type.lower() == 'lstm' and decoder.use_lstm_full_state:
output = new_state
return output, new_state
def update_pos(pos, symbol, max_pos=None):
if not decoder.pred_edits:
return pos
is_keep = tf.equal(symbol, utils.KEEP_ID)
is_del = tf.equal(symbol, utils.DEL_ID)
is_not_ins = tf.logical_or(is_keep, is_del)
pos = beam_search.resize_like(pos, symbol)
max_pos = beam_search.resize_like(max_pos, symbol)
pos += tf.to_float(is_not_ins)
if max_pos is not None:
pos = tf.minimum(pos, tf.to_float(max_pos))
return pos
def generate(state, input_, context):
if decoder.pred_use_lstm_state is False: # for back-compatibility
state = state[:,-decoder.cell_size:]
projection_input = [state, context]
if decoder.use_previous_word:
projection_input.insert(1, input_) # for back-compatibility
output_ = tf.concat(projection_input, axis=1)
if decoder.pred_deep_layer:
deep_layer_size = decoder.pred_deep_layer_size or decoder.embedding_size
if decoder.layer_norm:
output_ = dense(output_, deep_layer_size, use_bias=False, name='deep_output')
output_ = tf.contrib.layers.layer_norm(output_, activation_fn=tf.nn.tanh, scope='output_layer_norm')
else:
output_ = dense(output_, deep_layer_size, activation=tf.tanh, use_bias=True, name='deep_output')
if decoder.use_dropout:
size = tf.shape(output_)[1]
noise_shape = [1, size] if decoder.pervasive_dropout else None
output_ = tf.nn.dropout(output_, keep_prob=decoder.deep_layer_keep_prob, noise_shape=noise_shape)
else:
if decoder.pred_maxout_layer:
maxout_size = decoder.maxout_size or decoder.cell_size
output_ = dense(output_, maxout_size, use_bias=True, name='maxout')
if decoder.old_maxout: # for back-compatibility with old models
output_ = tf.nn.pool(tf.expand_dims(output_, axis=2), window_shape=[2], pooling_type='MAX',
padding='SAME', strides=[2])
output_ = tf.squeeze(output_, axis=2)
else:
output_ = tf.maximum(*tf.split(output_, num_or_size_splits=2, axis=1))
if decoder.pred_embed_proj:
# intermediate projection to embedding size (before projecting to vocabulary size)
# this is useful to reduce the number of parameters, and
# to use the output embeddings for output projection (tie_embeddings parameter)
output_ = dense(output_, decoder.embedding_size, use_bias=False, name='softmax0')
if decoder.tie_embeddings and (decoder.pred_embed_proj or decoder.pred_deep_layer):
bias = get_variable('softmax1/bias', shape=[decoder.vocab_size])
output_ = tf.matmul(output_, tf.transpose(embedding)) + bias
else:
output_ = dense(output_, decoder.vocab_size, use_bias=True, name='softmax1')
return output_
state_size = (decoder.cell_size * 2 if decoder.cell_type.lower() == 'lstm' else decoder.cell_size) * decoder.layers
if decoder.use_dropout:
initial_state = tf.nn.dropout(initial_state, keep_prob=decoder.initial_state_keep_prob)
with tf.variable_scope(scope_name):
if decoder.layer_norm:
initial_state = dense(initial_state, state_size, use_bias=False, name='initial_state_projection')
initial_state = tf.contrib.layers.layer_norm(initial_state, activation_fn=tf.nn.tanh,
scope='initial_state_layer_norm')
else:
initial_state = dense(initial_state, state_size, use_bias=True, name='initial_state_projection',
activation=tf.nn.tanh)
if decoder.cell_type.lower() == 'lstm' and decoder.use_lstm_full_state:
initial_output = initial_state
else:
initial_output = initial_state[:, -decoder.cell_size:]
time = tf.constant(0, dtype=tf.int32, name='time')
outputs = tf.TensorArray(dtype=tf.float32, size=time_steps)
samples = tf.TensorArray(dtype=tf.int64, size=time_steps)
inputs = tf.TensorArray(dtype=tf.int64, size=time_steps).unstack(tf.to_int64(tf.transpose(decoder_inputs)))
states = tf.TensorArray(dtype=tf.float32, size=time_steps)
weights = tf.TensorArray(dtype=tf.float32, size=time_steps)
attns = tf.TensorArray(dtype=tf.float32, size=time_steps)
initial_symbol = inputs.read(0) # first symbol is BOS
initial_input = embed(initial_symbol)
initial_pos = tf.zeros([batch_size], tf.float32)
initial_weights = tf.zeros(tf.shape(attention_states[align_encoder_id])[:2])
with tf.variable_scope('decoder_{}'.format(decoder.name)):
initial_context, _ = look(initial_output, initial_input, pos=initial_pos, prev_weights=initial_weights)
initial_data = tf.concat([initial_state, initial_context, tf.expand_dims(initial_pos, axis=1), initial_weights],
axis=1)
context_size = initial_context.shape[1].value
def get_logits(state, ids, time): # for beam-search decoding
with tf.variable_scope('decoder_{}'.format(decoder.name)):
state, context, pos, prev_weights = tf.split(state, [state_size, context_size, 1, -1], axis=1)
input_ = embed(ids)
pos = tf.squeeze(pos, axis=1)
pos = tf.cond(tf.equal(time, 0),
lambda: pos,
lambda: update_pos(pos, ids, encoder_input_length[align_encoder_id]))
if decoder.cell_type.lower() == 'lstm' and decoder.use_lstm_full_state:
output = state
else:
# output is always the right-most part of state. However, this only works at test time,
# because different dropout operations can be used on state and output.
output = state[:, -decoder.cell_size:]
if decoder.conditional_rnn:
with tf.variable_scope('conditional_1'):
output, state = update(state, input_)
elif decoder.update_first:
output, state = update(state, input_, None, ids)
elif decoder.generate_first:
output, state = tf.cond(tf.equal(time, 0),
lambda: (output, state),
lambda: update(state, input_, context, ids))
context, new_weights = look(output, input_, pos=pos, prev_weights=prev_weights)
if decoder.conditional_rnn:
with tf.variable_scope('conditional_2'):
output, state = update(state, context)
elif not decoder.generate_first:
output, state = update(state, input_, context, ids)
logits = generate(output, input_, context)
pos = tf.expand_dims(pos, axis=1)
state = tf.concat([state, context, pos, new_weights], axis=1)
return state, logits
def _time_step(time, input_, input_symbol, pos, state, output, outputs, states, weights, attns, prev_weights,
samples):
if decoder.conditional_rnn:
with tf.variable_scope('conditional_1'):
output, state = update(state, input_)
elif decoder.update_first:
output, state = update(state, input_, None, input_symbol)
context, new_weights = look(output, input_, pos=pos, prev_weights=prev_weights)
if decoder.conditional_rnn:
with tf.variable_scope('conditional_2'):
output, state = update(state, context)
elif not decoder.generate_first:
output, state = update(state, input_, context, input_symbol)
output_ = generate(output, input_, context)
argmax = lambda: tf.argmax(output_, 1)
target = lambda: inputs.read(time + 1)
softmax = lambda: tf.squeeze(tf.multinomial(tf.log(tf.nn.softmax(output_)), num_samples=1),
axis=1)
use_target = tf.logical_and(time < time_steps - 1, tf.random_uniform([]) >= feed_previous)
predicted_symbol = tf.case([
(use_target, target),
(tf.logical_not(feed_argmax), softmax)],
default=argmax) # default case is useful for beam-search
predicted_symbol.set_shape([None])
predicted_symbol = tf.stop_gradient(predicted_symbol)
samples = samples.write(time, predicted_symbol)
input_ = embed(predicted_symbol)
pos = update_pos(pos, predicted_symbol, encoder_input_length[align_encoder_id])
attns = attns.write(time, context)
weights = weights.write(time, new_weights)
states = states.write(time, state)
outputs = outputs.write(time, output_)
if not decoder.conditional_rnn and not decoder.update_first and decoder.generate_first:
output, state = update(state, input_, context, predicted_symbol)
return (time + 1, input_, predicted_symbol, pos, state, output, outputs, states, weights, attns, new_weights,
samples)
with tf.variable_scope('decoder_{}'.format(decoder.name)):
_, _, _, new_pos, new_state, _, outputs, states, weights, attns, new_weights, samples = tf.while_loop(
cond=lambda time, *_: time < time_steps,
body=_time_step,
loop_vars=(time, initial_input, initial_symbol, initial_pos, initial_state, initial_output, outputs,
weights, states, attns, initial_weights, samples),
parallel_iterations=decoder.parallel_iterations,
swap_memory=decoder.swap_memory)
outputs = outputs.stack()
weights = weights.stack() # batch_size, encoders, output time, input time
states = states.stack()
attns = attns.stack()
samples = samples.stack()
# put batch_size as first dimension
outputs = tf.transpose(outputs, perm=(1, 0, 2))
weights = tf.transpose(weights, perm=(1, 0, 2))
states = tf.transpose(states, perm=(1, 0, 2))
attns = tf.transpose(attns, perm=(1, 0, 2))
samples = tf.transpose(samples)
return outputs, weights, states, attns, samples, get_logits, initial_data
def _build_graph(dialog, turn_lens, labels, dropout_keep_prob, c):
slu_states = [666] * c.max_dial_len
for t in range(c.max_dial_len):
# FIXME separate into function
reuse_it = True if t > 0 else None
with tf.variable_scope('turn_encoder', reuse=reuse_it):
word_embeddings = tf.get_variable(
'word_embeddings',
initializer=tf.random_uniform(
[c.vocab_size, c.embedding_size], -1.0, 1.0))
embedded_inputs = [
tf.nn.embedding_lookup(
word_embeddings, tf.squeeze(input_, squeeze_dims=[1]))
for input_ in tf.split(1, c.max_turn_len, dialog[:, t, :])
]
dropped_embedded_inputs = [
tf.nn.dropout(i, dropout_keep_prob)
for i in embedded_inputs
]
forward_slu_gru = tf.nn.rnn_cell.GRUCell(
c.rnn_size, input_size=c.embedding_size)
logger.debug('c.embedding_size: %s', c.embedding_size)
logger.debug('dropped_embedded_inputs[0].get_shape(): %s',
dropped_embedded_inputs[0].get_shape())
with tf.variable_scope('forward_slu'):
outputs, last_slu_state = tf.nn.rnn(
cell=forward_slu_gru,
inputs=dropped_embedded_inputs,
dtype=tf.float32,
sequence_length=turn_lens[:, t])
slu_states[t] = last_slu_state
masked_turns = tf.to_int64(
tf.greater(turn_lens, tf.zeros_like(turn_lens)))
logger.debug('masked_turns.get_shape(): %s', masked_turns.get_shape())
dial_len = tf.reduce_sum(masked_turns, 1)
masked_turnsf = tf.to_float(masked_turns)
forward_dst_gru = tf.nn.rnn_cell.GRUCell(
c.rnn_size, input_size=c.rnn_size) # FIXME use different rnn_size
with tf.variable_scope('dialog_state'):
dialog_states, last_dial_state = tf.nn.rnn(
cell=forward_dst_gru,
inputs=slu_states,
dtype=tf.float32,
sequence_length=dial_len)
logitss = [444] * c.max_dial_len
for t in range(c.max_dial_len):
with tf.variable_scope('slot_prediction',
reuse=True if t > 0 else None):
# FIXME better initialization
w_project = tf.get_variable(
'project2labels',
initializer=tf.random_uniform([c.rnn_size, c.labels_size],
-1.0, 1.0)
) # FIXME dynamically change size based on the input not used fixed c.rnn_size
logitss[t] = tf.matmul(dialog_states[t], w_project)
logger.debug('dialog_states[0].get_shape(): %s',
dialog_states[0].get_shape())
logger.debug('w_project.get_shape(): %s', w_project.get_shape())
logits = tf.reshape(
tf.concat(1, logitss),
(np.prod(masked_turns.get_shape().as_list()), c.labels_size))
logger.debug('logits.get_shape(): %s', logits.get_shape())
with tf.variable_scope('loss'):
logger.debug('labels.get_shape(): %s', labels.get_shape())
masked_logits = tf.mul(
logits,
tf.reshape(masked_turnsf,
(np.prod(masked_turnsf.get_shape().as_list()), 1)))
logger.debug(
'masked_logits.get_shape(): %s, masked_logits.dtype %s',
masked_logits.get_shape(), masked_logits.dtype)
labels_vec = tf.reshape(labels, [-1])
xents = tf.nn.sparse_softmax_cross_entropy_with_logits(
masked_logits, labels_vec)
logger.debug('xents.get_shape(): %s, dtype %s', xents.get_shape(),
xents.dtype)
loss = tf.reduce_sum(xents) / tf.reduce_sum(masked_turnsf)
with tf.variable_scope('eval'):
predicts = tf.argmax(masked_logits, 1)
true_count = tf.reduce_sum(
tf.to_int64(tf.equal(predicts, labels_vec)) *
tf.reshape(masked_turns, [-1]))
num_turns = tf.reduce_sum(dial_len)
batch_accuracy = tf.div(tf.to_floa(true_count),
tf.to_float(num_turns))
logger.debug('true_count.get_shape(): %s', true_count.get_shape())
logger.info(
'trainable variables: %s', '\n'.join([
str((v.name, v.get_shape())) for v in tf.trainable_variables()
]))
return (predicts, loss, num_turns, true_count, batch_accuracy)
def _build_similarity_graph(self):
tf.get_variable_scope().reuse_variables()
with tf.name_scope("Inputs"):
# word or phrase we want similarities for
# self._query_word = tf.placeholder(tf.int32, [1], name="q_word")
self._query_phrase = tf.placeholder(tf.int32, [self.max_num_steps, 3], name="q_phrase")
self._query_length = tf.placeholder(tf.int32, [1], name="q_len") # lengths for RNN
self._query_target = tf.placeholder(tf.int32, [1,2], name="q_target")
# words and phrases to compute similarities over
# self._sim_words = tf.placeholder(tf.int32, [None, 1])
self._sim_phrases = tf.placeholder(tf.int32, [None, self.max_num_steps, 3])
self._sim_lengths = tf.placeholder(tf.int32, [None, 1]) # lengths for RNN
self._sim_targets = tf.placeholder(tf.int32, [None, 2])
sim_size = tf.shape(self._sim_lengths)[0]
with tf.name_scope("Embeddings"):
query_phrase_embed = tf.nn.embedding_lookup(self._word_embeddings,
tf.slice(self._query_phrase, [0,0], [-1, 1]))
query_dep_embed = tf.nn.embedding_lookup(self._dependency_embeddings,
tf.slice(self._query_phrase, [0,1], [-1, 1]))
query_pos_embed = tf.nn.embedding_lookup(self._pos_embeddings,
tf.slice(self._query_phrase, [0,2], [-1, 1]))
q_left_target_embed = tf.nn.embedding_lookup(self._left_target_embeddings,
tf.slice(self._query_target, [0,0], [-1, 1]))
q_right_target_embed = tf.nn.embedding_lookup(self._right_target_embeddings,
tf.slice(self._query_target, [0,1], [-1, 1]))
q_target_embed = tf.squeeze(tf.concat(2, [q_left_target_embed, q_right_target_embed]), [1])
# query_word_embed = tf.nn.embedding_lookup(self._word_embeddings, self._query_word)
# query_phrase_embed = tf.nn.embedding_lookup(self._word_embeddings, self._query_phrase)
# sim_word_embed = tf.nn.embedding_lookup(self._word_embeddings, tf.squeeze(self._sim_words, [1]))
sim_phrase_embed = tf.nn.embedding_lookup(self._word_embeddings,
tf.slice(self._sim_phrases, [0, 0, 0], [-1, -1, 1]))
sim_dep_embed = tf.nn.embedding_lookup(self._dependency_embeddings,
tf.slice(self._sim_phrases, [0, 0, 1], [-1, -1, 1]))
sim_pos_embed = tf.nn.embedding_lookup(self._pos_embeddings,
tf.slice(self._sim_phrases, [0, 0, 2], [-1, -1, 1]))
sim_left_target_embeds = tf.nn.embedding_lookup(self._left_target_embeddings,
tf.slice(self._sim_targets, [0,0], [-1, 1]))
sim_right_target_embeds = tf.nn.embedding_lookup(self._right_target_embeddings,
tf.slice(self._sim_targets, [0,1], [-1, 1]))
sim_target_embeds = tf.squeeze(tf.concat(2, [sim_left_target_embeds, sim_right_target_embeds]), [1])
with tf.name_scope("RNN"):
# compute rep of a query phrase
query_phrase = [tf.squeeze(qw, [1]) for qw in tf.split(0, self.max_num_steps, query_phrase_embed)]
query_dep = [tf.squeeze(qd, [1]) for qd in tf.split(0, self.max_num_steps, query_dep_embed)]
query_pos = [tf.squeeze(qd, [1]) for qd in tf.split(0, self.max_num_steps, query_pos_embed)]
# print(query_phrase[0].get_shape(), query_dep[0].get_shape())
query_input = [ tf.concat(1, [qw, qd, qp]) for (qw, qd, qp) in zip(query_phrase, query_dep, query_pos)]
# just words
# query_input = query_phrase
if self.bidirectional:
outs = tf.nn.bidirectional_rnn(self.fwcell, self.bwcell, query_input,
sequence_length=tf.to_int64(self._query_length),
dtype=tf.float32)
# splice out the final forward and backward hidden states since apparently the documentation lies
fw_state = tf.split(1, 2, outs[-1])[0]
bw_state = tf.split(1, 2, outs[0])[1]
query_phrase_state = tf.concat(1, [fw_state, bw_state])
else:
_, query_phrase_state = tf.nn.rnn(self.cell, query_input,
sequence_length=tf.to_int64(self._query_length),
dtype=tf.float32)
# compute reps of similarity phrases
sim_phrases = [tf.squeeze(qw, [1,2]) for qw in tf.split(1, self.max_num_steps, sim_phrase_embed)]
sim_deps = [tf.squeeze(qd, [1,2]) for qd in tf.split(1, self.max_num_steps, sim_dep_embed)]
sim_pos = [tf.squeeze(qp, [1,2]) for qp in tf.split(1, self.max_num_steps, sim_pos_embed)]
sim_input = [ tf.concat(1, [qw, qd, qp]) for (qw, qd, qp) in zip(sim_phrases, sim_deps, sim_pos)]
#jsut words
# sim_input = sim_phrases
if self.bidirectional:
outs = tf.nn.bidirectional_rnn(self.fwcell, self.bwcell, sim_input,
sequence_length=tf.to_int64(tf.squeeze(self._sim_lengths, [1])),
dtype=tf.float32)
# splice out the final forward and backward hidden states since apparently the documentation lies
fw_state = tf.split(1, 2, outs[-1])[0]
bw_state = tf.split(1, 2, outs[0])[1]
sim_phrase_states = tf.concat(1, [fw_state, bw_state])
else:
_, sim_phrase_states = tf.nn.rnn(self.cell, sim_input,
sequence_length=tf.to_int64(tf.squeeze(self._sim_lengths, [1])),
dtype=tf.float32)
with tf.name_scope("Similarities"):
with tf.name_scope("Normalize"):
# query_phrase = tf.nn.l2_normalize(tf.concat(1, [query_phrase_state, q_target_embed]), 1)
query_phrase = tf.nn.l2_normalize(query_phrase_state, 1)
# query_word = tf.nn.l2_normalize(query_word_embed, 1)
# sim_phrases = tf.nn.l2_normalize(tf.concat(1, [sim_phrase_states, sim_target_embeds]), 1)
sim_phrases = tf.nn.l2_normalize(sim_phrase_states, 1)
# sim_word = tf.nn.l2_normalize(sim_word_embed, 1)
with tf.name_scope("Calc_distances"):
# do for words
# print(q)
# query_word_nearby_dist = tf.matmul(query_word, sim_word, transpose_b=True)
# qw_nearby_val, qw_nearby_idx = tf.nn.top_k(query_word_nearby_dist, min(1000, self.vocab_size))
# self.qw_nearby_val = tf.squeeze(qw_nearby_val)
# self.qw_nearby_idx = tf.squeeze(qw_nearby_idx)
# self.qw_nearby_words = tf.squeeze(tf.gather(self._sim_words, qw_nearby_idx))
# do for phrases
query_phrase_nearby_dist = tf.matmul(query_phrase, sim_phrases, transpose_b=True)
qp_nearby_val, qp_nearby_idx = tf.nn.top_k(query_phrase_nearby_dist, min(1000, sim_size))
# self.sanity_check = tf.squeeze(tf.matmul(query_phrase, query_phrase, transpose_b=True))
self.qp_nearby_val = tf.squeeze(qp_nearby_val)
self.qp_nearby_idx = tf.squeeze(qp_nearby_idx)
def build_whole_detection_network(self,
input_img_batch,
gtboxes_batch,
sess=None):
if self.is_training:
# ensure shape is [M, 5]
gtboxes_batch = tf.reshape(gtboxes_batch, [-1, 5])
gtboxes_batch = tf.cast(gtboxes_batch, tf.float32)
img_shape = tf.shape(input_img_batch)
# 1. build base network
feature_to_cropped = self.build_base_network(input_img_batch)
# 2. build rpn
with tf.variable_scope('build_rpn',
regularizer=slim.l2_regularizer(
cfgs.WEIGHT_DECAY)):
rpn_conv3x3 = slim.conv2d(feature_to_cropped,
512, [3, 3],
trainable=self.is_training,
weights_initializer=cfgs.INITIALIZER,
activation_fn=tf.nn.relu,
scope='rpn_conv/3x3')
rpn_cls_score = slim.conv2d(rpn_conv3x3,
self.num_anchors_per_location * 2,
[1, 1],
stride=1,
trainable=self.is_training,
weights_initializer=cfgs.INITIALIZER,
activation_fn=None,
scope='rpn_cls_score')
rpn_box_pred = slim.conv2d(rpn_conv3x3,
self.num_anchors_per_location * 4,
[1, 1],
stride=1,
trainable=self.is_training,
weights_initializer=cfgs.INITIALIZER,
activation_fn=None,
scope='rpn_bbox_pred')
rpn_box_pred = tf.reshape(rpn_box_pred, [-1, 4])
rpn_cls_score = tf.reshape(rpn_cls_score, [-1, 2])
rpn_cls_prob = slim.softmax(rpn_cls_score, scope='rpn_cls_prob')
# 3. generate_anchors
featuremap_height, featuremap_width = tf.shape(
feature_to_cropped)[1], tf.shape(feature_to_cropped)[2]
featuremap_height = tf.cast(featuremap_height, tf.float32)
featuremap_width = tf.cast(featuremap_width, tf.float32)
anchors = anchor_utils.make_anchors(
base_anchor_size=cfgs.BASE_ANCHOR_SIZE_LIST[0],
anchor_scales=cfgs.ANCHOR_SCALES,
anchor_ratios=cfgs.ANCHOR_RATIOS,
featuremap_height=featuremap_height,
featuremap_width=featuremap_width,
stride=cfgs.ANCHOR_STRIDE,
name="make_anchors_forRPN")
# with tf.variable_scope('make_anchors'):
# anchors = anchor_utils.make_anchors(height=featuremap_height,
# width=featuremap_width,
# feat_stride=cfgs.ANCHOR_STRIDE[0],
# anchor_scales=cfgs.ANCHOR_SCALES,
# anchor_ratios=cfgs.ANCHOR_RATIOS, base_size=16
# )
# 4. postprocess rpn proposals. such as: decode, clip, NMS
with tf.variable_scope('postprocess_RPN'):
# rpn_cls_prob = tf.reshape(rpn_cls_score, [-1, 2])
# rpn_cls_prob = slim.softmax(rpn_cls_prob, scope='rpn_cls_prob')
# rpn_box_pred = tf.reshape(rpn_box_pred, [-1, 4])
rois, roi_scores = postprocess_rpn_proposals(
rpn_bbox_pred=rpn_box_pred,
rpn_cls_prob=rpn_cls_prob,
img_shape=img_shape,
anchors=anchors,
is_training=self.is_training)
# rois shape [-1, 4]
# +++++++++++++++++++++++++++++++++++++add img smry+++++++++++++++++++++++++++++++++++++++++++++++++++++++
if self.is_training:
rois_in_img = show_box_in_tensor.draw_boxes_with_scores(
img_batch=input_img_batch, boxes=rois, scores=roi_scores)
tf.summary.image('all_rpn_rois', rois_in_img)
score_gre_05 = tf.reshape(
tf.where(tf.greater_equal(roi_scores, 0.5)), [-1])
score_gre_05_rois = tf.gather(rois, score_gre_05)
score_gre_05_score = tf.gather(roi_scores, score_gre_05)
score_gre_05_in_img = show_box_in_tensor.draw_boxes_with_scores(
img_batch=input_img_batch,
boxes=score_gre_05_rois,
scores=score_gre_05_score)
tf.summary.image('score_greater_05_rois', score_gre_05_in_img)
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
if self.is_training:
with tf.variable_scope('sample_anchors_minibatch'):
rpn_labels, rpn_bbox_targets = \
tf.py_func(
anchor_target_layer,
[gtboxes_batch, img_shape, anchors],
[tf.float32, tf.float32])
rpn_bbox_targets = tf.reshape(rpn_bbox_targets, [-1, 4])
rpn_labels = tf.to_int32(rpn_labels, name="to_int32")
rpn_labels = tf.reshape(rpn_labels, [-1])
self.add_anchor_img_smry(input_img_batch, anchors, rpn_labels)
# --------------------------------------add smry----------------------------------------------------------------
rpn_cls_category = tf.argmax(rpn_cls_prob, axis=1)
kept_rpppn = tf.reshape(tf.where(tf.not_equal(rpn_labels, -1)),
[-1])
rpn_cls_category = tf.gather(rpn_cls_category, kept_rpppn)
acc = tf.reduce_mean(
tf.to_float(
tf.equal(rpn_cls_category,
tf.to_int64(tf.gather(rpn_labels, kept_rpppn)))))
tf.summary.scalar('ACC/rpn_accuracy', acc)
with tf.control_dependencies([rpn_labels]):
with tf.variable_scope('sample_RCNN_minibatch'):
rcnn_rois, rcnn_labels, rcnn_bbox_targets = \
tf.py_func(proposal_target_layer,
[rois, gtboxes_batch, cfgs.FAST_RCNN_IOU_POSITIVE_THRESHOLD[0]],
[tf.float32, tf.float32, tf.float32])
rcnn_rois = tf.reshape(rcnn_rois, [-1, 4])
rcnn_labels = tf.to_int32(rcnn_labels)
rcnn_labels = tf.reshape(rcnn_labels, [-1])
rcnn_bbox_targets = tf.reshape(
rcnn_bbox_targets, [-1, 4 * (cfgs.CLASS_NUM + 1)])
self.add_roi_batch_img_smry(input_img_batch, rcnn_rois,
rcnn_labels, 'rcnn')
else:
rcnn_rois = rois
# -------------------------------------------------------------------------------------------------------------#
# Fast-RCNN #
# -------------------------------------------------------------------------------------------------------------#
# 5. build Fast-RCNN-before1
# rois = tf.Print(rois, [tf.shape(rois)], 'rois shape', summarize=10)
rcnn_bbox_pred, rcnn_cls_score = self.build_fastrcnn(
feature_to_cropped=feature_to_cropped,
rois=rcnn_rois,
img_shape=img_shape)
# bbox_pred shape: [-1, 4*(cls_num+1)].
# cls_score shape： [-1, cls_num+1]
rcnn_cls_prob = slim.softmax(rcnn_cls_score, 'faster_cls_prob')
rcnn_cls_category = tf.argmax(rcnn_cls_prob, axis=1)
# ----------------------------------------------add smry-------------------------------------------------------
if self.is_training:
faster_rcnn_acc = tf.reduce_mean(
tf.to_float(
tf.equal(rcnn_cls_category, tf.to_int64(rcnn_labels))))
tf.summary.scalar('ACC/stage1_fast_acc', faster_rcnn_acc)
# postprocess_fastrcnn
if not self.is_training:
return self.postprocess_fastrcnn(rois=rcnn_rois,
bbox_ppred=rcnn_bbox_pred,
scores=rcnn_cls_prob,
img_shape=img_shape,
sess=sess)
else:
'''
when trian. We need build Loss
'''
loss_dict = self.build_loss(rpn_box_pred=rpn_box_pred,
rpn_bbox_targets=rpn_bbox_targets,
rpn_cls_score=rpn_cls_score,
rpn_labels=rpn_labels,
bbox_pred=rcnn_bbox_pred,
bbox_targets=rcnn_bbox_targets,
cls_score=rcnn_cls_score,
labels=rcnn_labels)
final_bbox, fina_score, final_category = self.postprocess_fastrcnn(
rois=rcnn_rois,
bbox_ppred=rcnn_bbox_pred,
scores=rcnn_cls_prob,
img_shape=img_shape,
sess=sess)
return final_bbox, fina_score, final_category, loss_dict
def _build_forward_graph(self):
# input tensor of zero padded indices to get to max_num_steps
# None allows for variable batch sizes
with tf.name_scope("Inputs"):
self._input_phrases = tf.placeholder(tf.int32, [None, self.max_num_steps, 3]) # [batch_size, w_{1:N}, 2]
self._input_targets = tf.placeholder(tf.int32, [None, 2]) # [batch_size, w_x]
self._input_labels = tf.placeholder(tf.int32, [None, 1]) # [batch_size, cluster_pair] \in num)clusters**2
self._input_lengths = tf.placeholder(tf.int32, [None, 1]) # [batch_size, N] (len of each sequence)
batch_size = tf.shape(self._input_lengths)[0]
self._keep_prob = tf.placeholder(tf.float32)
with tf.name_scope("Embeddings"):
if np.any(self.pretrained_word_embeddings):
self._word_embeddings = tf.Variable(self.pretrained_word_embeddings,name="word_embeddings")
self._left_target_embeddings = tf.Variable(self.pretrained_word_embeddings, name="left_target_embeddings")
self._right_target_embeddings = tf.Variable(self.pretrained_word_embeddings, name="right_target_embeddings")
else:
self._word_embeddings = tf.get_variable("word_embeddings",
[self.vocab_size, self.word_embed_size],
initializer=self.word_initializer,
dtype=tf.float32)
self._left_target_embeddings = tf.get_variable("left_target_embeddings",
[self.vocab_size, self.word_embed_size],
initializer=self.word_initializer,
dtype=tf.float32)
self._right_target_embeddings = tf.get_variable("right_target_embeddings",
[self.vocab_size, self.word_embed_size],
initializer=self.word_initializer,
dtype=tf.float32)
self._dependency_embeddings = tf.get_variable("dependency_embeddings",
[self.dep_vocab_size, self.dep_embed_size],
initializer=self.dep_initializer,
dtype=tf.float32)
self._pos_embeddings = tf.get_variable("pos_embeddings",
[self.pos_vocab_size, self.pos_embed_size],
initializer=self.pos_initializer,
dtype=tf.float32)
input_embeds = tf.nn.dropout(tf.nn.embedding_lookup(self._word_embeddings,
tf.slice(self._input_phrases, [0,0,0], [-1, -1, 1])),
keep_prob=self._keep_prob)
dep_embeds = tf.nn.dropout(tf.nn.embedding_lookup(self._dependency_embeddings,
tf.slice(self._input_phrases, [0,0,1], [-1, -1, 1])),
keep_prob=self._keep_prob)
pos_embeds = tf.nn.dropout(tf.nn.embedding_lookup(self._pos_embeddings,
tf.slice(self._input_phrases, [0,0,2], [-1, -1, 1])),
keep_prob=self._keep_prob)
### SEPARATE TARGET EMBEDDING MATRIX ###
# left_target_embeds = tf.nn.dropout(tf.nn.embedding_lookup(self._left_target_embeddings,
# tf.slice(self._input_targets, [0,0], [-1, 1])),
# keep_prob=self._keep_prob)
# right_target_embeds = tf.nn.dropout(tf.nn.embedding_lookup(self._right_target_embeddings,
# tf.slice(self._input_targets, [0,1], [-1, 1])),
# keep_prob=self._keep_prob)
# now delay dropout so we can tanh it first
# left_target_embeds = tf.nn.embedding_lookup(self._left_target_embeddings,
# tf.slice(self._input_targets, [0,0], [-1, 1]))
# right_target_embeds = tf.nn.embedding_lookup(self._right_target_embeddings,
# tf.slice(self._input_targets, [0,1], [-1, 1]))
### ALL SAME EMBEDDING MATRIX ###
left_target_embeds = tf.nn.embedding_lookup(self._word_embeddings,
tf.slice(self._input_targets, [0,0], [-1, 1]))
right_target_embeds = tf.nn.embedding_lookup(self._word_embeddings,
tf.slice(self._input_targets, [0,1], [-1, 1]))
# print(tf.slice(self._input_phrases, [0,0,1], [-1, -1, 1]).get_shape(), dep_embeds.get_shape())
# print(left_target_embeds.get_shape(), right_target_embeds.get_shape())
self._target_embeds = tf.squeeze(tf.concat(2, [left_target_embeds, right_target_embeds]), [1])
# self._target_embeds = tf.nn.dropout(tf.nn.l2_normalize(self._target_embeds, 1 ), keep_prob=self._keep_prob)
self._target_embeds = tf.nn.dropout(self._target_embeds, keep_prob=self._keep_prob)
# print(target_embeds.get_shape())
with tf.name_scope("RNN"):
# TODO: Make it multilevel
# self._initial_state = self.cell.zero_state(batch_size, tf.float32)
# print(self._initial_state.get_shape())
input_words = [ tf.squeeze(input_, [1, 2]) for input_ in tf.split(1, self.max_num_steps, input_embeds)]
input_deps = [ tf.squeeze(input_, [1, 2]) for input_ in tf.split(1, self.max_num_steps, dep_embeds)]
input_pos = [ tf.squeeze(input_, [1, 2]) for input_ in tf.split(1, self.max_num_steps, pos_embeds)]
inputs = [ tf.concat(1, [input_word, input_dep, input_pos_])
for (input_word, input_dep, input_pos_) in zip(input_words, input_deps, input_pos)]
# inputs = input_words # just use words
# start off with a basic configuration
if self.bidirectional:
self.fwcell = tf.nn.rnn_cell.GRUCell(self.hidden_size/2,
input_size=self.input_size)
self.bwcell = tf.nn.rnn_cell.GRUCell(self.hidden_size/2,
input_size=self.input_size)
outs = tf.nn.bidirectional_rnn(self.fwcell, self.bwcell, inputs,
sequence_length=tf.to_int64(tf.squeeze(self._input_lengths, [1])),
dtype=tf.float32)
# splice out the final forward and backward hidden states since apparently the documentation lies
# fw_state = tf.split(1, 2, outs[-1])[0]
# bw_state = tf.split(1, 2, outs[0])[1]
# state = tf.concat(1, [fw_state, bw_state])
state = outs[-1]
else:
self.cell = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_size,
input_size=self.input_size)
# self.cell = tf.nn.rnn_cell.GRUCell(self.hidden_size,
# input_size=self.input_size)
_, state = tf.nn.rnn(self.cell, inputs,
sequence_length=tf.squeeze(self._input_lengths, [1]),
dtype=tf.float32)
# initial_state=self._initial_state)
# self._final_state = tf.nn.dropout(tf.nn.l2_normalize(state, 1), keep_prob= self._keep_prob)
self._final_state = tf.nn.dropout(state, keep_prob=self._keep_prob)
# get references to the RNN vars
# with tf.variable_scope('RNN', reuse=True):
# self._gate_matrix = tf.get_variable('GRUCell/Gates/Linear/Matrix')
# self._gate_bias = tf.get_variable('GRUCell/Gates/Linear/Bias')
# self._cand_matrix = tf.get_variable('GRUCell/Candidate/Linear/Matrix')
# self._cand_bias = tf.get_variable('GRUCell/Candidate/Linear/Bias')
# self._lambda2 = tf.Variable(10e-6, trainable=False, name="L2_Lambda2")
self._lambda = tf.Variable(10e-7, trainable=False, name="L2_Lambda")
with tf.name_scope("Loss"):
# self.cluster_input = tf.concat(1, [self._final_state, self._target_embeds])
self._cluster_input = self._final_state
### softmax with hidden layer ###
# hidden_layer_size = np.sqrt((self.word_embed_size)*self.num_clusters*self.num_clusters) #sqrt(in*out)
# self._c_hidden_w = tf.get_variable("c_hidden_w", [self._cluster_input.get_shape()[1], hidden_layer_size])
# self._c_hidden_b = tf.Variable(tf.zeros([hidden_layer_size], dtype=tf.float32), name="c_hidden_b")
# self._clusters_w = tf.get_variable("clusters_w", [hidden_layer_size, self.num_clusters**2])
# self._clusters_b = tf.Variable(tf.zeros([self.num_clusters**2], dtype=tf.float32), name="clusters_b")
# hidden_logits = tf.nn.dropout(tf.nn.tanh(tf.nn.xw_plus_b(self._cluster_input,
# self._c_hidden_w,
# self._c_hidden_b)),
# keep_prob=self._keep_prob)
# logits = tf.matmul(hidden_logits, self._clusters_w) + self._clusters_b
# self._xent = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits,
# tf.to_int64(tf.squeeze(self._input_labels, [1]))))
## just softmax ###
self._clusters_w = tf.get_variable("clusters_w", [self._cluster_input.get_shape()[1], self.num_clusters**2])
self._clusters_b = tf.Variable(tf.zeros([self.num_clusters**2], dtype=tf.float32), name="clusters_b")
logits = tf.matmul(self._cluster_input, self._clusters_w) + self._clusters_b
self._xent = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits,
tf.to_int64(tf.squeeze(self._input_labels, [1]))))
self._l2_penalty = self._lambda*(#tf.nn.l2_loss(self._gate_matrix)
# + tf.nn.l2_loss(self._gate_bias)
#+ tf.nn.l2_loss(self._cand_matrix)
#+ tf.nn.l2_loss(self._cand_bias)
#+
tf.nn.l2_loss(self._clusters_w)
+ tf.nn.l2_loss(self._clusters_b))
# + tf.nn.l2_loss(self._c_hidden_w)
# + tf.nn.l2_loss(self._c_hidden_b))
# + tf.nn.l2_loss(self._word_embeddings))
#+tf.nn.l2_loss(self._dependency_embeddings)
# tf.nn.l2_loss(self._left_target_embeddings)
# +tf.nn.l2_loss(self._right_target_embeddings))
# self._xent = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits,
# tf.to_float(self._input_labels)),
# name="neg_sample_loss")
self._loss = self._xent + self._l2_penalty
with tf.name_scope("Summaries"):
logit_mag = tf.histogram_summary("Logit_magnitudes", logits)
l2 = tf.scalar_summary("L2_penalty", self._l2_penalty)
target_embed_mag = tf.histogram_summary("Target_Embed_L2", tf.nn.l2_loss(self._target_embeds))
state_mag = tf.histogram_summary("RNN_final_state_L2", tf.nn.l2_loss(self._final_state))
self._penalty_summary = tf.merge_summary([logit_mag, l2, target_embed_mag, state_mag])
self._train_cost_summary = tf.merge_summary([tf.scalar_summary("Train_Loss", self._loss)])
self._valid_cost_summary = tf.merge_summary([tf.scalar_summary("Validation_Loss", self._loss)])
def main():
# Pre-process hyperparameters
FEATURE_SIZE = FLAGS.feature_size
LABEL_SIZE = FLAGS.label_size
EPOCH_NUMBER = FLAGS.epoch_number
if EPOCH_NUMBER <= 0:
EPOCH_NUMBER = None
BATCH_THREAD_NUMBER = FLAGS.batch_thread_number
MIN_AFTER_DEQUEUE = FLAGS.min_after_dequeue
BATCH_CAPACITY = BATCH_THREAD_NUMBER * FLAGS.batch_size + MIN_AFTER_DEQUEUE
MODE = FLAGS.mode
MODEL = FLAGS.model
CHECKPOINT_PATH = FLAGS.checkpoint_path
if not CHECKPOINT_PATH.startswith("fds://") and not os.path.exists(
CHECKPOINT_PATH):
os.makedirs(CHECKPOINT_PATH)
CHECKPOINT_FILE = CHECKPOINT_PATH + "/checkpoint.ckpt"
LATEST_CHECKPOINT = tf.train.latest_checkpoint(CHECKPOINT_PATH)
OUTPUT_PATH = FLAGS.output_path
if not OUTPUT_PATH.startswith("fds://") and not os.path.exists(OUTPUT_PATH):
os.makedirs(OUTPUT_PATH)
pprint.PrettyPrinter().pprint(FLAGS.__flags)
# Process TFRecoreds files
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
"label": tf.FixedLenFeature([], tf.float32),
"features": tf.FixedLenFeature([FEATURE_SIZE], tf.float32),
})
label = features["label"]
features = features["features"]
return label, features
# Read TFRecords files for training
filename_queue = tf.train.string_input_producer(
tf.train.match_filenames_once(FLAGS.train_tfrecords_file),
num_epochs=EPOCH_NUMBER)
label, features = read_and_decode(filename_queue)
batch_labels, batch_features = tf.train.shuffle_batch(
[label, features],
batch_size=FLAGS.batch_size,
num_threads=BATCH_THREAD_NUMBER,
capacity=BATCH_CAPACITY,
min_after_dequeue=MIN_AFTER_DEQUEUE)
# Read TFRecords file for validatioin
validate_filename_queue = tf.train.string_input_producer(
tf.train.match_filenames_once(FLAGS.validate_tfrecords_file),
num_epochs=EPOCH_NUMBER)
validate_label, validate_features = read_and_decode(validate_filename_queue)
validate_batch_labels, validate_batch_features = tf.train.shuffle_batch(
[validate_label, validate_features],
batch_size=FLAGS.validate_batch_size,
num_threads=BATCH_THREAD_NUMBER,
capacity=BATCH_CAPACITY,
min_after_dequeue=MIN_AFTER_DEQUEUE)
# Define the model
input_units = FEATURE_SIZE
output_units = LABEL_SIZE
model_network_hidden_units = [int(i) for i in FLAGS.model_network.split()]
def full_connect(inputs, weights_shape, biases_shape, is_train=True):
weights = tf.get_variable("weights",
weights_shape,
initializer=tf.random_normal_initializer())
biases = tf.get_variable("biases",
biases_shape,
initializer=tf.random_normal_initializer())
layer = tf.matmul(inputs, weights) + biases
if FLAGS.enable_bn and is_train:
mean, var = tf.nn.moments(layer, axes=[0])
scale = tf.get_variable("scale",
biases_shape,
initializer=tf.random_normal_initializer())
shift = tf.get_variable("shift",
biases_shape,
initializer=tf.random_normal_initializer())
layer = tf.nn.batch_normalization(layer, mean, var, shift, scale,
FLAGS.bn_epsilon)
return layer
def full_connect_relu(inputs, weights_shape, biases_shape, is_train=True):
layer = full_connect(inputs, weights_shape, biases_shape, is_train)
layer = tf.nn.relu(layer)
return layer
def customized_inference(inputs, is_train=True):
hidden1_units = 128
hidden2_units = 32
hidden3_units = 8
with tf.variable_scope("input"):
layer = full_connect_relu(inputs, [input_units, hidden1_units],
[hidden1_units], is_train)
with tf.variable_scope("layer0"):
layer = full_connect_relu(layer, [hidden1_units, hidden2_units],
[hidden2_units], is_train)
with tf.variable_scope("layer1"):
layer = full_connect_relu(layer, [hidden2_units, hidden3_units],
[hidden3_units], is_train)
if FLAGS.enable_dropout and is_train:
layer = tf.nn.dropout(layer, FLAGS.dropout_keep_prob)
with tf.variable_scope("output"):
layer = full_connect(layer, [hidden3_units, output_units],
[output_units], is_train)
return layer
def dnn_inference(inputs, is_train=True):
with tf.variable_scope("input"):
layer = full_connect_relu(inputs,
[input_units, model_network_hidden_units[0]],
[model_network_hidden_units[0]], is_train)
for i in range(len(model_network_hidden_units) - 1):
with tf.variable_scope("layer{}".format(i)):
layer = full_connect_relu(
layer,
[model_network_hidden_units[i], model_network_hidden_units[i + 1]],
[model_network_hidden_units[i + 1]], is_train)
with tf.variable_scope("output"):
layer = full_connect(layer,
[model_network_hidden_units[-1], output_units],
[output_units], is_train)
return layer
def lr_inference(inputs, is_train=True):
with tf.variable_scope("lr"):
layer = full_connect(inputs, [input_units, output_units], [output_units])
return layer
def wide_and_deep_inference(inputs, is_train=True):
return lr_inference(inputs, is_train) + dnn_inference(inputs, is_train)
def cnn_inference(inputs, is_train=True):
# TODO: Change if validate_batch_size is different
# [BATCH_SIZE, 512 * 512 * 1] -> [BATCH_SIZE, 512, 512, 1]
# inputs = tf.reshape(inputs, [FLAGS.batch_size, 512, 512, 1])
# inputs = tf.reshape(inputs, [FLAGS.batch_size,120,7,1])
inputs = tf.reshape(inputs, [FLAGS.batch_size,128,32,1])
# [BATCH_SIZE, 512, 512, 1] -> [BATCH_SIZE, 128, 128, 8]
with tf.variable_scope("conv0"):
weights = tf.get_variable("weights", [3, 3, 1, 8],
initializer=tf.random_normal_initializer())
bias = tf.get_variable("bias", [8],
initializer=tf.random_normal_initializer())
layer = tf.nn.conv2d(inputs,
weights,
strides=[1, 1, 1, 1],
padding="SAME")
layer = tf.nn.bias_add(layer, bias)
layer = tf.nn.relu(layer)
layer = tf.nn.max_pool(layer,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="SAME")
# [BATCH_SIZE, 128, 128, 8] -> [BATCH_SIZE, 32, 32, 8]
with tf.variable_scope("conv1"):
weights = tf.get_variable("weights", [3, 3, 8, 8],
initializer=tf.random_normal_initializer())
bias = tf.get_variable("bias", [8],
initializer=tf.random_normal_initializer())
layer = tf.nn.conv2d(layer,
weights,
strides=[1, 1, 1, 1],
padding="SAME")
layer = tf.nn.bias_add(layer, bias)
layer = tf.nn.relu(layer)
layer = tf.nn.max_pool(layer,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="SAME")
# [BATCH_SIZE, 32, 32, 8] -> [BATCH_SIZE, 8, 8, 8]
with tf.variable_scope("conv2"):
weights = tf.get_variable("weights", [3, 3, 8, 8],
initializer=tf.random_normal_initializer())
bias = tf.get_variable("bias", [8],
initializer=tf.random_normal_initializer())
layer = tf.nn.conv2d(layer,
weights,
strides=[1, 1, 1, 1],
padding="SAME")
layer = tf.nn.bias_add(layer, bias)
layer = tf.nn.relu(layer)
layer = tf.nn.max_pool(layer,
ksize=[1, 4, 1, 1],
strides=[1, 4, 1, 1],
padding="SAME")
# [BATCH_SIZE, 8, 8, 8] -> [BATCH_SIZE, 8 * 8 * 8]
layer = tf.reshape(layer, [-1, 8 * 8 * 8])
# [BATCH_SIZE, 8 * 8 * 8] -> [BATCH_SIZE, LABEL_SIZE]
with tf.variable_scope("output"):
weights = tf.get_variable("weights", [8 * 8 * 8, LABEL_SIZE],
initializer=tf.random_normal_initializer())
bias = tf.get_variable("bias", [LABEL_SIZE],
initializer=tf.random_normal_initializer())
layer = tf.add(tf.matmul(layer, weights), bias)
return layer
def inference(inputs, is_train=True):
if MODEL == "dnn":
return dnn_inference(inputs, is_train)
elif MODEL == "lr":
return lr_inference(inputs, is_train)
elif MODEL == "wide_and_deep":
return wide_and_deep_inference(inputs, is_train)
elif MODEL == "customized":
return customized_inference(inputs, is_train)
elif MODEL == "cnn":
return cnn_inference(inputs, is_train)
else:
print("Unknown model, exit now")
exit(1)
print("Use the model: {}, model network: {}".format(MODEL,
FLAGS.model_network))
logits = inference(batch_features, True)
batch_labels = tf.to_int64(batch_labels)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=batch_labels)
loss = tf.reduce_mean(cross_entropy, name="loss")
global_step = tf.Variable(0, name="global_step", trainable=False)
if FLAGS.enable_lr_decay:
print("Enable learning rate decay rate: {}".format(FLAGS.lr_decay_rate))
starter_learning_rate = FLAGS.learning_rate
learning_rate = tf.train.exponential_decay(starter_learning_rate,
global_step,
100000,
FLAGS.lr_decay_rate,
staircase=True)
else:
learning_rate = FLAGS.learning_rate
optimizer = get_optimizer(FLAGS.optimizer, learning_rate)
train_op = optimizer.minimize(loss, global_step=global_step)
tf.get_variable_scope().reuse_variables()
# Define accuracy op for train data
train_accuracy_logits = inference(batch_features, False)
train_softmax = tf.nn.softmax(train_accuracy_logits)
train_correct_prediction = tf.equal(
tf.argmax(train_softmax, 1), batch_labels)
train_accuracy = tf.reduce_mean(tf.cast(train_correct_prediction,
tf.float32))
# Define auc op for train data
batch_labels = tf.cast(batch_labels, tf.int32)
sparse_labels = tf.reshape(batch_labels, [-1, 1])
derived_size = tf.shape(batch_labels)[0]
indices = tf.reshape(tf.range(0, derived_size, 1), [-1, 1])
concated = tf.concat(axis=1, values=[indices, sparse_labels])
outshape = tf.stack([derived_size, LABEL_SIZE])
new_batch_labels = tf.sparse_to_dense(concated, outshape, 1.0, 0.0)
_, train_auc = tf.contrib.metrics.streaming_auc(train_softmax,
new_batch_labels)
# Define accuracy op for validate data
validate_accuracy_logits = inference(validate_batch_features, False)
validate_softmax = tf.nn.softmax(validate_accuracy_logits)
validate_batch_labels = tf.to_int64(validate_batch_labels)
validate_correct_prediction = tf.equal(
tf.argmax(validate_softmax, 1), validate_batch_labels)
validate_accuracy = tf.reduce_mean(tf.cast(validate_correct_prediction,
tf.float32))
# Define auc op for validate data
validate_batch_labels = tf.cast(validate_batch_labels, tf.int32)
sparse_labels = tf.reshape(validate_batch_labels, [-1, 1])
derived_size = tf.shape(validate_batch_labels)[0]
indices = tf.reshape(tf.range(0, derived_size, 1), [-1, 1])
concated = tf.concat(axis=1, values=[indices, sparse_labels])
outshape = tf.stack([derived_size, LABEL_SIZE])
new_validate_batch_labels = tf.sparse_to_dense(concated, outshape, 1.0, 0.0)
_, validate_auc = tf.contrib.metrics.streaming_auc(validate_softmax,
new_validate_batch_labels)
# Define inference op
inference_features = tf.placeholder("float", [None, FEATURE_SIZE])
inference_logits = inference(inference_features, False)
inference_softmax = tf.nn.softmax(inference_logits)
inference_op = tf.argmax(inference_softmax, 1)
keys_placeholder = tf.placeholder(tf.int32, shape=[None, 1])
keys = tf.identity(keys_placeholder)
model_signature = {
"inputs": exporter.generic_signature({"keys": keys_placeholder,
"features": inference_features}),
"outputs": exporter.generic_signature({"keys": keys,
"softmax": inference_softmax,
"prediction": inference_op})
}
# Initialize saver and summary
saver = tf.train.Saver()
tf.summary.scalar("loss", loss)
tf.summary.scalar("train_accuracy", train_accuracy)
tf.summary.scalar("train_auc", train_auc)
tf.summary.scalar("validate_accuracy", validate_accuracy)
tf.summary.scalar("validate_auc", validate_auc)
summary_op = tf.summary.merge_all()
# Create session to run
with tf.Session() as sess:
print("Start to run with mode: {}".format(MODE))
writer = tf.summary.FileWriter(OUTPUT_PATH, sess.graph)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
if MODE == "train":
# Restore session and start queue runner
restore_session_from_checkpoint(sess, saver, LATEST_CHECKPOINT)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
start_time = datetime.datetime.now()
try:
icount = 0
while not coord.should_stop():
_, loss_value, step = sess.run([train_op, loss, global_step])
# Print state while training
if step % FLAGS.steps_to_validate == 0:
train_accuracy_value, train_auc_value, validate_accuracy_value, validate_auc_value, summary_value = sess.run(
[train_accuracy, train_auc, validate_accuracy, validate_auc,
summary_op])
end_time = datetime.datetime.now()
print(
"[{}] Step: {}, loss: {}, train_acc: {}, train_auc: {}, valid_acc: {}, valid_auc: {}".format(
end_time - start_time, step, loss_value,
train_accuracy_value, train_auc_value,
validate_accuracy_value, validate_auc_value))
# icount = icount + 1
# if icount>= 10 :
writer.add_summary(summary_value, step)
saver.save(sess, CHECKPOINT_FILE, global_step=step)
# icount = 0 ;
start_time = end_time
except tf.errors.OutOfRangeError:
# Export the model after training
export_model(sess, saver, model_signature, FLAGS.model_path,
FLAGS.model_version)
finally:
coord.request_stop()
coord.join(threads)
elif MODE == "export":
if not restore_session_from_checkpoint(sess, saver, LATEST_CHECKPOINT):
print("No checkpoint found, exit now")
exit(1)
# Export the model
export_model(sess, saver, model_signature, FLAGS.model_path,
FLAGS.model_version)
elif MODE == "inference":
if not restore_session_from_checkpoint(sess, saver, LATEST_CHECKPOINT):
print("No checkpoint found, exit now")
exit(1)
# Load inference test data
inference_result_file_name = FLAGS.inference_result_file
inference_test_file_name = FLAGS.inference_test_file
inference_data = np.genfromtxt(inference_test_file_name, delimiter=",")
print(inference_data.shape)
inference_data_features = inference_data[:, 0:2048]
inference_data_labels = inference_data[:, 2048]
# Run inference
start_time = datetime.datetime.now()
# print(tf.shape(inference_features))
# print(inference_data_features.shape)
# temp = {inference_features: inference_data_features}
# print(temp)
prediction, prediction_softmax = sess.run(
[inference_op, inference_softmax],
feed_dict={inference_features: inference_data_features})
end_time = datetime.datetime.now()
# Compute accuracy
label_number = len(inference_data_labels)
correct_label_number = 0
for i in range(label_number):
if inference_data_labels[i] == prediction[i]:
correct_label_number += 1
accuracy = float(correct_label_number) / label_number
# Compute auc
expected_labels = np.array(inference_data_labels)
predict_labels = prediction_softmax[:, 0]
fpr, tpr, thresholds = metrics.roc_curve(expected_labels,
predict_labels,
pos_label=0)
auc = metrics.auc(fpr, tpr)
print("[{}] Inference accuracy: {}, auc: {}".format(
end_time - start_time, accuracy, auc))
# Save result into the file
np.savetxt(inference_result_file_name, prediction, delimiter=",")
print("Save result to file: {}".format(inference_result_file_name))
def range_with_reset_mask(reset_mask):
"""Converts a reset mask to ranges that restart counting after each reset.
Example usage:
```python
reset_mask = np.array( # This could be a bool array too
[[0, 0, 1, 0, 0, 0, 1, 0, 0, 0],
[1, 0, 0, 1, 0, 0, 1, 0, 1, 0],
[0, 1, 1, 1, 0, 0, 0, 0, 0, 1]])
expected = np.array(
[[0, 1, 0, 1, 2, 3, 0, 1, 2, 3],
[0, 1, 2, 0, 1, 2, 0, 1, 0, 1],
[0, 0, 0, 0, 1, 2, 3, 4, 5, 0]])
assertAllEqual(expected, session.run(range_with_reset_mask(reset_mask)))
```
Args:
reset_mask: matrix shaped `[batch_size, n]`, any numeric or bool dtype.
For convenience, numeric matrices are treated as reset matrices:
zero values are treated as no reset, non-zero values are treated as reset.
Returns:
ranges: int64 matrix shaped `[batch_size, n]` containing ranges (along rows)
that reset to 0 wherever `reset_mask` is `True` / nonzero.
Raises:
InvalidArgument: (at runtime) if `n == 0`.
"""
reset_mask = tf.convert_to_tensor(reset_mask)
shape = tf.to_int64(tf.shape(reset_mask))
indices = tf.where(reset_mask)
def _with_resets():
"""Perform the calculation for non-empty resets.
Example calculation:
```python
reset_mask = [[0, 1, 1], [1, 0, 1]]
indices = [[0, 1], [0, 2], [1, 0], [1, 2]]
indices_diff = concat(([[1, -1]], [[0, 1], [1, -2], [0, 2]]))
col_ix_diff_clean = where([1, 0, 1, 0] > 0, [1, 2, 0, 2], [-1, 1, -2, 2])
= [1, 1, 0, 2]
col_ix_reset_count = [[0, 1, 1], [0, 0, 2]]
counter = [[1, 1, 1], [1, 1, 1]] - [[0, 1, 1], [0, 0, 2]]
= [[1, 0, 0], [1, 1, -1]]
ranges_after_reset = [[1, 1, 1], [1, 2, 1]] - 1
= [[0, 0, 0], [0, 1, 0]]
Returns:
ranges_after_reset
"""
# Diff of indices (identify distances between resets); row column is
# negative at row transitions; second column contains column distances.
indices_diff = tf.concat(([[1, -1]], indices[1:] - indices[:-1]),
axis=0)
# Replace row transition diff values with the col ix at that location.
col_ix_diff_clean = tf.where(indices_diff[:, 0] > 0, indices[:, 1],
indices_diff[:, 1])
# Store these diffs at the mask locations.
col_ix_reset_count = tf.scatter_nd(indices, col_ix_diff_clean, shape)
# Create a counter that adds 1 for every col, but resets by the
# "between reset" count at the reset locations.
counter = tf.ones_like(reset_mask, dtype=tf.int64) - col_ix_reset_count
# Accumulate the counts to get a 1-based column counter that resets
# wherever reset_mask != 0. Then convert to a 0-based range.
ranges_after_reset = tf.cumsum(counter, axis=1) - 1
return ranges_after_reset
def _with_no_resets():
batch_size = shape[0]
n = shape[1]
ranges_row = tf.expand_dims(tf.range(n, dtype=tf.int64), 0)
return tf.tile(ranges_row, tf.stack([batch_size, 1]))
return tf.cond(tf.size(indices) > 0, _with_resets, _with_no_resets)
def preprocess_example(self, example, mode, unused_hparams): example["inputs"].set_shape([_CIFAR100_IMAGE_SIZE, _CIFAR100_IMAGE_SIZE, 3]) example["inputs"] = tf.to_int64(example["inputs"]) return example
def _run(self, sess, model_file, embeddings_set, output_dir,
model_version):
self.word2index, vocab_word = TaggerTensorFlowExporter.read_vocab(
model_file, 'word')
self.char2index, vocab_char = TaggerTensorFlowExporter.read_vocab(
model_file, 'char')
upchars = tf.constant([chr(i) for i in range(65, 91)])
self.lchars = tf.constant([chr(i) for i in range(97, 123)])
self.upchars_lut = tf.contrib.lookup.index_table_from_tensor(
mapping=upchars, num_oov_buckets=1, default_value=-1)
labels = self.load_labels(model_file)
# Make the TF example, network input
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
FIELD_NAME: tf.FixedLenFeature(shape=[], dtype=tf.string),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
raw_posts = tf_example[FIELD_NAME]
# Run for each post
x, xch, lengths = tf.map_fn(self._preproc_post_creator(),
raw_posts,
dtype=(tf.int64, tf.int64, tf.int32),
back_prop=False)
word_embeddings = self.task.config_params["word_embeddings"]
dsz = embeddings_set[word_embeddings["label"]]["dsz"]
char_dsz = self.task.config_params["charsz"]
init_word_vectors = baseline.RandomInitVecModel(dsz, vocab_word, False)
init_char_vectors = baseline.RandomInitVecModel(
char_dsz, vocab_char, False)
embeddings = {}
embeddings['word'] = init_word_vectors
embeddings['char'] = init_char_vectors
vocabs = {}
vocabs['word'] = vocab_word
vocabs['char'] = vocab_char
# WARNING: This can be a bug if the user defaults the values (-1)
# for conll, the mxlen=124, for idr, the mxlen is forced to a max BPTT
# for twpos, the mxlen=38
# this should probably be fixed by serializing the mxlen of the model
# or rereading it from the tensor from file
mxlen = self.task.config_params['preproc']['mxlen']
mxwlen = self.task.config_params['preproc']['mxwlen']
model_params = self.task.config_params["model"]
model_params["x"] = x
model_params["xch"] = xch
model_params["lengths"] = lengths
model_params["pkeep"] = 1
model_params["sess"] = sess
model_params["maxs"] = mxlen
model_params["maxw"] = mxwlen
print(model_params)
model = baseline.tf.tagger.create_model(labels, embeddings,
**model_params)
model.create_loss()
softmax_output = tf.nn.softmax(model.probs)
values, indices = tf.nn.top_k(softmax_output, 1)
if model.crf is True:
indices, _ = tf.contrib.crf.crf_decode(
model.probs, model.A,
tf.constant([mxlen])) ## We are assuming the batchsz is 1 here
list_of_labels = [''] * len(labels)
for label, idval in labels.items():
list_of_labels[idval] = label
class_tensor = tf.constant(list_of_labels)
table = tf.contrib.lookup.index_to_string_table_from_tensor(
class_tensor)
classes = table.lookup(tf.to_int64(indices))
self.restore_model(sess, model_file)
output_path = os.path.join(tf.compat.as_bytes(output_dir),
tf.compat.as_bytes(str(model_version)))
print('Exporting trained model to %s' % output_path)
builder = tf.saved_model.builder.SavedModelBuilder(output_path)
# Build the signature_def_map.
classify_inputs_tensor = tf.saved_model.utils.build_tensor_info(
serialized_tf_example)
classes_output_tensor = tf.saved_model.utils.build_tensor_info(classes)
scores_output_tensor = tf.saved_model.utils.build_tensor_info(values)
classification_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
tf.saved_model.signature_constants.CLASSIFY_INPUTS:
classify_inputs_tensor
},
outputs={
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES:
classes_output_tensor,
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_SCORES:
scores_output_tensor
},
method_name=tf.saved_model.signature_constants.
CLASSIFY_METHOD_NAME))
predict_inputs_tensor = tf.saved_model.utils.build_tensor_info(
raw_posts)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'tokens': predict_inputs_tensor},
outputs={
'classes': classes_output_tensor,
'scores': scores_output_tensor
},
method_name=tf.saved_model.signature_constants.
PREDICT_METHOD_NAME))
legacy_init_op = tf.group(tf.tables_initializer(),
name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'tag_text':
prediction_signature,
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
classification_signature,
},
legacy_init_op=legacy_init_op)
builder.save()
print('Successfully exported model to %s' % output_dir)
def _run(self, sess, model_file, embeddings_set, output_dir,
model_version):
self.word2index, vocab = ClassifyTensorFlowExporter.read_vocab(
model_file)
labels = self.load_labels(model_file)
# Make the TF example, network input
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
FIELD_NAME: tf.FixedLenFeature(shape=[], dtype=tf.string),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
raw_posts = tf_example[FIELD_NAME]
dense = tf.map_fn(self._preproc_post_creator(),
raw_posts,
dtype=tf.int64)
word_embeddings = self.task.config_params["word_embeddings"]
dsz = embeddings_set[word_embeddings["label"]]["dsz"]
init_vectors = baseline.RandomInitVecModel(dsz, vocab, False)
print(len(init_vectors.weights), len(vocab), init_vectors.vsz)
model_params = self.task.config_params["model"]
model_params["x"] = dense
model_params["pkeep"] = 1
model_params["sess"] = sess
print(model_params)
model = baseline.tf.classify.create_model({'word': init_vectors},
labels, **model_params)
softmax_output = tf.nn.softmax(model.logits)
values, indices = tf.nn.top_k(softmax_output, len(labels))
class_tensor = tf.constant(model.labels)
table = tf.contrib.lookup.index_to_string_table_from_tensor(
class_tensor)
classes = table.lookup(tf.to_int64(indices))
self.restore_model(sess, model_file)
output_path = os.path.join(tf.compat.as_bytes(output_dir),
tf.compat.as_bytes(str(model_version)))
print('Exporting trained model to %s' % output_path)
builder = tf.saved_model.builder.SavedModelBuilder(output_path)
# Build the signature_def_map.
classify_inputs_tensor = tf.saved_model.utils.build_tensor_info(
serialized_tf_example)
classes_output_tensor = tf.saved_model.utils.build_tensor_info(classes)
scores_output_tensor = tf.saved_model.utils.build_tensor_info(values)
classification_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
tf.saved_model.signature_constants.CLASSIFY_INPUTS:
classify_inputs_tensor
},
outputs={
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES:
classes_output_tensor,
tf.saved_model.signature_constants.CLASSIFY_OUTPUT_SCORES:
scores_output_tensor
},
method_name=tf.saved_model.signature_constants.
CLASSIFY_METHOD_NAME))
predict_inputs_tensor = tf.saved_model.utils.build_tensor_info(
raw_posts)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'tokens': predict_inputs_tensor},
outputs={
'classes': classes_output_tensor,
'scores': scores_output_tensor
},
method_name=tf.saved_model.signature_constants.
PREDICT_METHOD_NAME))
legacy_init_op = tf.group(tf.tables_initializer(),
name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'predict_text':
prediction_signature,
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
classification_signature,
},
legacy_init_op=legacy_init_op)
builder.save()
print('Successfully exported model to %s' % output_dir)
def export():
with tf.Graph().as_default():
# build inference model
# imagenet labels
names = imagenet.create_readable_names_for_imagenet_labels()
names_tensor = tf.constant(list(names.values()))
names_lookup_table = tf.contrib.lookup.index_to_string_table_from_tensor(names_tensor)
# input transformation
serialized_tf_example = tf.placeholder(tf.string, name="tf_example")
feature_configs = {
"image/encoded": tf.FixedLenFeature(shape=[], dtype=tf.string),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
jpegs = tf_example["image/encoded"]
images = tf.map_fn(preprocess_image, jpegs, dtype=tf.float32)
# run inference
with slim.arg_scope(inception.inception_v3_arg_scope()):
# inception v3 models
logits, end_points = inception.inception_v3(images, num_classes=NUM_CLASSES, is_training=False)
# logits = tf.Print(logits, [logits])
probs = tf.nn.softmax(logits)
# transform output to topk result
topk_probs, topk_indices = tf.nn.top_k(probs, NUM_TOP_CLASSES)
topk_names = names_lookup_table.lookup(tf.to_int64(topk_indices))
init_fn = slim.assign_from_checkpoint_fn(
tf.train.latest_checkpoint(FLAGS.checkpoint_dir),
slim.get_model_variables(),
)
# sess config
config = tf.ConfigProto(
gpu_options={ "allow_growth": 1, },
allow_soft_placement=True,
log_device_placement=False,
)
with tf.Session(config=config) as sess:
init_fn(sess)
# # to print out all the tensornames in the attached layers to inception V3
for node_tensor in tf.get_default_graph().as_graph_def().node:
if str(node_tensor.name).startswith('InceptionV3/Logits'):
print(str(node_tensor.name))
prelogits = sess.graph.get_tensor_by_name("InceptionV3/Logits/SpatialSqueeze:0")
# an optional alternative
# prelogits = end_points['PreLogitsFlatten']
# export inference model.
output_path = os.path.join(
tf.compat.as_bytes(FLAGS.output_dir),
tf.compat.as_bytes(str(FLAGS.model_version))
)
print("Exporting trained model to", output_path)
builder = tf.saved_model.builder.SavedModelBuilder(output_path)
# build the signature_def_map.
predict_inputs_tensor_info = tf.saved_model.utils.build_tensor_info(jpegs)
classes_output_tensor_info = tf.saved_model.utils.build_tensor_info(topk_names)
scores_output_tensor_info = tf.saved_model.utils.build_tensor_info(topk_probs)
prelogits_output_tensor_info = tf.saved_model.utils.build_tensor_info(prelogits)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={
"images": predict_inputs_tensor_info
},
outputs={
"classes": classes_output_tensor_info,
"scores": scores_output_tensor_info,
"prelogits": prelogits_output_tensor_info,
},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME
)
)
legacy_init_op = tf.group(tf.tables_initializer(), name="legacy_init_op")
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={ "predict_images": prediction_signature, },
legacy_init_op=legacy_init_op
)
builder.save()
print("Successfully exported model to %s" % FLAGS.output_dir)
def main(argv):
### Parse command line arguments
args = parser.parse_args(argv[1:])
print('Storing data in: %s' % data_dir)
print('Storing logs in: %s' % logs_dir)
### Prepare logs directory
if not args.keeplogs_k and os.path.isdir(logs_dir):
print('Found previous log files. Deleting...')
shutil.rmtree(logs_dir)
print(
'When runnning locally, start TensorBoard with: tensorboard --logdir %s'
% logs_dir)
### Load the data using TensorFlow's MNIST tutorial function read_data_sets()
data = read_data_sets(data_dir, one_hot=False, fake_data=False)
### Create the input variables for images and their labels
with tf.name_scope('input'):
images = tf.placeholder(tf.float32, [None, IMAGE_PIXELS],
name='images')
labels = tf.placeholder(tf.float32, [None], name='labels')
### Build the neural network. It consists of two hidden layers with ReLu activation functions and a linear output layer.
# Hidden layer 1
with tf.name_scope('hidden1'):
with tf.name_scope('weights'):
weights1 = tf.Variable(tf.truncated_normal(
[IMAGE_PIXELS, hidden1_units],
stddev=1.0 / math.sqrt(float(IMAGE_PIXELS))),
name='weights1')
variable_summaries(weights1)
with tf.name_scope('biases'):
biases1 = tf.Variable(tf.zeros([hidden1_units]), name='biases1')
variable_summaries(biases1)
with tf.name_scope('activation_relu'):
hidden1 = tf.nn.relu(tf.matmul(images, weights1) + biases1)
tf.summary.histogram('activations', hidden1)
# Hidden layer 2
with tf.name_scope('hidden2'):
with tf.name_scope('weights'):
weights2 = tf.Variable(tf.truncated_normal(
[hidden1_units, hidden2_units],
stddev=1.0 / math.sqrt(float(hidden1_units))),
name='weights2')
variable_summaries(weights2)
with tf.name_scope('biases'):
biases2 = tf.Variable(tf.zeros([hidden2_units]), name='biases2')
variable_summaries(biases2)
with tf.name_scope('activation_rel'):
hidden2 = tf.nn.relu(tf.matmul(hidden1, weights2) + biases2)
tf.summary.histogram('activations', hidden2)
# Linear
with tf.name_scope('linear'):
with tf.name_scope('weights'):
weights_linear = tf.Variable(tf.truncated_normal(
[hidden2_units, NUM_CLASSES],
stddev=1.0 / math.sqrt(float(hidden2_units))),
name='weights_linear')
variable_summaries(weights_linear)
with tf.name_scope('biases'):
biases_linear = tf.Variable(tf.zeros([NUM_CLASSES]),
name='biases_linear')
variable_summaries(biases_linear)
with tf.name_scope('activation_linear'):
logits = tf.matmul(hidden2, weights_linear) + biases_linear
tf.summary.histogram('activations', logits)
### Define the loss calculation based on the labels
with tf.name_scope('cross_entropy'):
loss = tf.losses.sparse_softmax_cross_entropy(
labels=tf.to_int64(labels), logits=logits)
tf.summary.scalar('Loss', loss)
### Define the training operation
with tf.name_scope('train'):
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
global_step = tf.Variable(0, trainable=False)
train_op = optimizer.minimize(loss, global_step=global_step)
### Define the accuracy calculation
with tf.name_scope('Accuracy'):
correct_prediction = tf.equal(tf.argmax(logits, 1),
tf.to_int64(labels))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('Accuracy', accuracy)
### Create the session object and initialize the variables
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
### Create summary writers for test and train operations
train_writer = tf.summary.FileWriter(logs_dir + '/train', graph=sess.graph)
test_writer = tf.summary.FileWriter(logs_dir + '/test')
summary_op = tf.summary.merge_all()
print('Start training...')
### Train the model
for i in range(train_steps):
# Every 10th iteration, calculate accuracy and write to summary for TensorBoard
if i % 10 == 0:
test_summary, acc = sess.run([summary_op, accuracy],
feed_dict={
images: data.test.images,
labels: data.test.labels
})
test_writer.add_summary(test_summary, i)
# Every 100th iteration, print accuracy to console
if i % 100 == 0:
print('Accuracy at step %s: %s' % (i, acc))
# Training step is executed here
else:
xs, ys = data.train.next_batch(batch_size)
train_summary, _ = sess.run([summary_op, train_op],
feed_dict={
images: xs,
labels: ys
})
train_writer.add_summary(train_summary, i)
print('Training complete.')
def mask_adaptive_logsoftmax(hidden,
target,
n_token,
d_embed,
d_proj,
cutoffs,
params,
tie_projs,
initializer=None,
proj_initializer=None,
div_val=1,
scope='adaptive_softmax',
proj_same_dim=True,
return_mean=True,
**kwargs):
def _logit(x, W, b, proj):
y = x
if proj is not None:
y = tf.einsum('ibd,ed->ibe', y, proj)
return tf.einsum('ibd,nd->ibn', y, W) + b
params_W, params_projs = params[0], params[1]
def _gather_logprob(logprob, target):
lp_size = tf.shape(logprob)
r = tf.range(lp_size[0])
idx = tf.stack([r, target], 1)
return tf.gather_nd(logprob, idx)
with tf.variable_scope(scope):
if len(cutoffs) == 0:
softmax_b = tf.get_variable('bias', [n_token],
initializer=tf.zeros_initializer())
output = _logit(hidden, params_W, softmax_b, params_projs)
nll = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target,
logits=output)
else:
cutoff_ends = [0] + cutoffs + [n_token]
nll = tf.zeros_like(target, dtype=tf.float32)
for i in range(len(cutoff_ends) - 1):
with tf.variable_scope('cutoff_{}'.format(i)):
l_idx, r_idx = cutoff_ends[i], cutoff_ends[i + 1]
mask = (target >= l_idx) & (target < r_idx)
mask_idx = tf.where(mask)
cur_target = tf.boolean_mask(target, mask) - l_idx
cur_d_embed = d_embed // (div_val**i)
if div_val == 1:
cur_W = params_W[l_idx:r_idx]
else:
cur_W = params_W[i]
cur_b = tf.get_variable('b', [r_idx - l_idx],
initializer=tf.zeros_initializer())
if tie_projs[i]:
if div_val == 1:
cur_proj = params_projs
else:
cur_proj = params_projs[i]
else:
if (div_val == 1 or
not proj_same_dim) and d_proj == cur_d_embed:
cur_proj = None
else:
cur_proj = tf.get_variable(
'proj', [cur_d_embed, d_proj],
initializer=proj_initializer)
if i == 0:
cluster_W = tf.get_variable(
'cluster_W', [len(cutoffs), d_embed],
initializer=tf.zeros_initializer())
cluster_b = tf.get_variable(
'cluster_b', [len(cutoffs)],
initializer=tf.zeros_initializer())
cur_W = tf.concat([cur_W, cluster_W], 0)
cur_b = tf.concat([cur_b, cluster_b], 0)
head_logit = _logit(hidden, cur_W, cur_b, cur_proj)
head_logprob = tf.nn.log_softmax(head_logit)
cur_head_logprob = tf.boolean_mask(head_logprob, mask)
cur_logprob = _gather_logprob(cur_head_logprob,
cur_target)
else:
cur_head_logprob = tf.boolean_mask(head_logprob, mask)
cur_hidden = tf.boolean_mask(hidden, mask)
tail_logit = tf.squeeze(
_logit(cur_hidden[None], cur_W, cur_b, cur_proj),
0)
tail_logprob = tf.nn.log_softmax(tail_logit)
cur_logprob = (
cur_head_logprob[:, cutoff_ends[1] + i - 1] +
_gather_logprob(tail_logprob, cur_target))
nll += tf.scatter_nd(mask_idx, -cur_logprob,
tf.to_int64(tf.shape(nll)))
if return_mean:
nll = tf.reduce_mean(nll)
return nll
def eval_metrics_host_call_fn(policy_output,
value_output,
pi_tensor,
policy_cost,
value_cost,
l2_cost,
combined_cost,
step,
est_mode=tf.estimator.ModeKeys.TRAIN):
policy_entropy = -tf.reduce_mean(
tf.reduce_sum(policy_output * tf.log(policy_output), axis=1))
# pi_tensor is one_hot when generated from sgfs (for supervised learning)
# and soft-max when using self-play records. argmax normalizes the two.
policy_target_top_1 = tf.argmax(pi_tensor, axis=1)
policy_output_in_top1 = tf.to_float(
tf.nn.in_top_k(policy_output, policy_target_top_1, k=1))
policy_output_in_top3 = tf.to_float(
tf.nn.in_top_k(policy_output, policy_target_top_1, k=3))
policy_top_1_confidence = tf.reduce_max(policy_output, axis=1)
policy_target_top_1_confidence = tf.boolean_mask(
policy_output,
tf.one_hot(policy_target_top_1,
tf.shape(policy_output)[1]))
with tf.variable_scope("metrics"):
metric_ops = {
'policy_cost':
tf.metrics.mean(policy_cost),
'value_cost':
tf.metrics.mean(value_cost),
'l2_cost':
tf.metrics.mean(l2_cost),
'policy_entropy':
tf.metrics.mean(policy_entropy),
'combined_cost':
tf.metrics.mean(combined_cost),
'policy_accuracy_top_1':
tf.metrics.mean(policy_output_in_top1),
'policy_accuracy_top_3':
tf.metrics.mean(policy_output_in_top3),
'policy_top_1_confidence':
tf.metrics.mean(policy_top_1_confidence),
'policy_target_top_1_confidence':
tf.metrics.mean(policy_target_top_1_confidence),
'value_confidence':
tf.metrics.mean(tf.abs(value_output)),
}
if est_mode == tf.estimator.ModeKeys.EVAL:
return metric_ops
# NOTE: global_step is rounded to a multiple of FLAGS.summary_steps.
eval_step = tf.reduce_min(step)
# Create summary ops so that they show up in SUMMARIES collection
# That way, they get logged automatically during training
summary_writer = summary.create_file_writer(FLAGS.work_dir)
with summary_writer.as_default(), \
summary.record_summaries_every_n_global_steps(
params['summary_steps'], eval_step):
for metric_name, metric_op in metric_ops.items():
summary.scalar(metric_name, metric_op[1], step=eval_step)
# Reset metrics occasionally so that they are mean of recent batches.
reset_op = tf.variables_initializer(tf.local_variables("metrics"))
cond_reset_op = tf.cond(
tf.equal(eval_step % params['summary_steps'], tf.to_int64(1)),
lambda: reset_op, lambda: tf.no_op())
return summary.all_summary_ops() + [cond_reset_op]
def main(_):
if len(sys.argv) < 2 or sys.argv[-1].startswith('-'):
print('Usage: mnist_export.py [--training_iteration=x] '
'[--export_version=y] export_dir')
sys.exit(-1)
if FLAGS.training_iteration <= 0:
print('Please specify a positive value for training iteration.')
sys.exit(-1)
if FLAGS.export_version <= 0:
print('Please specify a positive value for version number.')
sys.exit(-1)
# Train model
print('Training model...')
mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True)
sess = tf.InteractiveSession()
serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
feature_configs = {
'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32),
}
tf_example = tf.parse_example(serialized_tf_example, feature_configs)
x = tf.identity(tf_example['x'], name='x') # use tf.identity() to assign name
y_ = tf.placeholder('float', shape=[None, 10])
w = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
sess.run(tf.initialize_all_variables())
y = tf.nn.softmax(tf.matmul(x, w) + b, name='y')
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
values, indices = tf.nn.top_k(y, 10)
prediction_classes = tf.contrib.lookup.index_to_string(
tf.to_int64(indices), mapping=tf.constant([str(i) for i in range(10)]))
for _ in range(FLAGS.training_iteration):
batch = mnist.train.next_batch(50)
train_step.run(feed_dict={x: batch[0], y_: batch[1]})
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
print('training accuracy %g' %
sess.run(accuracy,
feed_dict={x: mnist.test.images,
y_: mnist.test.labels}))
print('Done training!')
# Export model
# WARNING(break-tutorial-inline-code): The following code snippet is
# in-lined in tutorials, please update tutorial documents accordingly
# whenever code changes.
export_path = sys.argv[-1]
print('Exporting trained model to %s' % export_path)
init_op = tf.group(tf.initialize_all_tables(), name='init_op')
saver = tf.train.Saver(sharded=True)
model_exporter = exporter.Exporter(saver)
model_exporter.init(
sess.graph.as_graph_def(),
init_op=init_op,
default_graph_signature=exporter.classification_signature(
input_tensor=serialized_tf_example,
classes_tensor=prediction_classes,
scores_tensor=values),
named_graph_signatures={
'inputs': exporter.generic_signature({'images': x}),
'outputs': exporter.generic_signature({'scores': y})})
model_exporter.export(export_path, tf.constant(FLAGS.export_version), sess)
print('Done exporting!')
def loss(logits, labels):
labels = tf.to_int64(labels)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=labels, logits=logits, name='xentropy')
return tf.reduce_mean(cross_entropy, name='xentropy_mean')
def train():
with tf.Graph().as_default(), tf.device('/cpu:0'):
batch = tf.Variable(0, trainable=False)
learning_rate = tf_util.get_learning_rate(batch, BASE_LEARNING_RATE,
BATCH_SIZE, DECAY_STEP,
DECAY_RATE)
tf.summary.scalar('learning_rate', learning_rate)
bn_decay = tf_util.get_bn_decay(batch, BN_INIT_DECAY, BATCH_SIZE,
BN_DECAY_DECAY_STEP,
BN_DECAY_DECAY_RATE, BN_DECAY_CLIP)
tf.summary.scalar('bn_decay', bn_decay)
if OPTIMIZER == 'momentum':
print('Using SGD with Momentum as optimizer')
trainer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=MOMENTUM)
elif OPTIMIZER == 'adam':
print('Using Adam as optimizer')
trainer = tf.train.AdamOptimizer(learning_rate)
else:
raise Exception("Unknown optimizer")
tower_grads = []
inputs_phs = []
labels_phs = []
is_training_phs = []
with tf.variable_scope(tf.get_variable_scope()):
for i in range(NUM_GPU):
with tf.device('/gpu:%d' % i):
with tf.name_scope('%s_%d' % (TOWER_NAME, i)) as scope:
# Configure the neural network using every layers
nn = MLP(kernel_size=[1, 1],
stride=[1, 1],
padding='VALID',
weight_decay=0.0,
bn=True,
bn_decay=bn_decay,
is_dist=True)
# Configure the gcn vertex layer object
if GCN == 'mrgcn':
v_layer = tf_vertex.max_relat_conv_layer
elif GCN == 'edgeconv':
v_layer = tf_vertex.edge_conv_layer
elif GCN == 'graphsage':
v_layer = wrapped_partial(
tf_vertex.graphsage_conv_layer,
normalize=NORMALIZE_SAGE)
elif GCN == 'gin':
v_layer = wrapped_partial(
tf_vertex.gin_conv_layer,
zero_epsilon=ZERO_EPSILON_GIN)
else:
raise Exception("Unknown gcn type")
v_layer_builder = VertexLayer(v_layer, nn, K,
GCN_NUM_FILTERS)
# Configure the gcn edge layer object
if EDGE_LAY == 'dilated':
e_layer = wrapped_partial(
tf_edge.dilated_knn_graph,
stochastic=STOCHASTIC_DILATION,
epsilon=STO_DILATED_EPSILON)
elif EDGE_LAY == 'knn':
e_layer = tf_edge.knn_graph
else:
raise Exception("Unknown edge laer type")
distance_metric = tf_util.pairwise_distance
e_layer_builder = EdgeLayer(e_layer, K,
distance_metric)
# Get the whole model builer
model_obj = model_builder.Model(
BATCH_SIZE,
NUM_POINTS,
NUM_LAYERS,
NUM_CLASSES,
vertex_layer_builder=v_layer_builder,
edge_layer_builder=e_layer_builder,
mlp_builder=nn,
skip_connect=SKIP_CONNECT)
inputs_ph = model_obj.inputs
labels_ph = model_obj.labels
is_training_ph = model_obj.is_training
pred = model_obj.pred
inputs_phs.append(inputs_ph)
labels_phs.append(labels_ph)
is_training_phs.append(is_training_ph)
loss = model_obj.get_loss(pred, labels_phs[-1])
tf.summary.scalar('loss', loss)
correct = tf.equal(tf.argmax(pred, 2),
tf.to_int64(labels_phs[-1]))
accuracy = tf.reduce_sum(tf.cast(
correct, tf.float32)) / float(
BATCH_SIZE * NUM_POINTS)
tf.summary.scalar('accuracy', accuracy)
tf.get_variable_scope().reuse_variables()
grads = trainer.compute_gradients(loss)
tower_grads.append(grads)
grads = tf_util.average_gradients(tower_grads)
train_op = trainer.apply_gradients(grads, global_step=batch)
saver = tf.train.Saver(tf.global_variables(),
sharded=True,
max_to_keep=None)
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
sess = tf.Session(config=config)
# Add summary writers
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(os.path.join(LOG_DIR, 'train'),
sess.graph)
test_writer = tf.summary.FileWriter(os.path.join(LOG_DIR, 'test'))
# Init variables on GPUs
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init)
if (CHECKPOINT != ''):
saver.restore(sess, CHECKPOINT)
sem_seg_util.log_string(LOG_FOUT, "Model restored.")
start_epoch = int(CHECKPOINT.split('.')[0].split('epoch_')[1])
print('Resuming from epoch: {}'.format(start_epoch))
else:
start_epoch = 0
ops = {
'inputs_phs': inputs_phs,
'labels_phs': labels_phs,
'is_training_phs': is_training_phs,
'pred': pred,
'loss': loss,
'train_op': train_op,
'merged': merged,
'step': batch
}
for epoch in range(start_epoch + 1, MAX_EPOCH):
sem_seg_util.log_string(LOG_FOUT, '**** EPOCH %03d ****' % (epoch))
sys.stdout.flush()
train_one_epoch(sess, ops, train_writer)
# Save the variables to disk.
if epoch % 10 == 0:
save_path = saver.save(
sess, os.path.join(LOG_DIR,
'epoch_' + str(epoch) + '.ckpt'))
sem_seg_util.log_string(LOG_FOUT,
"Model saved in file: %s" % save_path)
def run_training(tfrecords_path, batch_size, epoch, model_path, log_dir,
start_lr, wd, kp):
with tf.Graph().as_default():
sess = tf.Session()
features, age_labels, gender_labels, _ = get_inputs(
path=tfrecords_path, batch_size=batch_size, num_epochs=epoch)
net, gender_logits, age_logits = inference(features,
age_labels,
gender_labels,
training=True)
# Add to the Graph the loss calculation.
age_cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=age_labels, logits=age_logits)
age_cross_entropy_mean = tf.reduce_mean(age_cross_entropy)
gender_cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=gender_labels, logits=gender_logits)
gender_cross_entropy_mean = tf.reduce_mean(gender_cross_entropy)
# l2 regularization
total_loss = tf.add_n(
[gender_cross_entropy_mean, age_cross_entropy_mean] +
tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
age_ = tf.cast(tf.constant([i for i in range(0, 117)]), tf.float32)
age = tf.reduce_sum(tf.multiply(tf.nn.softmax(age_logits), age_),
axis=1)
abs_loss = tf.losses.absolute_difference(age_labels, age)
prob_gender = tf.argmax(tf.nn.softmax(gender_logits), 1)
gender_acc = tf.reduce_mean(
tf.to_float(tf.equal(tf.to_int64(prob_gender), gender_labels)))
tf.summary.scalar("age_cross_entropy", age_cross_entropy_mean)
tf.summary.scalar("gender_cross_entropy", gender_cross_entropy_mean)
tf.summary.scalar("total loss", total_loss)
tf.summary.scalar("train_abs_age_error", abs_loss)
tf.summary.scalar("gender_accuracy", gender_acc)
# Add to the Graph operations that train the model.
global_step = tf.Variable(0, name="global_step", trainable=False)
lr = tf.train.exponential_decay(start_lr,
global_step=global_step,
decay_steps=2000,
decay_rate=0.1,
staircase=True)
optimizer = tf.train.AdamOptimizer(lr)
tf.summary.scalar("lr", lr)
# # only train age branch
# trainable = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Net/Branch1') + tf.get_collection(
# tf.GraphKeys.GLOBAL_VARIABLES, scope='Logits/Age')
all_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
update_ops = tf.get_collection(
tf.GraphKeys.UPDATE_OPS) # update batch normalization layer
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(total_loss,
global_step) #, var_list=trainable
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init_op)
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(log_dir, sess.graph)
new_saver = tf.train.Saver(all_vars, max_to_keep=100)
ckpt = tf.train.get_checkpoint_state(model_path)
if ckpt and ckpt.model_checkpoint_path:
new_saver.restore(sess, ckpt.model_checkpoint_path)
print("restore and continue training!")
else:
pass
# Start input enqueue threads.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
step = sess.run(global_step)
start_time = time.time()
while not coord.should_stop():
# start_time = time.time()
# Run one step of the model. The return values are
# the activations from the `train_op` (which is
# discarded) and the `loss` op. To inspect the values
# of your ops or variables, you may include them in
# the list passed to sess.run() and the value tensors
# will be returned in the tuple from the call.
_, summary = sess.run([train_op, merged])
train_writer.add_summary(summary, step)
# duration = time.time() - start_time
# Print an overview fairly often.
if step % 100 == 0:
duration = time.time() - start_time
print('%.3f sec' % duration)
start_time = time.time()
if step % 1000 == 0:
save_path = new_saver.save(sess,
os.path.join(
model_path, "model.ckpt"),
global_step=global_step)
print("Model saved in file: %s" % save_path)
step = sess.run(global_step)
except tf.errors.OutOfRangeError:
print('Done training for %d epochs, %d steps.' % (epoch, step))
finally:
# When done, ask the threads to stop.
save_path = new_saver.save(sess,
os.path.join(model_path, "model.ckpt"),
global_step=global_step)
print("Model saved in file: %s" % save_path)
coord.request_stop()
# Wait for threads to finish.
coord.join(threads)
sess.close()
def __init__(self, sent_length, class_num, embedding_size,
initial_embedding_dict, l2_lambda, filter_sizes, filter_num):
self.input_x = tf.placeholder(tf.int32, [None, sent_length],
name="input_x")
self.input_y = tf.placeholder(tf.float32, [None, class_num],
name="input_y")
self.dropout_keep_prob_1 = tf.placeholder(tf.float32,
name="dropout_keep_prob_1")
self.dropout_keep_prob_2 = tf.placeholder(tf.float32,
name="dropout_keep_prob_2")
l2_loss = tf.constant(0.0)
with tf.device("/cpu:0"), tf.name_scope("embedding"):
self.embedding_dict = tf.Variable(initial_embedding_dict,
name="Embedding",
dtype=tf.float32,
trainable=True)
self.embedded_chars = tf.nn.embedding_lookup(
self.embedding_dict, self.input_x)
self.embedded_chars_expanded = tf.expand_dims(
self.embedded_chars, -1)
# Add dropout
with tf.name_scope("dropout_1"):
self.embedded_chars_dropped = tf.nn.dropout(
self.embedded_chars_expanded,
self.dropout_keep_prob_1,
noise_shape=[
tf.shape(self.embedded_chars_expanded)[0], sent_length, 1,
1
])
# Create a convolution + maxpool layer for each filter size
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
with tf.name_scope("conv-maxpool-%s" % filter_size):
# Convolution Layer
filter_shape = [filter_size, embedding_size, 1, filter_num]
weights = tf.Variable(tf.truncated_normal(filter_shape,
stddev=0.01),
name="W")
bias = tf.Variable(tf.constant(0.01, shape=[filter_num]),
name="b")
conv = tf.nn.conv2d(self.embedded_chars_dropped,
weights,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
# Apply nonlinearity
h = tf.nn.relu(tf.nn.bias_add(conv, bias), name="relu")
# Maxpooling over the outputs
pooled = tf.nn.max_pool(
h,
ksize=[1, sent_length - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding='VALID',
name="pool")
pooled_outputs.append(pooled)
# Combine all the pooled features
num_filters_total = filter_num * len(filter_sizes)
self.h_pool = tf.concat(pooled_outputs, 3)
self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total])
# Add dropout
with tf.name_scope("dropout_2"):
self.h_drop = tf.nn.dropout(self.h_pool_flat,
self.dropout_keep_prob_2)
with tf.name_scope("linear"):
weights = tf.get_variable(
"W_1",
shape=[num_filters_total, 64],
initializer=tf.contrib.layers.xavier_initializer())
bias = tf.Variable(tf.constant(0.1, shape=[64]), name="b1")
l2_loss += tf.nn.l2_loss(weights)
l2_loss += tf.nn.l2_loss(bias)
self.linear_result_1 = tf.nn.xw_plus_b(self.h_drop,
weights,
bias,
name="linear")
self.softmax = tf.nn.softmax(self.linear_result_1)
with tf.name_scope("output"):
weights = tf.get_variable(
"W_2",
shape=[64, 1],
initializer=tf.contrib.layers.xavier_initializer())
bias = tf.Variable(tf.constant(0.1, shape=[1]), name="b2")
l2_loss += tf.nn.l2_loss(weights)
l2_loss += tf.nn.l2_loss(bias)
self.linear_result_2 = tf.nn.xw_plus_b(self.softmax,
weights,
bias,
name="linear2")
self.output = 5 * tf.exp(self.linear_result_2) / (
1 + tf.exp(self.linear_result_2)) - 2.5
self.predictions = tf.to_int64(self.output + 2.5)
with tf.name_scope("loss"):
losses = tf.reduce_mean(
tf.square(self.output + 2 -
tf.to_float(tf.argmax(self.input_y, 1))))
self.loss = tf.sqrt(losses) + l2_lambda * l2_loss
with tf.name_scope("accuracy"):
correct_predictions = tf.equal(self.predictions,
tf.argmax(self.input_y, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_predictions,
"float"),
name="accuracy")
def evaluate():
with tf.Graph().as_default():
with tf.device('/gpu:'+str(GPU_INDEX)):
pointclouds_pl, labels_pl, smpws_pl = MODEL.placeholder_inputs(BATCH_SIZE, NUM_POINT)
is_training_pl = tf.placeholder(tf.bool, shape=())
print is_training_pl
# Note the global_step=batch parameter to minimize.
# That tells the optimizer to helpfully increment the 'batch' parameter for you every time it trains.
batch = tf.Variable(0)
bn_decay = get_bn_decay(batch)
tf.summary.scalar('bn_decay', bn_decay)
print "--- Get model and loss"
# Get model and loss
pred, end_points = MODEL.get_model(pointclouds_pl, is_training_pl, NUM_CLASSES, bn_decay=bn_decay)
loss = MODEL.get_loss(pred, labels_pl, smpws_pl)
tf.summary.scalar('loss', loss)
correct = tf.equal(tf.argmax(pred, 2), tf.to_int64(labels_pl))
accuracy = tf.reduce_sum(tf.cast(correct, tf.float32)) / float(BATCH_SIZE*NUM_POINT)
tf.summary.scalar('accuracy', accuracy)
print "--- Get training operator"
# Get training operator
learning_rate = get_learning_rate(batch)
tf.summary.scalar('learning_rate', learning_rate)
if OPTIMIZER == 'momentum':
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum=MOMENTUM)
elif OPTIMIZER == 'adam':
optimizer = tf.train.AdamOptimizer(learning_rate)
train_op = optimizer.minimize(loss, global_step=batch)
# Add ops to save and restore all the variables.
saver = tf.train.Saver()
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
# sess = tf.Session(config=config)
sess = tf.Session(config=config)
saver.restore(sess, 'log/run1/best_model_epoch_120.ckpt')
log_string('--- Loaded weights from log/run1/best_model_epoch_120.ckpt')
# Add summary writers
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(os.path.join(LOG_DIR, 'train'), sess.graph)
test_writer = tf.summary.FileWriter(os.path.join(LOG_DIR, 'test'), sess.graph)
# Init variables
# init = tf.global_variables_initializer()
# sess.run(init, {is_training_pl: True})
# sess.run(init)
ops = {'pointclouds_pl': pointclouds_pl,
'labels_pl': labels_pl,
'smpws_pl': smpws_pl,
'is_training_pl': is_training_pl,
'pred': pred,
'loss': loss,
'train_op': train_op,
'merged': merged,
'step': batch,
'end_points': end_points}
for epoch in range(MAX_EPOCH):
log_string('**** EPOCH %03d ****' % (epoch))
sys.stdout.flush()
eval_whole_scene_one_epoch(sess, ops, test_writer)
def _slow_greedy_infer(self, features, decode_length):
"""A slow greedy inference method.
Quadratic time in decode_length.
Args:
features: an map of string to `Tensor`
decode_length: an integer. How many additional timesteps to decode.
Returns:
samples: an integer `Tensor`.
logits: `Tensor` of shape [batch_size, time, 1, 1, vocab_size].
losses: a dictionary: {loss-name (string): floating point `Scalar`}
"""
if not features:
features = {}
inputs_old = None
if "inputs" in features and len(features["inputs"].shape) < 4:
inputs_old = features["inputs"]
features["inputs"] = tf.expand_dims(features["inputs"], 2)
if not self.has_input:
features["partial_targets"] = tf.to_int64(features["inputs"])
# Save the targets in a var and reassign it after the tf.while loop to avoid
# having targets being in a 'while' frame. This ensures targets when used
# in metric functions stays in the same frame as other vars.
targets_old = features.get("targets", None)
target_modality = self._hparams.problems[
self._problem_idx].target_modality
def infer_step(recent_output, recent_logits, unused_loss):
"""Inference step."""
recent_output.set_shape([None, None, None, 1])
padded = tf.pad(recent_output, [[0, 0], [0, 1], [0, 0], [0, 0]])
features["targets"] = padded
# This is inefficient in that it generates samples at all timesteps,
# not just the last one, except if target_modality is pointwise.
samples, logits, losses = self.sample(features)
# Concatenate the already-generated recent_output with last timestep
# of the newly-generated samples.
if target_modality.top_is_pointwise:
cur_sample = samples[:, -1, :, :]
else:
cur_sample = samples[:, tf.shape(recent_output)[1], :, :]
cur_sample = tf.to_int64(tf.expand_dims(cur_sample, axis=1))
samples = tf.concat([recent_output, cur_sample], axis=1)
samples.set_shape([None, None, None, 1])
# Assuming we have one shard for logits.
logits = tf.concat([recent_logits, logits[0][:, -1:]], 1)
loss = sum([l for l in losses.values() if l is not None])
return samples, logits, loss
# Create an initial output tensor. This will be passed
# to the infer_step, which adds one timestep at every iteration.
if "partial_targets" in features:
initial_output = tf.to_int64(features["partial_targets"])
while len(initial_output.get_shape().as_list()) < 4:
initial_output = tf.expand_dims(initial_output, 2)
batch_size = tf.shape(initial_output)[0]
else:
batch_size = tf.shape(features["inputs"])[0]
initial_output = tf.zeros((batch_size, 0, 1, 1), dtype=tf.int64)
# Hack: foldl complains when the output shape is less specified than the
# input shape, so we confuse it about the input shape.
initial_output = tf.slice(initial_output, [0, 0, 0, 0],
tf.shape(initial_output))
target_modality = self._hparams.problems[
self._problem_idx].target_modality
if is_class_modality(target_modality):
decode_length = 1
else:
decode_length = tf.shape(features["inputs"])[1] + decode_length
# Initial values of result, logits and loss.
result = initial_output
# tensor of shape [batch_size, time, 1, 1, vocab_size]
logits = tf.zeros(
(batch_size, 0, 1, 1, target_modality.top_dimensionality))
logits.set_shape([None, None, None, None, None])
loss = 0.0
def while_exit_cond(result, logits, loss): # pylint: disable=unused-argument
"""Exit the loop either if reach decode_length or EOS."""
length = tf.shape(result)[1]
not_overflow = length < decode_length
if self._problem_hparams.stop_at_eos:
def fn_not_eos():
return tf.not_equal( # Check if the last predicted element is a EOS
tf.squeeze(result[:, -1, :, :]), text_encoder.EOS_ID)
not_eos = tf.cond(
# We only check for early stoping if there is at least 1 element (
# otherwise not_eos will crash)
tf.not_equal(length, 0),
fn_not_eos,
lambda: True,
)
return tf.cond(
tf.equal(batch_size, 1),
# If batch_size == 1, we check EOS for early stoping
lambda: tf.logical_and(not_overflow, not_eos),
# Else, just wait for max length
lambda: not_overflow)
return not_overflow
result, logits, loss = tf.while_loop(
while_exit_cond,
infer_step, [result, logits, loss],
shape_invariants=[
tf.TensorShape([None, None, None, None]),
tf.TensorShape([None, None, None, None, None]),
tf.TensorShape([]),
],
back_prop=False,
parallel_iterations=1)
if inputs_old is not None: # Restore to not confuse Estimator.
features["inputs"] = inputs_old
# Reassign targets back to the previous value.
if targets_old is not None:
features["targets"] = targets_old
losses = {"training": loss}
if "partial_targets" in features:
partial_target_length = tf.shape(features["partial_targets"])[1]
result = tf.slice(result, [0, partial_target_length, 0, 0],
[-1, -1, -1, -1])
return result, logits, losses
def mean_accuracy_multi_binary_label_with_logits(label, logits):
return tf.reduce_mean(tf.to_float(
tf.equal(tf.to_int64(tf.greater(logits, 0.0)), label)),
axis=0,
keep_dims=True)