def loss(logits, labels, num_classes):
"""Calculate the loss from the logits and the labels.
Args:
logits: tensor, float - [batch_size, width, height, num_classes].
Use vgg_fcn.up as logits.
labels: Labels tensor, int32 - [batch_size, width, height, num_classes].
The ground truth of your data.
head: numpy array - [num_classes]
Weighting the loss of each class
Optional: Prioritize some classes
Returns:
loss: Loss tensor of type float.
"""
with tf.name_scope('loss'):
logits = tf.reshape(logits, (-1, num_classes))
epsilon = tf.constant(value=1e-4)
logits = logits + epsilon
labels = tf.to_float(tf.reshape(labels, (-1, num_classes)))
softmax = tf.nn.softmax(logits)
cross_entropy = -tf.reduce_sum(labels * tf.log(softmax), reduction_indices=[1])
cross_entropy_mean = tf.reduce_mean(cross_entropy,
name='xentropy_mean')
tf.add_to_collection('losses', cross_entropy_mean)
loss = tf.add_n(tf.get_collection('losses'), name='total_loss')
return loss
def batch_norm(x, n_out, phase_train, scope='bn', affine=True):
"""
Batch normalization on convolutional maps.
Args:
x: Tensor, 4D BHWD input maps
n_out: integer, depth of input maps
phase_train: boolean tf.Variable, true indicates training phase
scope: string, variable scope
affine: whether to affine-transform outputs
Return:
normed: batch-normalized maps
"""
with tf.variable_scope(scope):
beta = tf.Variable(tf.constant(0.0, shape=[n_out]),
name='beta', trainable=True)
gamma = tf.Variable(tf.constant(1.0, shape=[n_out]),
name='gamma', trainable=affine)
tf.add_to_collection('biases', beta)
tf.add_to_collection('weights', gamma)
batch_mean, batch_var = tf.nn.moments(x, [0,1,2], name='moments')
ema = tf.train.ExponentialMovingAverage(decay=0.99)
def mean_var_with_update():
ema_apply_op = ema.apply([batch_mean, batch_var])
with tf.control_dependencies([ema_apply_op]):
return tf.identity(batch_mean), tf.identity(batch_var)
mean, var = control_flow_ops.cond(phase_train,
mean_var_with_update,
lambda: (ema.average(batch_mean), ema.average(batch_var)))
normed = tf.nn.batch_norm_with_global_normalization(x, mean, var,
beta, gamma, 1e-3, affine)
return normed
def loss(logits, labels):
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, labels, name='xentropy')
data_loss = tf.reduce_mean(cross_entropy, name='xentropy_mean')
tf.add_to_collection('losses', data_loss)
total_loss = tf.add_n(tf.get_collection('losses'), name='total_loss')
return total_loss
def add_gradients_summary(grads, name_prefix="", name_suffix="",
collection_key=None):
""" add_gradients_summary.
Add histogram summary for given gradients.
Arguments:
grads: A list of `Tensor`. The gradients to summarize.
name_prefix: `str`. A prefix to add to summary scope.
name_suffix: `str`. A suffix to add to summary scope.
collection_key: `str`. A collection to store the summaries.
Returns:
The list of created gradient summaries.
"""
# Add histograms for gradients.
summ = []
for grad, var in grads:
if grad is not None:
summ_name = format_scope_name(var.op.name, name_prefix,
"Gradients/" + name_suffix)
summ_exists = summary_exists(summ_name)
if summ_exists is not None:
tf.add_to_collection(collection_key, summ_exists)
summ.append(summ_exists)
else:
summ.append(get_summary("histogram", summ_name, grad,
collection_key))
return summ
def _variable_with_weight_decay(shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal
distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
initializer = tf.truncated_normal_initializer(stddev=stddev)
var = tf.get_variable('weights', shape=shape,
initializer=initializer)
# var = tf.get_variable(name="weights", shape=shape,
# initializer=tf.contrib.layers.xavier_initializer())
if wd and (not tf.get_variable_scope().reuse):
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def one_hot_encoding(target, n_classes, on_value=1.0, off_value=1.0,
name="OneHotEncoding"):
""" One Hot Encoding.
Transform numeric labels into a binary vector.
Input:
The Labels Placeholder.
Output:
2-D Tensor, The encoded labels.
Arguments:
target: `Placeholder`. The labels placeholder.
n_classes: `int`. Total number of classes.
on_value: `scalar`. A scalar defining the on-value.
off_value: `scalar`. A scalar defining the off-value.
name: A name for this layer (optional). Default: 'OneHotEncoding'.
"""
with tf.name_scope(name):
if target.dtype == tf.dtypes.int32:
target = standard_ops.to_int64(target)
target = standard_ops.one_hot(target, n_classes,
on_value=on_value,
off_value=off_value)
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, target)
return target
def _variable_with_weight_decay(name, shape, stddev, wd):
var = _variable_on_cpu(name, shape,
tf.truncated_normal_initializer(stddev=stddev))
if wd is not None:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def reshape(incoming, new_shape, name="Reshape"):
""" Reshape.
A layer that reshape the incoming layer tensor output to the desired shape.
Arguments:
incoming: A `Tensor`. The incoming tensor.
new_shape: A list of `int`. The desired shape.
name: A name for this layer (optional).
"""
with tf.name_scope(name) as scope:
inference = incoming
if isinstance(inference, list):
inference = tf.concat(inference, 0)
inference = tf.cast(inference, tf.float32)
inference = tf.reshape(inference, shape=new_shape)
inference.scope = scope
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def activation(incoming, activation='linear', name='activation'):
""" Activation.
Apply given activation to incoming tensor.
Arguments:
incoming: A `Tensor`. The incoming tensor.
activation: `str` (name) or `function` (returning a `Tensor`).
Activation applied to this layer (see tflearn.activations).
Default: 'linear'.
"""
if isinstance(activation, str):
x = activations.get(activation)(incoming)
elif hasattr(incoming, '__call__'):
x = activation(incoming)
else:
raise ValueError('Unknown activation type.')
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, x)
return x
def set_input_shape(self, input_shape):
batch_size, rows, cols, input_channels = input_shape
kernel_shape = tuple(self.kernel_shape) + (input_channels,
self.output_channels)
assert len(kernel_shape) == 4
assert all(isinstance(e, int) for e in kernel_shape), kernel_shape
with tf.variable_scope(self.name) as scope:
init = tf.random_normal(kernel_shape, dtype=tf.float32)
init = init / tf.sqrt(1e-7 + tf.reduce_sum(tf.square(init),
axis=(0, 1, 2)))
self.kernels = tf.Variable(init, name='Weight')
# tf.add_to_collection('test', self.kernels)
if(self.l2 != 0):
weight_loss = tf.multiply(tf.nn.l2_loss(
self.kernels), self.l2, name='weight_loss')
tf.add_to_collection('losses', weight_loss)
self.b = tf.Variable(
np.zeros((self.output_channels,)).astype('float32'),name='Bias')
input_shape = list(input_shape)
input_shape[0] = 1
dummy_batch = tf.zeros(input_shape)
dummy_output = self.fprop(dummy_batch)
output_shape = [int(e) for e in dummy_output.get_shape()]
output_shape[0] = batch_size
self.output_shape = tuple(output_shape)
def batch_norm(x, decay=0.999, epsilon=1e-03, is_training=True,
scope="scope"):
x_shape = x.get_shape()
num_inputs = x_shape[-1]
reduce_dims = list(range(len(x_shape) - 1))
with tf.variable_scope(scope):
beta = create_var("beta", [num_inputs,],
initializer=tf.zeros_initializer())
gamma = create_var("gamma", [num_inputs,],
initializer=tf.ones_initializer())
# for inference
moving_mean = create_var("moving_mean", [num_inputs,],
initializer=tf.zeros_initializer(),
trainable=False)
moving_variance = create_var("moving_variance", [num_inputs],
initializer=tf.ones_initializer(),
trainable=False)
if is_training:
mean, variance = tf.nn.moments(x, axes=reduce_dims)
update_move_mean = moving_averages.assign_moving_average(moving_mean,
mean, decay=decay)
update_move_variance = moving_averages.assign_moving_average(moving_variance,
variance, decay=decay)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_move_mean)
tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, update_move_variance)
else:
mean, variance = moving_mean, moving_variance
return tf.nn.batch_normalization(x, mean, variance, beta, gamma, epsilon)
def bacthnorm(inputs, scope, epsilon=1e-05, momentum=0.99, is_training=True):
inputs_shape = inputs.get_shape().as_list()# 输出 形状尺寸
params_shape = inputs_shape[-1:]# 输入参数的长度
axis = list(range(len(inputs_shape) - 1))
with tf.variable_scope(scope):
beta = create_variable("beta", params_shape,
initializer=tf.zeros_initializer())
gamma = create_variable("gamma", params_shape,
initializer=tf.ones_initializer())
# 均值 常量 不需要训练 for inference
moving_mean = create_variable("moving_mean", params_shape,
initializer=tf.zeros_initializer(), trainable=False)
# 方差 常量 不需要训练
moving_variance = create_variable("moving_variance", params_shape,
initializer=tf.ones_initializer(), trainable=False)
if is_training:
mean, variance = tf.nn.moments(inputs, axes=axis)# 计算均值和方差
# 移动平均求 均值和 方差 考虑上一次的量 xt = a * x_t-1 +(1-a)*x_now
update_move_mean = moving_averages.assign_moving_average(moving_mean,
mean, decay=momentum)
update_move_variance = moving_averages.assign_moving_average(moving_variance,
variance, decay=momentum)
tf.add_to_collection(UPDATE_OPS_COLLECTION, update_move_mean)
tf.add_to_collection(UPDATE_OPS_COLLECTION, update_move_variance)
else:
mean, variance = moving_mean, moving_variance
return tf.nn.batch_normalization(inputs, mean, variance, beta, gamma, epsilon)
def loss(H, logits, labels):
"""Calculates the loss from the logits and the labels.
Args:
logits: Logits tensor, float - [batch_size, NUM_CLASSES].
labels: Labels tensor, int32 - [batch_size].
Returns:
loss: Loss tensor of type float.
"""
# Convert from sparse integer labels in the range [0, NUM_CLASSSES)
# to 1-hot dense float vectors (that is we will have batch_size vectors,
# each with NUM_CLASSES values, all of which are 0.0 except there will
# be a 1.0 in the entry corresponding to the label).
with tf.name_scope('loss'):
batch_size = tf.size(labels)
labels = tf.expand_dims(labels, 1)
indices = tf.expand_dims(tf.range(0, batch_size), 1)
concated = tf.concat(1, [indices, labels])
onehot_labels = tf.sparse_to_dense(
concated, tf.pack([batch_size, H['arch']['num_classes']]), 1.0, 0.0)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits,
onehot_labels,
name='xentropy')
cross_entropy_mean = tf.reduce_mean(
cross_entropy, name='xentropy_mean')
tf.add_to_collection('losses', cross_entropy_mean)
loss = tf.add_n(tf.get_collection('losses'), name='total_loss')
return loss
def loss(self, predicts, labels, objects_num):
"""Add Loss to all the trainable variables
Args:
predicts: 4-D tensor [batch_size, cell_size, cell_size, 5 * boxes_per_cell]
===> (num_classes, boxes_per_cell, 4 * boxes_per_cell)
labels : 3-D tensor of [batch_size, max_objects, 5]
objects_num: 1-D tensor [batch_size]
"""
class_loss = tf.constant(0, tf.float32)
object_loss = tf.constant(0, tf.float32)
noobject_loss = tf.constant(0, tf.float32)
coord_loss = tf.constant(0, tf.float32)
loss = [0, 0, 0, 0]
for i in range(self.batch_size):
predict = predicts[i, :, :, :]
label = labels[i, :, :]
object_num = objects_num[i]
nilboy = tf.ones([7,7,2])
tuple_results = tf.while_loop(self.cond1, self.body1, [tf.constant(0), object_num, [class_loss, object_loss, noobject_loss, coord_loss], predict, label, nilboy])
for j in range(4):
loss[j] = loss[j] + tuple_results[2][j]
nilboy = tuple_results[5]
tf.add_to_collection('losses', (loss[0] + loss[1] + loss[2] + loss[3])/self.batch_size)
tf.summary.scalar('class_loss', loss[0]/self.batch_size)
tf.summary.scalar('object_loss', loss[1]/self.batch_size)
tf.summary.scalar('noobject_loss', loss[2]/self.batch_size)
tf.summary.scalar('coord_loss', loss[3]/self.batch_size)
tf.summary.scalar('weight_loss', tf.add_n(tf.get_collection('losses')) - (loss[0] + loss[1] + loss[2] + loss[3])/self.batch_size )
return tf.add_n(tf.get_collection('losses'), name='total_loss'), nilboy
def func_wrapper(weights):
if weights.dtype.base_dtype == tf.float16:
tf.add_to_collection('REGULARIZATION_FUNCTIONS', (weights, regularizer))
# disabling the inner regularizer
return None
else:
return regularizer(weights)
def local_response_normalization(incoming, depth_radius=5, bias=1.0,
alpha=0.0001, beta=0.75,
name="LocalResponseNormalization"):
""" Local Response Normalization.
Input:
4-D Tensor Layer.
Output:
4-D Tensor Layer. (Same dimension as input).
Arguments:
incoming: `Tensor`. Incoming Tensor.
depth_radius: `int`. 0-D. Half-width of the 1-D normalization window.
Defaults to 5.
bias: `float`. An offset (usually positive to avoid dividing by 0).
Defaults to 1.0.
alpha: `float`. A scale factor, usually positive. Defaults to 0.0001.
beta: `float`. An exponent. Defaults to `0.5`.
name: `str`. A name for this layer (optional).
"""
with tf.name_scope(name) as scope:
inference = tf.nn.lrn(incoming, depth_radius=depth_radius,
bias=bias, alpha=alpha,
beta=beta, name=name)
inference.scope = scope
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def inference(input_tensor,train,regularizer):
#第一层卷积
with tf.variable_scope('layer1-conv1'):
conv1_weights = tf.get_variable("weight",
[CONV1_SIZE,CONV1_SIZE,NUM_CHANNELS,CONV1_DEEP],
initializer=tf.truncated_normal_initializer(stddev=0.1))
conv1_biases = tf.get_variable("biases",[CONV1_DEEP],
initializer=tf.constant_initializer(0.0))
conv1 = tf.nn.conv2d(input_tensor,conv1_weights,
strides=[1,1,1,1],padding='SAME')
relu1 = tf.nn.relu(tf.nn.bias_add(conv1,conv1_biases))
#第二层池化
with tf.name_scope('layer2-pool1'):
pool1 = tf.nn.max_pool(relu1,ksize=[1,2,2,1],
strides=[1,2,2,1],padding='SAME')
#第三层卷积
with tf.variable_scope('layer3-conv2'):
conv2_weights = tf.get_variable("weight",
[CONV2_SIZE,CONV2_SIZE,CONV1_DEEP,CONV2_DEEP],
initializer=tf.truncated_normal_initializer(stddev=0.1))
conv2_biases = tf.get_variable("biases",[CONV2_DEEP],
initializer=tf.constant_initializer(0.0))
conv2 = tf.nn.conv2d(pool1,conv2_weights,
strides=[1,1,1,1],padding='SAME')
relu2 = tf.nn.relu(tf.nn.bias_add(conv2,conv2_biases))
#第四层池化
with tf.name_scope('layer4-pool2'):
pool2 = tf.nn.max_pool(relu2,ksize=[1,2,2,1],
strides=[1,2,2,1],padding='SAME')
pool_shape = pool2.get_shape().as_list()
nodes = pool_shape[1] * pool_shape[2] * pool_shape[3]
reshaped = tf.reshape(pool2,[pool_shape[0],nodes])
#第五层全连接层
with tf.variable_scope('layer5-fc1'):
fc1_weights = tf.get_variable("weight",[nodes,FC_SIZE],
initializer=tf.truncated_normal_initializer(stddev=0.1))
#只有全连接层的权重需要加入正则化
if regularizer != None:
tf.add_to_collection('losses',regularizer(fc1_weights))
fc1_biases = tf.get_variable("bias",[FC_SIZE],
initializer=tf.constant_initializer(0.1))
fc1 = tf.nn.relu(tf.matmul(reshaped,fc1_weights) + fc1_biases)
if train: fc1 = tf.nn.dropout(fc1,0.5)
#第六层全连接层
with tf.variable_scope('layer6-fc2'):
fc2_weights = tf.get_variable("weight",[FC_SIZE,NUM_LABELS],
initializer=tf.truncated_normal_initializer(stddev=0.1))
#只有全连接层的权重需要加入正则化
if regularizer != None:
tf.add_to_collection('losses',regularizer(fc2_weights))
fc2_biases = tf.get_variable("bias",[NUM_LABELS],
initializer=tf.constant_initializer(0.1))
logit = tf.matmul(fc1,fc2_weights) + fc2_biases
return logit
def loss_fun(logits, labels):
"""Add L2Loss to all the trainable variables.
Add summary for "Loss" and "Loss/avg".
Args:
logits: Logits from inference().
labels: Labels from distorted_inputs or inputs(). 1-D tensor
of shape [batch_size]
distillation: if set to True, use probabilities and not class labels to
compute softmax loss
Returns:
Loss tensor of type float.
"""
# Calculate the cross entropy between labels and predictions
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels, name='cross_entropy_per_example')
# Calculate the average cross entropy loss across the batch.
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
# Add to TF collection for losses
tf.add_to_collection('losses', cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
return tf.add_n(tf.get_collection('losses'), name='total_loss')
def loss(logits, labels):
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits, labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
return tf.add_n(tf.get_collection('losses'), name='total_loss')
def bn(x, c):
x_shape = x.get_shape()
params_shape = x_shape[-1:]
if c["use_bias"]:
bias = _get_variable("bias", params_shape, initializer=tf.zeros_initializer)
return x + bias
axis = list(range(len(x_shape) - 1))
beta = _get_variable("beta", params_shape, initializer=tf.zeros_initializer)
gamma = _get_variable("gamma", params_shape, initializer=tf.ones_initializer)
moving_mean = _get_variable("moving_mean", params_shape, initializer=tf.zeros_initializer, trainable=False)
moving_variance = _get_variable("moving_variance", params_shape, initializer=tf.ones_initializer, trainable=False)
# These ops will only be preformed when training.
mean, variance = tf.nn.moments(x, axis)
update_moving_mean = moving_averages.assign_moving_average(moving_mean, mean, BN_DECAY)
update_moving_variance = moving_averages.assign_moving_average(moving_variance, variance, BN_DECAY)
tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_mean)
tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_variance)
mean, variance = control_flow_ops.cond(
c["is_training"], lambda: (mean, variance), lambda: (moving_mean, moving_variance)
)
x = tf.nn.batch_normalization(x, mean, variance, beta, gamma, BN_EPSILON)
# x.set_shape(inputs.get_shape()) ??
return x
def test_raise_error_if_more_than_one_cached_item(self):
with tf.Graph().as_default():
tf.Variable([1])
tf.add_to_collection(tf.GraphKeys.SAVERS, tf.train.Saver())
tf.add_to_collection(tf.GraphKeys.SAVERS, tf.train.Saver())
with self.assertRaisesRegexp(RuntimeError, 'More than one item'):
supervised_session.Scaffold()
def cross_entropy_loss(logits,one_hot_labels,label_smoothing = 0,
weight = 1.0,scope = None) :
"""Define a Cross Entropy loss using softmax_cross_entropy_with_logits.
It can scale the loss by weight factor, and smooth the labels.
Args:
logits: [batch_size, num_classes] logits outputs of the network .
one_hot_labels: [batch_size, num_classes] target one_hot_encoded labels.
label_smoothing: if greater than 0 then smooth the labels.
weight: scale the loss by this factor.
scope: Optional scope for op_scope.
Returns:
A tensor with the softmax_cross_entropy loss.
"""
logits.get_shape().assert_is_compatible_with(one_hot_labels.get_shape())
#with tf.op_scope([logits,one_hot_labels],scope,'CrossEntropyLoss') :
with tf.name_scope(scope,'CrossEntropyLoss',[logits,one_hot_labels]) :
num_classes = one_hot_labels.get_shape()[-1].value
one_hot_labels = tf.cast(one_hot_labels,logits.dtype)
if label_smoothing > 0 :
smooth_positives = 1.0 - label_smoothing
smooth_negatives = label_smoothing / num_classes
one_hot_labels = one_hot_labels * smooth_positives + smooth_negatives
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits,
one_hot_labels,
name = 'xentropy')
weight = tf.convert_to_tensor(weight,
dtype = logits.dtype.base_dtype,
name = 'loss_weight')
loss = tf.mul(weight,tf.reduce_mean(cross_entropy),name = 'value')
tf.add_to_collection(LOSSES_COLLECTION,loss)
return loss
def loss(logits, labels):
"""Add L2Loss to all the trainable variables.
Add summary for for "Loss" and "Loss/avg".
Args:
logits: Logits from inference().
labels: Labels from distorted_inputs or inputs(). 1-D tensor
of shape [batch_size]
Returns:
Loss tensor of type float.
"""
# Reshape the labels into a dense Tensor of
# shape [batch_size, NUM_CLASSES].
sparse_labels = tf.reshape(labels, [FLAGS.batch_size, 1])
indices = tf.reshape(tf.range(0, FLAGS.batch_size, 1), [FLAGS.batch_size, 1])
concated = tf.concat(1, [indices, sparse_labels])
dense_labels = tf.sparse_to_dense(concated,
[FLAGS.batch_size, NUM_CLASSES],
1.0, 0.0)
# Calculate the average cross entropy loss across the batch.
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
logits, dense_labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
# The total loss is defined as the cross entropy loss plus all of the weight
# decay terms (L2 loss).
return tf.add_n(tf.get_collection('losses'), name='total_loss')
def weight_variable(shape, initializer=None, init_val=None, wd=None, name=None, trainable=True):
"""Initialize weights.
Args:
shape: shape of the weights, list of int
wd: weight decay
"""
log = logger.get()
if initializer is None:
# initializer = tf.truncated_normal(shape, stddev=0.01)
initializer = tf.truncated_normal_initializer(stddev=0.01)
if init_val is None:
var = tf.Variable(initializer(shape), name=name, trainable=trainable)
else:
var = tf.Variable(init_val, name=name, trainable=trainable)
# log.info(var.name)
# if init_val is not None:
# if hasattr(init_val, 'shape'):
# log.info('Initialized with array shape {}'.format(init_val.shape))
# else:
# log.info('Initialized with {}'.format(init_val))
if wd:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def _variable(name, shape, initializer, wd=None):
var = tf.get_variable(name, shape, initializer=initializer)
if wd is not None:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def weighted_loss(logits, labels, num_classes, head=None):
""" median-frequency re-weighting """
with tf.name_scope('loss'):
logits = tf.reshape(logits, (-1, num_classes))
epsilon = tf.constant(value=1e-10)
logits = logits + epsilon
# consturct one-hot label array
label_flat = tf.reshape(labels, (-1, 1))
# should be [batch ,num_classes]
labels = tf.reshape(tf.one_hot(label_flat, depth=num_classes), (-1, num_classes))
softmax = tf.nn.softmax(logits)
cross_entropy = -tf.reduce_sum(tf.multiply(labels * tf.log(softmax + epsilon), head), axis=[1])
cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
loss = tf.add_n(tf.get_collection('losses'), name='total_loss')
return loss
def _construct(self):
"""
Construct the model; main part of it goes here
"""
# our query = m_u + e_i
query = (self._cur_user, self._cur_item)
neg_query = (self._cur_user, self._cur_item_negative)
# Positive
neighbor = self._mem_layer(query,
self.user_memory(self.input_neighborhoods),
self.user_output(self.input_neighborhoods),
self.input_neighborhood_lengths,
self.config.max_neighbors)[-1].output
self.score = self._output_module(tf.concat([self._cur_user * self._cur_item,
neighbor], axis=1))
# Negative
neighbor_negative = self._mem_layer(neg_query,
self.user_memory(self.input_neighborhoods_negative),
self.user_output(self.input_neighborhoods_negative),
self.input_neighborhood_lengths_negative,
self.config.max_neighbors)[-1].output
negative_output = self._output_module(tf.concat(
[self._cur_user * self._cur_item_negative, neighbor_negative], axis=1))
# Loss and Optimizer
self.loss = LossLayer()(self.score, negative_output)
self._optimizer = OptimizerLayer(self.config.optimizer, clip=self.config.grad_clip,
params=self.config.optimizer_params)
self.train = self._optimizer(self.loss)
tf.add_to_collection(GraphKeys.PREDICTION, self.score)
def _variable_with_weight_decay(name, shape, stddev, wd):
"""Helper to create an initialized Variable with weight decay.
Note that the Variable is initialized with a truncated normal distribution.
A weight decay is added only if one is specified.
Args:
name: name of the variable
shape: list of ints
stddev: standard deviation of a truncated Gaussian
wd: add L2Loss weight decay multiplied by this float. If None, weight
decay is not added for this Variable.
Returns:
Variable Tensor
"""
dtype = tf.float16 if FLAGS.use_fp16 else tf.float32
var = _variable_on_cpu(
name,
shape,
tf.truncated_normal_initializer(stddev=stddev, dtype=dtype))
if wd is not None:
weight_decay = tf.mul(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def add_trainable_vars_summary(variables, name_prefix="", name_suffix="",
collection_key=None):
""" add_trainable_vars_summary.
Add histogram summary for given variables weights.
Arguments:
variables: A list of `Variable`. The variables to summarize.
name_prefix: `str`. A prefix to add to summary scope.
name_suffix: `str`. A suffix to add to summary scope.
collection_key: `str`. A collection to store the summaries.
Returns:
The list of created weights summaries.
"""
# Add histograms for trainable variables.
summ = []
for var in variables:
summ_name = format_scope_name(var.op.name, name_prefix, name_suffix)
summ_exists = summary_exists(summ_name)
if summ_exists is not None:
tf.add_to_collection(collection_key, summ_exists)
summ.append(summ_exists)
else:
summ.append(get_summary("histogram", summ_name, var, collection_key))
return summ
def get_output_for(self):
"""Perform the convolution operation, activation and return the output
``tf.Tensor``.
Returns
-------
``tf.Tensor``
Output tensor of this layer.
"""
states = []
outputs = []
lstm = rnn_cell.BasicLSTMCell(self.num_units, state_is_tuple=True)
initial_state = state = lstm.zero_state(batch_size, tf.float32)
with tf.name_scope(self.name) as scope:
for _id in xrange(self.num_of_cells):
if _id > 0:
scope.reuse_variables()
output, state = lstm(self.input_layer, state)
if self.activation is not None:
output = self.activation(output)
outputs.append(output)
states.append(state)
final_state = state
if self.return_cell_out:
output = tf.reshape(tf.concat(1, outputs), [-1, size])
else:
output = outputs[-1]
tf.add_to_collection(BerryKeys.LAYER_OUTPUTS, output)
return output
def batch_norm(inputs,
scope_name,
decay=0.999,
center=True,
scale=False,
epsilon=0.001,
moving_vars='moving_vars',
activation=None,
is_training=True,
trainable=True,
restore=True,
scope=None,
reuse=None):
"""Adds a Batch Normalization layer.
Args:
inputs: a tensor of size [batch_size, height, width, channels]
or [batch_size, channels].
decay: decay for the moving average.
center: If True, subtract beta. If False, beta is not created and ignored.
scale: If True, multiply by gamma. If False, gamma is
not used. When the next layer is linear (also e.g. ReLU), this can be
disabled since the scaling can be done by the next layer.
epsilon: small float added to variance to avoid dividing by zero.
moving_vars: collection to store the moving_mean and moving_variance.
activation: activation function.
is_training: whether or not the model is in training mode.
trainable: whether or not the variables should be trainable or not.
restore: whether or not the variables should be marked for restore.
scope: Optional scope for variable_op_scope.
reuse: whether or not the layer and its variables should be reused. To be
able to reuse the layer scope must be given.
Returns:
a tensor representing the output of the operation.
"""
inputs_shape = inputs.get_shape()
with tf.variable_scope(scope_name, [inputs], scope, reuse=reuse):
axis = list(range(len(inputs_shape) - 1))
params_shape = inputs_shape[-1:]
# Allocate parameters for the beta and gamma of the normalization.
beta, gamma = None, None
if center:
beta = tf.get_variable('beta',
params_shape,
initializer=tf.zeros_initializer(),
trainable=trainable)
if scale:
gamma = tf.get_variable('gamma',
params_shape,
initializer=tf.ones_initializer(),
trainable=trainable)
moving_collections = [
moving_vars, tf.GraphKeys.MOVING_AVERAGE_VARIABLES
]
moving_mean = tf.get_variable('moving_mean',
params_shape,
initializer=tf.zeros_initializer(),
trainable=False)
moving_variance = tf.get_variable('moving_variance',
params_shape,
initializer=tf.ones_initializer(),
trainable=False)
if is_training:
# Calculate the moments based on the individual batch.
mean, variance = tf.nn.moments(inputs, axis)
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay)
tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_mean)
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay)
tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_variance)
else:
# Just use the moving_mean and moving_variance.
mean = moving_mean
variance = moving_variance
# Normalize the activations.
outputs = tf.nn.batch_normalization(inputs, mean, variance, beta,
gamma, epsilon)
outputs.set_shape(inputs.get_shape())
if activation:
outputs = activation(outputs)
return outputs
def run_training():
#Read the training data
examples, n_classes = read_training_list()
np.random.seed(42) #shuffle the same way each time for consistency
np.random.shuffle(examples)
# TODO TODO - implement some sort of cross validation
fetcher = Fetcher(examples)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels and mark as input.
x = tf.placeholder(tf.float32, shape=(None, 512, 512, 3))
y_ = tf.placeholder(tf.float32, shape=(None, n_classes))
# See "Using instance keys": https://cloud.google.com/ml/docs/how-tos/preparing-models
# for why we have keys_placeholder
keys_placeholder = tf.placeholder(tf.int64, shape=(None, ))
# IMPORTANT: Do not change the input map
inputs = {'key': keys_placeholder.name, 'image': x.name}
tf.add_to_collection('inputs', json.dumps(inputs))
# Build a the network
net = network(x)
# Add to the Graph the Ops for loss calculation.
loss = slim.losses.softmax_cross_entropy(net, y_)
tf.scalar_summary(loss.op.name,
loss) # keep track of value for TensorBoard
# To be able to extract the id, we need to add the identity function.
keys = tf.identity(keys_placeholder)
# The prediction will be the index in logits with the highest score.
# We also use a softmax operation to produce a probability distribution
# over all possible digits.
# DO NOT REMOVE OR CHANGE VARIABLE NAMES - used when predicting with a model
prediction = tf.argmax(net, 1)
scores = tf.nn.softmax(net)
# Mark the outputs.
outputs = {
'key': keys.name,
'prediction': prediction.name,
'scores': scores.name
}
tf.add_to_collection('outputs', json.dumps(outputs))
# Add to the Graph the Ops that calculate and apply gradients.
train_op = tf.train.AdamOptimizer(1e-4).minimize(loss)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
# Add the variable initializer Op.
init = tf.initialize_all_variables()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.train.SummaryWriter(FLAGS.train_output_dir,
sess.graph)
# And then after everything is built:
# Run the Op to initialize the variables.
sess.run(init)
# Start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
images, labels = fetcher.load_batch(FLAGS.batch_size)
feed_dict = {x: images, y_: labels}
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 1 == 0:
# Print status to stdout.
print('Step %d: loss = %.2f (%.3f sec)' %
(step, loss_value, duration))
sys.stdout.flush()
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
# Export the model so that it can be loaded and used later for predictions.
file_io.create_dir(FLAGS.model_dir)
saver.save(sess, os.path.join(FLAGS.model_dir, 'export'))
#make world readable for submission to evaluation server
if FLAGS.model_dir.startswith('gs://'):
subprocess.call(
['gsutil', 'acl', 'ch', '-u', 'AllUsers:R', FLAGS.model_dir])
def ssd_losses_old(logits,
localisations,
gclasses,
glocalisations,
gscores,
match_threshold=0.5,
negative_ratio=3.,
alpha=1.,
label_smoothing=0.,
device='/cpu:0',
scope=None):
"""Loss functions for training the SSD 300 VGG network.
This function defines the different loss components of the SSD, and
adds them to the TF loss collection.
Arguments:
logits: (list of) predictions logits Tensors;
localisations: (list of) localisations Tensors;
gclasses: (list of) groundtruth labels Tensors;
glocalisations: (list of) groundtruth localisations Tensors;
gscores: (list of) groundtruth score Tensors;
"""
with tf.device(device):
with tf.name_scope(scope, 'ssd_losses'):
l_cross_pos = []
l_cross_neg = []
l_loc = []
for i in range(len(logits)):
dtype = logits[i].dtype
with tf.name_scope('block_%i' % i):
# Sizing weight...
wsize = tfe.get_shape(logits[i], rank=5)
wsize = wsize[1] * wsize[2] * wsize[3]
# Positive mask.
pmask = gscores[i] > match_threshold
fpmask = tf.cast(pmask, dtype)
n_positives = tf.reduce_sum(fpmask)
# Select some random negative entries.
# n_entries = np.prod(gclasses[i].get_shape().as_list())
# r_positive = n_positives / n_entries
# r_negative = negative_ratio * n_positives / (n_entries - n_positives)
# Negative mask.
no_classes = tf.cast(pmask, tf.int32)
predictions = slim.softmax(logits[i])
nmask = tf.logical_and(tf.logical_not(pmask),
gscores[i] > -0.5)
fnmask = tf.cast(nmask, dtype)
nvalues = tf.where(nmask, predictions[:, :, :, :, 0],
1. - fnmask)
nvalues_flat = tf.reshape(nvalues, [-1])
# Number of negative entries to select.
n_neg = tf.cast(negative_ratio * n_positives, tf.int32)
n_neg = tf.maximum(n_neg, tf.size(nvalues_flat) // 8)
n_neg = tf.maximum(n_neg, tf.shape(nvalues)[0] * 4)
max_neg_entries = 1 + tf.cast(tf.reduce_sum(fnmask),
tf.int32)
n_neg = tf.minimum(n_neg, max_neg_entries)
val, idxes = tf.nn.top_k(-nvalues_flat, k=n_neg)
max_hard_pred = -val[-1]
# Final negative mask.
nmask = tf.logical_and(nmask, nvalues < max_hard_pred)
fnmask = tf.cast(nmask, dtype)
# Add cross-entropy loss.
with tf.name_scope('cross_entropy_pos'):
fpmask = wsize * fpmask
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits[i], labels=gclasses[i])
loss = tf.losses.compute_weighted_loss(loss, fpmask)
l_cross_pos.append(loss)
with tf.name_scope('cross_entropy_neg'):
fnmask = wsize * fnmask
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits[i], labels=no_classes)
loss = tf.losses.compute_weighted_loss(loss, fnmask)
l_cross_neg.append(loss)
# Add localization loss: smooth L1, L2, ...
with tf.name_scope('localization'):
# Weights Tensor: positive mask + random negative.
weights = tf.expand_dims(alpha * fpmask, axis=-1)
loss = custom_layers.abs_smooth(localisations[i] -
glocalisations[i])
loss = tf.losses.compute_weighted_loss(loss, weights)
l_loc.append(loss)
# Additional total losses...
with tf.name_scope('total'):
total_cross_pos = tf.add_n(l_cross_pos, 'cross_entropy_pos')
total_cross_neg = tf.add_n(l_cross_neg, 'cross_entropy_neg')
total_cross = tf.add(total_cross_pos, total_cross_neg,
'cross_entropy')
total_loc = tf.add_n(l_loc, 'localization')
# Add to EXTRA LOSSES TF.collection
tf.add_to_collection('EXTRA_LOSSES', total_cross_pos)
tf.add_to_collection('EXTRA_LOSSES', total_cross_neg)
tf.add_to_collection('EXTRA_LOSSES', total_cross)
tf.add_to_collection('EXTRA_LOSSES', total_loc)
def build_graph(reader,
model,
train_data_pattern,
label_loss_fn=losses.CrossEntropyLoss(),
batch_size=1000,
base_learning_rate=0.01,
learning_rate_decay_examples=1000000,
learning_rate_decay=0.95,
optimizer_class=tf.train.AdamOptimizer,
clip_gradient_norm=1.0,
regularization_penalty=1,
num_readers=1,
num_epochs=None):
"""Creates the Tensorflow graph.
This will only be called once in the life of
a training model, because after the graph is created the model will be
restored from a meta graph file rather than being recreated.
Args:
reader: The data file reader. It should inherit from BaseReader.
model: The core model (e.g. logistic or neural net). It should inherit
from BaseModel.
train_data_pattern: glob path to the training data files.
label_loss_fn: What kind of loss to apply to the model. It should inherit
from BaseLoss.
batch_size: How many examples to process at a time.
base_learning_rate: What learning rate to initialize the optimizer with.
optimizer_class: Which optimization algorithm to use.
clip_gradient_norm: Magnitude of the gradient to clip to.
regularization_penalty: How much weight to give the regularization loss
compared to the label loss.
num_readers: How many threads to use for I/O operations.
num_epochs: How many passes to make over the data. 'None' means an
unlimited number of passes.
"""
global_step = tf.Variable(0, trainable=False, name="global_step")
local_device_protos = device_lib.list_local_devices()
gpus = [x.name for x in local_device_protos if x.device_type == 'GPU']
gpus = gpus[:FLAGS.num_gpu]
num_gpus = len(gpus)
if num_gpus > 0:
logging.info("Using the following GPUs to train: " + str(gpus))
num_towers = num_gpus
device_string = '/gpu:%d'
else:
logging.info("No GPUs found. Training on CPU.")
num_towers = 1
device_string = '/cpu:%d'
learning_rate = tf.train.exponential_decay(base_learning_rate,
global_step * batch_size *
num_towers,
learning_rate_decay_examples,
learning_rate_decay,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = optimizer_class(learning_rate)
unused_video_id, model_input_raw, labels_batch, num_frames = (
get_input_data_tensors(reader,
train_data_pattern,
batch_size=batch_size * num_towers,
num_readers=num_readers,
num_epochs=num_epochs))
tf.summary.histogram("model/input_raw", model_input_raw)
feature_dim = len(model_input_raw.get_shape()) - 1
model_input = tf.nn.l2_normalize(model_input_raw, feature_dim)
tower_inputs = tf.split(model_input, num_towers)
tower_labels = tf.split(labels_batch, num_towers)
tower_num_frames = tf.split(num_frames, num_towers)
tower_gradients = []
tower_predictions = []
tower_label_losses = []
tower_reg_losses = []
for i in range(num_towers):
# For some reason these 'with' statements can't be combined onto the same
# line. They have to be nested.
with tf.device(device_string % i):
with (tf.variable_scope(("tower"), reuse=True if i > 0 else None)):
with (slim.arg_scope(
[slim.model_variable, slim.variable],
device="/cpu:0" if num_gpus != 1 else "/gpu:0")):
result = model.create_model(tower_inputs[i],
num_frames=tower_num_frames[i],
vocab_size=reader.num_classes,
labels=tower_labels[i])
for variable in slim.get_model_variables():
tf.summary.histogram(variable.op.name, variable)
predictions = result["predictions"]
tower_predictions.append(predictions)
if "loss" in result.keys():
label_loss = result["loss"]
else:
label_loss = label_loss_fn.calculate_loss(
predictions, tower_labels[i])
if "regularization_loss" in result.keys():
reg_loss = result["regularization_loss"]
else:
reg_loss = tf.constant(0.0)
reg_losses = tf.losses.get_regularization_losses()
if reg_losses:
reg_loss += tf.add_n(reg_losses)
tower_reg_losses.append(reg_loss)
# Adds update_ops (e.g., moving average updates in batch normalization) as
# a dependency to the train_op.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
if "update_ops" in result.keys():
update_ops += result["update_ops"]
if update_ops:
with tf.control_dependencies(update_ops):
barrier = tf.no_op(name="gradient_barrier")
with tf.control_dependencies([barrier]):
label_loss = tf.identity(label_loss)
tower_label_losses.append(label_loss)
# Incorporate the L2 weight penalties etc.
final_loss = regularization_penalty * reg_loss + label_loss
gradients = optimizer.compute_gradients(
final_loss, colocate_gradients_with_ops=False)
tower_gradients.append(gradients)
label_loss = tf.reduce_mean(tf.stack(tower_label_losses))
tf.summary.scalar("label_loss", label_loss)
if regularization_penalty != 0:
reg_loss = tf.reduce_mean(tf.stack(tower_reg_losses))
tf.summary.scalar("reg_loss", reg_loss)
merged_gradients = utils.combine_gradients(tower_gradients)
if clip_gradient_norm > 0:
with tf.name_scope('clip_grads'):
merged_gradients = utils.clip_gradient_norms(
merged_gradients, clip_gradient_norm)
video_vars = set([
'tower/embeddings:0', 'tower/project_mat:0', 'tower/gates/weights:0',
'tower/experts/weights:0', 'tower/experts/biases:0',
'tower/output_bn_1/beta:0', 'tower/output_bn_1/gamma:0',
'tower/gating_layer_0/weights:0', 'tower/gating_prob_bn_0/beta:0',
'tower/gating_prob_bn_0/gamma:0', 'tower/output_layer_2/weights:0',
'tower/output_bn_2/beta:0', 'tower/output_bn_2/gamma:0',
'tower/gating_layer_1/weights:0', 'tower/gating_prob_bn_1/beta:0',
'tower/gating_prob_bn_1/gamma:0', 'tower/output_layer/weights:0',
'tower/output_bn/beta:0', 'tower/output_bn/gamma:0',
'tower/gating_layer/weights:0', 'tower/gating_prob_bn/beta:0',
'tower/gating_prob_bn/gamma:0'
])
frame_vars = set([
'tower/input_bn/beta:0', 'tower/input_bn/gamma:0',
'tower/video_VLAD/cluster_weights:0',
'tower/video_VLAD/cluster_bn/beta:0',
'tower/video_VLAD/cluster_bn/gamma:0',
'tower/audio_VLAD/cluster_weights:0',
'tower/audio_VLAD/cluster_bn/beta:0',
'tower/audio_VLAD/cluster_bn/gamma:0', 'tower/hidden1_weights:0',
'tower/hidden2_weights:0', 'tower/SVD_mat1_biases:0',
'tower/SVD_mat2_biases:0', 'tower/hidden1_bn/beta:0',
'tower/hidden1_bn/gamma:0', 'tower/gating_weights_2:0',
'tower/gating_bn/beta:0', 'tower/gating_bn/gamma:0'
])
all_vars_to_save = [v[1] for v in merged_gradients]
if FLAGS.freeze_frame:
merged_gradients = [
i for i in merged_gradients if i[1].name not in frame_vars
]
if FLAGS.freeze_video:
merged_gradients = [
i for i in merged_gradients if i[1].name not in video_vars
]
train_op = optimizer.apply_gradients(merged_gradients,
global_step=global_step)
for i in merged_gradients:
print(i[1])
print("\n\n\nAll vars to save: ")
for i in all_vars_to_save:
print(i)
print("\n\n\nTrainables ", tf.trainable_variables())
tf.add_to_collection("global_step", global_step)
tf.add_to_collection("loss", label_loss)
tf.add_to_collection("predictions", tf.concat(tower_predictions, 0))
tf.add_to_collection("input_batch_raw", model_input_raw)
tf.add_to_collection("input_batch", model_input)
tf.add_to_collection("num_frames", num_frames)
tf.add_to_collection("labels", tf.cast(labels_batch, tf.float32))
tf.add_to_collection("train_op", train_op)
return all_vars_to_save
def variable_with_weight_decay(shape, stddev, wd):
var = tf.Variable(tf.truncated_normal(shape=shape, stddev=stddev))
if wd is not None:
weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def cnn_model(inputs, is_training, get_activations=False):
with tf.variable_scope("cnn_model"):
conv_1 = tf.contrib.layers.conv2d(
inputs,
num_outputs=32,
kernel_size=(
7,
7,
),
stride=(2, 2),
padding="same",
scope="conv_1",
data_format="NCHW",
weights_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004),
biases_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004),
activation_fn=tf.nn.leaky_relu)
if get_activations:
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, conv_1)
dropout_conv_1 = tf.contrib.layers.dropout(conv_1,
keep_prob=1.0,
is_training=is_training,
scope="dropout_conv_1")
conv_2 = tf.contrib.layers.conv2d(
dropout_conv_1,
num_outputs=60,
kernel_size=(
5,
5,
),
stride=(2, 2),
padding="same",
scope="conv_2",
data_format="NCHW",
weights_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004),
biases_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004),
activation_fn=tf.nn.leaky_relu)
dropout_conv_2 = tf.contrib.layers.dropout(conv_2,
keep_prob=1.0,
is_training=is_training,
scope="dropout_conv_2")
conv_3 = tf.contrib.layers.conv2d(
dropout_conv_2,
num_outputs=60,
kernel_size=(
5,
5,
),
stride=(2, 2),
padding="same",
scope="conv_3",
data_format="NCHW",
weights_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004),
biases_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004),
activation_fn=tf.nn.leaky_relu)
conv_4 = tf.contrib.layers.conv2d(
conv_3,
num_outputs=100,
kernel_size=(
3,
3,
),
stride=(2, 2),
padding="same",
scope="conv_4",
data_format="NCHW",
weights_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004),
biases_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004),
activation_fn=tf.nn.leaky_relu)
conv_5 = tf.contrib.layers.conv2d(
conv_4,
num_outputs=200,
kernel_size=(
3,
3,
),
stride=(2, 2),
padding="same",
scope="conv_5",
data_format="NCHW",
weights_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004),
biases_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004),
activation_fn=tf.nn.leaky_relu)
conv_6 = tf.contrib.layers.conv2d(
conv_5,
num_outputs=300,
kernel_size=(
3,
3,
),
stride=(2, 2),
padding="same",
scope="conv_6",
data_format="NCHW",
activation_fn=None,
weights_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004),
biases_regularizer=tf.contrib.layers.l2_regularizer(scale=0.0004))
if get_activations:
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, conv_6)
return conv_6
def add_to_weights(self):
for key in self.vars:
tf.add_to_collection(tf.GraphKeys.WEIGHTS, self.vars[key])
def get_weight(shape, regularizer):
w = tf.Variable(tf.truncated_normal(shape, stddev=0.1))
if regularizer != None:
tf.add_to_collection('losses',
tf.contrib.layers.l2_regularizer(regularizer)(w))
return w
def _variable_with_weight_decay(name, shape, wd):
var = _variable_on_cpu(name, shape, tf.contrib.layers.xavier_initializer())
if wd is not None:
weight_decay = tf.nn.l2_loss(var) * wd
tf.add_to_collection('weightdecay_losses', weight_decay)
return var
def create_model(inputs, output_size):
layer_activations = []
with tf.variable_scope('784-800-800-10'):
temperature = tf.placeholder(tf.float32, name='temperature')
with tf.variable_scope('fc1'):
w = tf.Variable(tf.truncated_normal(
[int(inputs.get_shape()[-1]), 800]),
name='w')
b = tf.Variable(tf.constant(0.1, shape=[800]), name='b')
h = tf.matmul(inputs, w) + b
z = tf.nn.relu(h, name='relu')
layer_activations.append((h, 800))
tf.add_to_collection('fc1_w', w)
tf.add_to_collection('fc1_b', b)
tf.add_to_collection('fc1', h)
with tf.variable_scope('fc2'):
w = tf.Variable(tf.truncated_normal([800, 800]), name='w')
b = tf.Variable(tf.constant(0.1, shape=[800]), name='b')
h = tf.matmul(z, w) + b
z = tf.nn.relu(h, name='relu')
layer_activations.append((h, 800))
tf.add_to_collection('fc2_w', w)
tf.add_to_collection('fc2_b', b)
tf.add_to_collection('fc2', h)
with tf.variable_scope('fc3'):
w = tf.Variable(tf.truncated_normal([800, output_size]), name='w')
b = tf.Variable(tf.constant(0.1, shape=[output_size]), name='b')
h = tf.matmul(z, w) + b
tf.add_to_collection('fc3_w', w)
tf.add_to_collection('fc3_b', b)
with tf.variable_scope('temp'):
h_soft = tf.div(h, temperature)
layer_activations.append((h_soft, output_size))
# if procedure is train, then this model will be the teacher.
# stats and optimize both load up saved models
# if procedure is distill, then we already loaded up a saved model (teacher)
# and we're creating this graph to be the student.
tf.add_to_collection('inputs', inputs)
tf.add_to_collection('outputs', h_soft)
tf.add_to_collection('temperature', temperature)
feed_dicts = create_feed_dicts(temperature)
return h_soft, layer_activations, feed_dicts
def __init__(self, inner_model, dummy_batch, loss_func, metrics):
if hasattr(dummy_batch, '_asdict'):
dummy_batch = dummy_batch._asdict()
# Convert input to tensors, possibly from nested lists that need to be
# converted to a single top-level tensor.
dummy_tensors = collections.OrderedDict([
(k, tf.convert_to_tensor_or_sparse_tensor(v))
for k, v in six.iteritems(dummy_batch)
])
# NOTE: sub-classed `tf.keras.Model`s do not have fully initialized
# variables until they are called on input. We forced that here.
inner_model(dummy_tensors['x'])
def _tensor_spec_with_undefined_batch_dim(tensor):
# Remove the batch dimension and leave it unspecified.
spec = tf.TensorSpec(shape=[None] + tensor.shape.dims[1:],
dtype=tensor.dtype)
return spec
self._input_spec = nest.map_structure(
_tensor_spec_with_undefined_batch_dim, dummy_tensors)
self._keras_model = inner_model
self._loss_fn = loss_func
self._metrics = metrics if metrics is not None else []
# This is defined here so that it closes over the `loss_func`.
class _WeightedMeanLossMetric(keras_metrics.Metric):
"""A `tf.keras.metrics.Metric` wrapper for the loss function."""
def __init__(self, name='loss', dtype=tf.float32):
super(_WeightedMeanLossMetric, self).__init__(name, dtype)
self._total_loss = self.add_weight('total_loss',
initializer='zeros')
self._total_weight = self.add_weight('total_weight',
initializer='zeros')
self._loss_fn = loss_func
def update_state(self, y_true, y_pred, sample_weight=None):
y_true = tf.cast(y_true, self._dtype)
y_pred = tf.cast(y_pred, self._dtype)
# _loss_fn is expected to return the scalar mean loss, so we multiply by
# the batch_size to get back to total loss.
batch_size = tf.cast(tf.shape(y_pred)[0], self._dtype)
batch_total_loss = self._loss_fn(y_true, y_pred) * batch_size
op = self._total_loss.assign_add(batch_total_loss)
with tf.control_dependencies([op]):
return self._total_weight.assign_add(batch_size)
def result(self):
return tf.div_no_nan(self._total_loss, self._total_weight)
self._loss_metric = _WeightedMeanLossMetric()
# Keras creates variables that are not added to any collection, making it
# impossible for TFF to extract them and create the appropriate initializer
# before call a tff.Computation. Here we store them in a TFF specific
# collection so that they can be retrieved later.
# TODO(b/122081673): this likely goes away in TF2.0
for variable in itertools.chain(self.trainable_variables,
self.non_trainable_variables,
self.local_variables):
tf.add_to_collection(
graph_keys.GraphKeys.VARS_FOR_TFF_TO_INITIALIZE, variable)
# dropout
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# softmax
W_fc2 = tf_tools.weight_variable([1024, 23])
b_fc2 = tf_tools.bias_variable([23])
y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
cross_entropy = -tf.reduce_sum(y_ * tf.log(y_conv))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.add_to_collection('pred_network', y_conv)
array = tf_tools.get_files('./train_data')
array = tf_tools.batches(array[0], array[1])
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=1)
time_start = time.time()
for i in range(2000):
batch = tf_tools.get_batche(array, 100)
if i % 100 == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batch[0],
y_: batch[1],
keep_prob: 1.0
})
def _coefficients(cls, input_tensor, params, is_training):
bs = input_tensor.get_shape().as_list()[0]
gd = params['luma_bins']
cm = params['channel_multiplier']
spatial_bin = params['spatial_bin']
# -----------------------------------------------------------------------
with tf.variable_scope('splat'):
n_ds_layers = int(np.log2(params['net_input_size'] / spatial_bin))
current_layer = input_tensor
for i in range(n_ds_layers):
if i > 0: # don't normalize first layer
use_bn = params['batch_norm']
else:
use_bn = False
current_layer = conv(current_layer,
cm * (2**i) * gd,
3,
stride=2,
batch_norm=use_bn,
is_training=is_training,
scope='conv{}'.format(i + 1))
splat_features = current_layer
# -----------------------------------------------------------------------
# -----------------------------------------------------------------------
with tf.variable_scope('global'):
n_global_layers = int(np.log2(spatial_bin /
4)) # 4x4 at the coarsest lvl
current_layer = splat_features
for i in range(2):
current_layer = conv(current_layer,
8 * cm * gd,
3,
stride=2,
batch_norm=params['batch_norm'],
is_training=is_training,
scope="conv{}".format(i + 1))
_, lh, lw, lc = current_layer.get_shape().as_list()
current_layer = tf.reshape(current_layer, [bs, lh * lw * lc])
current_layer = fc(current_layer,
32 * cm * gd,
batch_norm=params['batch_norm'],
is_training=is_training,
scope="fc1")
current_layer = fc(current_layer,
16 * cm * gd,
batch_norm=params['batch_norm'],
is_training=is_training,
scope="fc2")
# don't normalize before fusion
current_layer = fc(current_layer,
8 * cm * gd,
activation_fn=None,
scope="fc3")
global_features = current_layer
# -----------------------------------------------------------------------
# -----------------------------------------------------------------------
with tf.variable_scope('local'):
current_layer = splat_features
current_layer = conv(current_layer,
8 * cm * gd,
3,
batch_norm=params['batch_norm'],
is_training=is_training,
scope='conv1')
# don't normalize before fusion
current_layer = conv(current_layer,
8 * cm * gd,
3,
activation_fn=None,
use_bias=False,
scope='conv2')
grid_features = current_layer
# -----------------------------------------------------------------------
# -----------------------------------------------------------------------
with tf.name_scope('fusion'):
fusion_grid = grid_features
fusion_global = tf.reshape(global_features,
[bs, 1, 1, 8 * cm * gd])
fusion = tf.nn.relu(fusion_grid + fusion_global)
# -----------------------------------------------------------------------
# -----------------------------------------------------------------------
with tf.variable_scope('prediction'):
current_layer = fusion
current_layer = conv(current_layer,
gd * cls.n_out() * cls.n_in(),
1,
activation_fn=None,
scope='conv1')
with tf.name_scope('unroll_grid'):
current_layer = tf.stack(tf.split(current_layer,
cls.n_out() * cls.n_in(),
axis=3),
axis=4)
current_layer = tf.stack(tf.split(current_layer,
cls.n_in(),
axis=4),
axis=5)
tf.add_to_collection('packed_coefficients', current_layer)
# -----------------------------------------------------------------------
return current_layer
def get_weight(shape, regularizer):
w = tf.Variable(tf.random_normal(shape), dtype=tf.float32)
tf.add_to_collection('losses',
tf.contrib.layers.l2_regularizer(regularizer)(w))
return w
def _build_vars(self):
with tf.variable_scope(self._name + str(1)):
nil_word_slot = tf.zeros([1, self._embedding_size])
A = tf.concat(axis=0,
values=[
nil_word_slot,
self._init(
[self._vocab_size - 1, self._embedding_size])
])
B = tf.concat(axis=0,
values=[
nil_word_slot,
self._init(
[self._vocab_size - 1, self._embedding_size])
])
C = tf.concat(axis=0,
values=[
nil_word_slot,
self._init(
[self._vocab_size - 1, self._embedding_size])
])
self.A = tf.Variable(A, name="A")
self.B = tf.Variable(B, name="B")
self.C = tf.Variable(C, name="C")
self.TA = tf.Variable(self._init(
[self._memory_size, self._embedding_size]),
name='TA')
self.TC = tf.Variable(self._init(
[self._memory_size, self._embedding_size]),
name='TC')
self.H = tf.Variable(self._init(
[self._embedding_size, self._embedding_size]),
name="H")
self.W = tf.Variable(self._init(
[self._embedding_size, self._vocab_size]),
name="W")
self._nil_vars = set([self.A.name, self.B.name, self.C.name])
tf.add_to_collection('reg_loss', tf.nn.l2_loss(self.A))
tf.add_to_collection('reg_loss', tf.nn.l2_loss(self.B))
tf.add_to_collection('reg_loss', tf.nn.l2_loss(self.C))
tf.add_to_collection('reg_loss', tf.nn.l2_loss(self.TA))
tf.add_to_collection('reg_loss', tf.nn.l2_loss(self.TC))
tf.add_to_collection('reg_loss', tf.nn.l2_loss(self.W))
tf.add_to_collection('reg_loss', tf.nn.l2_loss(self.H))
b_fc1 = bias_variable([200])
# [n_samples, 7, 7, 32] ->> [n_samples, 7x7x32]
h_pool4_flat = tf.reshape(h_pool4, [-1, 7 * 7 * 32])
h_fc1 = tf.nn.relu(tf.matmul(h_pool4_flat, W_fc1) + b_fc1)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
## func2 layer ##
W_fc2 = weight_variable([200, 15])
b_fc2 = bias_variable([15])
prediction = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2, name='out')
cross_entropy = tf.reduce_mean(
-tf.reduce_sum(ys * tf.log(prediction), reduction_indices=[1])) # loss
train_step = tf.train.AdamOptimizer(1 * (1e-5)).minimize(cross_entropy)
tf.add_to_collection("train_step", train_step)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
img_test = []
label_test = []
fp = open('little/test.txt', "r")
# little/?.jpg 3
line = fp.readline()
dataset = tf.data.Dataset.from_tensor_slices((img_list, label_list))
dataset = dataset.batch(100)
dataset = dataset.shuffle(buffer_size=1)
def lstm(incoming, n_units, activation='sigmoid', inner_activation='tanh',
bias=True, weights_init='truncated_normal', forget_bias=1.0,
return_seq=False, return_states=False, initial_state=None,
trainable=True, restore=True, name="LSTM"):
""" LSTM.
Long Short Term Memory Recurrent Layer.
Input:
3-D Tensor [samples, timesteps, input dim].
Output:
if `return_seq`: 3-D Tensor [samples, timesteps, output dim].
else: 2-D Tensor [samples, output dim].
Arguments:
incoming: `Tensor`. Incoming 3-D Tensor.
n_units: `int`, number of units for this layer.
activation: `str` (name) or `Tensor`. Activation applied to this layer.
(See tflearn.activations). Default: 'sigmoid'.
inner_activation: `str` (name) or `Tensor`. LSTM inner activation.
Default: 'tanh'.
bias: `bool`. If True, a bias is used.
weights_init: `str` (name) or `Tensor`. Weights initialization.
(See tflearn.initializations) Default: 'truncated_normal'.
forget_bias: `float`. Bias of the forget gate. Default: 1.0.
return_seq: `bool`. If True, returns the full sequence instead of
last sequence output only.
return_states: `bool`. If True, returns a tuple with output and
states: (output, states).
initial_state: `Tensor`. An initial state for the RNN. This must be
a tensor of appropriate type and shape [batch_size x cell.state_size].
name: `str`. A name for this layer (optional).
References:
Long Short Term Memory, Sepp Hochreiter & Jurgen Schmidhuber,
Neural Computation 9(8): 1735-1780, 1997.
Links:
[http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf]
(http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf)
"""
input_shape = utils.get_incoming_shape(incoming)
W_init = weights_init
if isinstance(weights_init, str):
W_init = initializations.get(weights_init)()
with tf.name_scope(name) as scope:
cell = BasicLSTMCell(n_units, activation, inner_activation, bias,
W_init, forget_bias, trainable, restore)
inference = incoming
# If a tensor given, convert it to a per timestep list
if type(inference) not in [list, np.array]:
ndim = len(input_shape)
assert ndim >= 3, "Input dim should be at least 3."
axes = [1, 0] + list(range(2, ndim))
inference = tf.transpose(inference, (axes))
inference = tf.unpack(inference)
outputs, states = _rnn(cell, inference, dtype=tf.float32,
initial_state=initial_state, scope=scope[:-1])
# Track per layer variables
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, cell.W)
if bias:
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope,
cell.b)
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[-1])
o = outputs if return_seq else outputs[-1]
s = states if return_seq else states[-1]
return (o, s) if return_states else o
# out_w1 = tf.Variable(tf.truncated_normal([10, 2]))
# out_b1 = tf.Variable(tf.truncated_normal([2]))
# logits = tf.matmul(fully_dropout2, out_w1) + out_b1
# pred = tf.nn.softmax(logits)
out_w1 = tf.Variable(tf.truncated_normal([4, 2]))
out_b1 = tf.Variable(tf.truncated_normal([2]))
logits = tf.matmul(fully_dropout1, out_w1) + out_b1
pred = tf.nn.softmax(logits)
# loss
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=tf_Y, logits=logits, name='cross_entropy')
cross_entropy_mean = tf.reduce_mean(cross_entropy,
name='cross_entropy_loss')
tf.add_to_collection('losses', cross_entropy_mean)
loss = tf.add_n(tf.get_collection('losses'), name='total_loss')
loss_summary = tf.summary.scalar('loss', loss)
# 计算参数数量
var_list = tf.global_variables()
var_trainable_list = tf.trainable_variables()
all_num_params = count_params(var_list, mode2='all')
num_params = count_params(var_trainable_list, mode2='trainable')
config['trainable params'] = num_params
config['all params'] = all_num_params
# train_step
with tf.name_scope('train'):
def bidirectional_rnn(incoming, rnncell_fw, rnncell_bw, return_seq=False,
return_states=False, initial_state_fw=None,
initial_state_bw=None, name="BidirectionalRNN"):
""" Bidirectional RNN.
Build a bidirectional recurrent neural network, it requires 2 RNN Cells
to process sequence in forward and backward order. Any RNN Cell can be
used i.e. SimpleRNN, LSTM, GRU... with its own parameters. But the two
cells number of units must match.
Input:
3-D Tensor Layer [samples, timesteps, input dim].
Output:
if `return_seq`: 3-D Tensor [samples, timesteps, output dim].
else: 2-D Tensor Layer [samples, output dim].
Arguments:
incoming: `Tensor`. The incoming Tensor.
rnncell_fw: `RNNCell`. The RNN Cell to use for foward computation.
rnncell_bw: `RNNCell`. The RNN Cell to use for backward computation.
return_seq: `bool`. If True, returns the full sequence instead of
last sequence output only.
return_states: `bool`. If True, returns a tuple with output and
states: (output, states).
initial_state_fw: `Tensor`. An initial state for the forward RNN.
This must be a tensor of appropriate type and shape [batch_size
x cell.state_size].
initial_state_bw: `Tensor`. An initial state for the backward RNN.
This must be a tensor of appropriate type and shape [batch_size
x cell.state_size].
name: `str`. A name for this layer (optional).
"""
assert (rnncell_fw._num_units == rnncell_bw._num_units), \
"RNN Cells number of units must match!"
with tf.name_scope(name) as scope:
inference = incoming
# If a tensor given, convert it to a per timestep list
if type(inference) not in [list, np.array]:
input_shape = utils.get_incoming_shape(inference)
ndim = len(input_shape)
assert ndim >= 3, "Input dim should be at least 3."
axes = [1, 0] + list(range(2, ndim))
inference = tf.transpose(inference, (axes))
inference = tf.unpack(inference)
outputs, states_fw, states_bw = _bidirectional_rnn(
rnncell_fw, rnncell_bw, inference,
initial_state_fw=initial_state_fw,
initial_state_bw=initial_state_bw,
scope="BiRNN")
c = tf.GraphKeys.LAYER_VARIABLES
for v in [rnncell_fw.W, rnncell_fw.b, rnncell_bw.W, rnncell_bw.b]:
if hasattr(v, "__len__"):
for var in v: tf.add_to_collection(c, var)
else:
tf.add_to_collection(c, v)
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[-1])
o = outputs if return_seq else outputs[-1]
sfw = states_fw if return_seq else states_fw[-1]
sbw = states_fw if return_seq else states_bw[-1]
return (o, sfw, sbw) if return_states else o
def main():
if a.seed is None:
a.seed = random.randint(0, 2**31 - 1)
tf.set_random_seed(a.seed)
np.random.seed(a.seed)
random.seed(a.seed)
if not os.path.exists(a.output_dir):
os.makedirs(a.output_dir)
if a.mode == "test" or a.mode == "export":
if a.checkpoint is None:
raise Exception("checkpoint required for test mode")
# load some options from the checkpoint
options = {"which_direction", "ngf", "ndf", "lab_colorization"}
with open(os.path.join(a.checkpoint, "options.json")) as f:
for key, val in json.loads(f.read()).items():
if key in options:
print("loaded", key, "=", val)
setattr(a, key, val)
# disable these features in test mode
a.scale_size = CROP_SIZE
a.flip = False
for k, v in a._get_kwargs():
print(k, "=", v)
with open(os.path.join(a.output_dir, "options.json"), "w") as f:
f.write(json.dumps(vars(a), sort_keys=True, indent=4))
if a.mode == "export":
# export the generator to a meta graph that can be imported later for standalone generation
if a.lab_colorization:
raise Exception("export not supported for lab_colorization")
input = tf.placeholder(tf.string, shape=[1])
input_data = tf.decode_base64(input[0])
input_image = tf.image.decode_png(input_data)
# remove alpha channel if present
input_image = tf.cond(tf.equal(tf.shape(input_image)[2], 4), lambda: input_image[:,:,:3], lambda: input_image)
# convert grayscale to RGB
input_image = tf.cond(tf.equal(tf.shape(input_image)[2], 1), lambda: tf.image.grayscale_to_rgb(input_image), lambda: input_image)
input_image = tf.image.convert_image_dtype(input_image, dtype=tf.float32)
input_image.set_shape([CROP_SIZE, CROP_SIZE, 3])
batch_input = tf.expand_dims(input_image, axis=0)
with tf.variable_scope("generator"):
batch_output = deprocess(create_generator(preprocess(batch_input), 3))
output_image = tf.image.convert_image_dtype(batch_output, dtype=tf.uint8)[0]
if a.output_filetype == "png":
output_data = tf.image.encode_png(output_image)
elif a.output_filetype == "jpeg":
output_data = tf.image.encode_jpeg(output_image, quality=80)
else:
raise Exception("invalid filetype")
output = tf.convert_to_tensor([tf.encode_base64(output_data)])
key = tf.placeholder(tf.string, shape=[1])
inputs = {
"key": key.name,
"input": input.name
}
tf.add_to_collection("inputs", json.dumps(inputs))
outputs = {
"key": tf.identity(key).name,
"output": output.name,
}
tf.add_to_collection("outputs", json.dumps(outputs))
init_op = tf.global_variables_initializer()
restore_saver = tf.train.Saver()
export_saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init_op)
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
restore_saver.restore(sess, checkpoint)
print("exporting model")
export_saver.export_meta_graph(filename=os.path.join(a.output_dir, "export.meta"))
export_saver.save(sess, os.path.join(a.output_dir, "export"), write_meta_graph=False)
return
examples = load_examples()
print("examples count = %d" % examples.count)
# inputs and targets are [batch_size, height, width, channels]
model = create_model(examples.inputs, examples.targets)
# undo colorization splitting on images that we use for display/output
if a.lab_colorization:
if a.which_direction == "AtoB":
# inputs is brightness, this will be handled fine as a grayscale image
# need to augment targets and outputs with brightness
targets = augment(examples.targets, examples.inputs)
outputs = augment(model.outputs, examples.inputs)
# inputs can be deprocessed normally and handled as if they are single channel
# grayscale images
inputs = deprocess(examples.inputs)
elif a.which_direction == "BtoA":
# inputs will be color channels only, get brightness from targets
inputs = augment(examples.inputs, examples.targets)
targets = deprocess(examples.targets)
outputs = deprocess(model.outputs)
else:
raise Exception("invalid direction")
else:
inputs = deprocess(examples.inputs)
targets = deprocess(examples.targets)
outputs = deprocess(model.outputs)
def convert(image):
if a.aspect_ratio != 1.0:
# upscale to correct aspect ratio
size = [CROP_SIZE, int(round(CROP_SIZE * a.aspect_ratio))]
image = tf.image.resize_images(image, size=size, method=tf.image.ResizeMethod.BICUBIC)
return tf.image.convert_image_dtype(image, dtype=tf.uint8, saturate=True)
# reverse any processing on images so they can be written to disk or displayed to user
with tf.name_scope("convert_inputs"):
converted_inputs = convert(inputs)
with tf.name_scope("convert_targets"):
converted_targets = convert(targets)
with tf.name_scope("convert_outputs"):
converted_outputs = convert(outputs)
with tf.name_scope("encode_images"):
display_fetches = {
"paths": examples.paths,
"inputs": tf.map_fn(tf.image.encode_png, converted_inputs, dtype=tf.string, name="input_pngs"),
"targets": tf.map_fn(tf.image.encode_png, converted_targets, dtype=tf.string, name="target_pngs"),
"outputs": tf.map_fn(tf.image.encode_png, converted_outputs, dtype=tf.string, name="output_pngs"),
}
# summaries
with tf.name_scope("inputs_summary"):
tf.summary.image("inputs", converted_inputs)
with tf.name_scope("targets_summary"):
tf.summary.image("targets", converted_targets)
with tf.name_scope("outputs_summary"):
tf.summary.image("outputs", converted_outputs)
with tf.name_scope("predict_real_summary"):
tf.summary.image("predict_real", tf.image.convert_image_dtype(model.predict_real, dtype=tf.uint8))
with tf.name_scope("predict_fake_summary"):
tf.summary.image("predict_fake", tf.image.convert_image_dtype(model.predict_fake, dtype=tf.uint8))
tf.summary.scalar("discriminator_loss", model.discrim_loss)
tf.summary.scalar("generator_loss_GAN", model.gen_loss_GAN)
tf.summary.scalar("generator_loss_L1", model.gen_loss_L1)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name + "/values", var)
for grad, var in model.discrim_grads_and_vars + model.gen_grads_and_vars:
tf.summary.histogram(var.op.name + "/gradients", grad)
with tf.name_scope("parameter_count"):
parameter_count = tf.reduce_sum([tf.reduce_prod(tf.shape(v)) for v in tf.trainable_variables()])
saver = tf.train.Saver(max_to_keep=1)
logdir = a.output_dir if (a.trace_freq > 0 or a.summary_freq > 0) else None
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=0, saver=None)
with sv.managed_session() as sess:
print("parameter_count =", sess.run(parameter_count))
if a.checkpoint is not None:
print("loading model from checkpoint")
checkpoint = tf.train.latest_checkpoint(a.checkpoint)
saver.restore(sess, checkpoint)
max_steps = 2**32
if a.max_epochs is not None:
max_steps = examples.steps_per_epoch * a.max_epochs
if a.max_steps is not None:
max_steps = a.max_steps
if a.mode == "test":
# testing
# at most, process the test data once
start = time.time()
max_steps = min(examples.steps_per_epoch, max_steps)
for step in range(max_steps):
results = sess.run(display_fetches)
filesets = save_images(results)
for i, f in enumerate(filesets):
print("evaluated image", f["name"])
index_path = append_index(filesets)
print("wrote index at", index_path)
print("rate", (time.time() - start) / max_steps)
else:
# training
start = time.time()
for step in range(max_steps):
def should(freq):
return freq > 0 and ((step + 1) % freq == 0 or step == max_steps - 1)
options = None
run_metadata = None
if should(a.trace_freq):
options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
fetches = {
"train": model.train,
"global_step": sv.global_step,
}
if should(a.progress_freq):
fetches["discrim_loss"] = model.discrim_loss
fetches["gen_loss_GAN"] = model.gen_loss_GAN
fetches["gen_loss_L1"] = model.gen_loss_L1
if should(a.summary_freq):
fetches["summary"] = sv.summary_op
if should(a.display_freq):
fetches["display"] = display_fetches
results = sess.run(fetches, options=options, run_metadata=run_metadata)
if should(a.summary_freq):
print("recording summary")
sv.summary_writer.add_summary(results["summary"], results["global_step"])
if should(a.display_freq):
print("saving display images")
filesets = save_images(results["display"], step=results["global_step"])
append_index(filesets, step=True)
if should(a.trace_freq):
print("recording trace")
sv.summary_writer.add_run_metadata(run_metadata, "step_%d" % results["global_step"])
if should(a.progress_freq):
# global_step will have the correct step count if we resume from a checkpoint
train_epoch = math.ceil(results["global_step"] / examples.steps_per_epoch)
train_step = (results["global_step"] - 1) % examples.steps_per_epoch + 1
rate = (step + 1) * a.batch_size / (time.time() - start)
remaining = (max_steps - step) * a.batch_size / rate
print("progress epoch %d step %d image/sec %0.1f remaining %dm" % (train_epoch, train_step, rate, remaining / 60))
print("discrim_loss", results["discrim_loss"])
print("gen_loss_GAN", results["gen_loss_GAN"])
print("gen_loss_L1", results["gen_loss_L1"])
if should(a.save_freq):
print("saving model")
saver.save(sess, os.path.join(a.output_dir, "model"), global_step=sv.global_step)
if sv.should_stop():
break
def _linear(args, output_size, bias, W=None, b=None, W_init=None,
bias_start=0.0, trainable=True, restore=True, scope=None):
""" Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.
Arguments:
args: a 2D Tensor or a list of 2D, batch x n, Tensors.
output_size: `int`. Second dimension of W[i].
bias: `bool`. Whether to add a bias term or not.
W: `Tensor`. The weights. If None, it will be automatically created.
b: `Tensor`. The bias. If None, it will be automatically created.
W_init: `str`. Weights initialization mode. See
tflearn.initializations.
bias_start: starting value to initialize the bias; 0 by default.
scope: VariableScope for the created subgraph; defaults to "Linear".
Returns:
A `tuple` containing:
- W: `Tensor` variable holding the weights.
- b: `Tensor` variable holding the bias.
- res: `2D tf.Tensor` with shape [batch x output_size] equal to
sum_i(args[i] * W[i]).
"""
# Creates W if it hasn't be created yet.
if not W:
assert args
if not isinstance(args, (list, tuple)):
args = [args]
# Calculate the total size of arguments on dimension 1.
total_arg_size = 0
shapes = [a.get_shape().as_list() for a in args]
for shape in shapes:
if len(shape) != 2:
raise ValueError(
"Linear is expecting 2D arguments: %s" % str(shapes))
if not shape[1]:
raise ValueError(
"Linear expects shape[1] of arguments: %s" % str(shapes))
else:
total_arg_size += shape[1]
with tf.variable_scope(scope, reuse=False):
W = tf.get_variable(name="W", shape=[total_arg_size, output_size],
initializer=W_init, trainable=trainable)
if not restore:
tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, W)
# Now the computation.
if len(args) == 1:
res = tf.matmul(args[0], W)
else:
res = tf.matmul(array_ops.concat(1, args), W)
if not bias:
return W, None, res
# Creates b if it hasn't be created yet.
if not b:
with tf.variable_scope(scope, reuse=False) as vs:
b = tf.get_variable(
"b", [output_size],
initializer=init_ops.constant_initializer(bias_start),
trainable=trainable)
if not restore:
tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, b)
return W, b, res + b
def dynamic_rnn(incoming, rnncell, sequence_length=None, time_major=False,
return_seq=False, return_states=False, initial_state=None,
name="DynamicRNN"):
""" Dynamic RNN.
RNN with dynamic sequence length.
Unlike `rnn`, the input `incoming` is not a Python list of `Tensors`.
Instead, it is a single `Tensor` where the maximum time is either the
first or second dimension (see the parameter `time_major`). The
corresponding output is a single `Tensor` having the same number of time
steps and batch size.
The parameter `sequence_length` is required and dynamic calculation is
automatically performed.
Input:
3-D Tensor Layer [samples, timesteps, input dim].
Output:
if `return_seq`: 3-D Tensor [samples, timesteps, output dim].
else: 2-D Tensor Layer [samples, output dim].
Arguments:
incoming: `Tensor`. The incoming 3-D Tensor.
rnncell: `RNNCell`. The RNN Cell to use for computation.
sequence_length: `int32` `Tensor`. A Tensor of shape [batch_size].
(Optional).
time_major: The shape format of the `inputs` and `outputs` Tensors.
If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`.
If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`.
Using time_major = False is a bit more efficient because it avoids
transposes at the beginning and end of the RNN calculation. However,
most TensorFlow data is batch-major, so by default this function
accepts input and emits output in batch-major form.
return_seq: `bool`. If True, returns the full sequence instead of
last sequence output only.
return_states: `bool`. If True, returns a tuple with output and
states: (output, states).
initial_state: `Tensor`. An initial state for the RNN. This must be
a tensor of appropriate type and shape [batch_size x cell.state_size].
name: `str`. A name for this layer (optional).
"""
# Variables initialization
with tf.name_scope(name) as scope:
inference = incoming
# If a tensor given, convert it to a per timestep list
if type(inference) not in [list, np.array]:
input_shape = utils.get_incoming_shape(inference)
ndim = len(input_shape)
assert ndim >= 3, "Input dim should be at least 3."
axes = [1, 0] + list(range(2, ndim))
inference = tf.transpose(inference, (axes))
inference = tf.unpack(inference)
outputs, final_state = _dynamic_rnn(rnncell, inference,
initial_state=initial_state,
sequence_length=sequence_length,
time_major=time_major,
scope="DynamicRNN")
c = tf.GraphKeys.LAYER_VARIABLES
for v in [rnncell.W, rnncell.b]:
if hasattr(v, "__len__"):
for var in v: tf.add_to_collection(c, var)
else:
tf.add_to_collection(c, v)
o = outputs if return_seq else outputs[-1]
s = final_state # Only final_state available
return (o, s) if return_states else o
def inference(input_tensor, train, regularizer):
with tf.variable_scope('layer1-fc1'):
fc1_weights = tf.get_variable("weight", [INPUT_NODE, FC1_SIZE],
initializer = tf.truncated_normal_initializer(stddev = 0.1))
# 只有全连接层的权重需要加入正则化
if regularizer != None:
tf.add_to_collection('losses', regularizer(fc1_weights))
fc1_biases = tf.get_variable('bias', [FC1_SIZE], initializer = tf.constant_initializer(0.1))
fc1 = tf.nn.relu(tf.matmul(input_tensor, fc1_weights)+fc1_biases)
with tf.name_scope('layer2-fc2'):
fc2_weights = tf.get_variable("weight", [FC1_SIZE, FC2_SIZE],
initializer=tf.truncated_normal_initializer(stddev=0.1))
# 只有全连接层的权重需要加入正则化
if regularizer != None:
tf.add_to_collection('losses', regularizer(fc2_weights))
fc2_biases = tf.get_variable('bias', [FC2_SIZE], initializer=tf.constant_initializer(0.1))
fc2 = tf.nn.relu(tf.matmul(fc1, fc2_weights) + fc2_biases)
with tf.variable_scope('layer3-fc3'):
fc3_weights = tf.get_variable("weight", [FC2_SIZE, FC3_SIZE],
initializer=tf.truncated_normal_initializer(stddev=0.1))
# 只有全连接层的权重需要加入正则化
if regularizer != None:
tf.add_to_collection('losses', regularizer(fc3_weights))
fc3_biases = tf.get_variable('bias', [FC3_SIZE], initializer=tf.constant_initializer(0.1))
fc3 = tf.nn.relu(tf.matmul(fc2, fc3_weights) + fc3_biases)
with tf.variable_scope('layer4-fc4'):
fc4_weights = tf.get_variable("weight", [FC3_SIZE, FC4_SIZE],
initializer=tf.truncated_normal_initializer(stddev=0.1))
# 只有全连接层的权重需要加入正则化
if regularizer != None:
tf.add_to_collection('losses', regularizer(fc4_weights))
fc4_biases = tf.get_variable('bias', [FC4_SIZE], initializer=tf.constant_initializer(0.1))
fc4 = tf.nn.relu(tf.matmul(fc3, fc4_weights) + fc4_biases)
with tf.variable_scope('layer5-fc5'):
fc5_weights = tf.get_variable("weight", [FC4_SIZE, FC5_SIZE],
initializer=tf.truncated_normal_initializer(stddev=0.1))
# 只有全连接层的权重需要加入正则化
if regularizer != None:
tf.add_to_collection('losses', regularizer(fc5_weights))
fc5_biases = tf.get_variable('bias', [FC5_SIZE], initializer=tf.constant_initializer(0.1))
fc5 = tf.nn.relu(tf.matmul(fc4, fc5_weights) + fc5_biases)
with tf.variable_scope('layer6-fc6'):
fc6_weights = tf.get_variable("weight", [FC5_SIZE, OUTPUT_NODE],
initializer=tf.truncated_normal_initializer(stddev=0.1))
# 只有全连接层的权重需要加入正则化
if regularizer != None:
tf.add_to_collection('losses', regularizer(fc6_weights))
fc6_biases = tf.get_variable('bias', [OUTPUT_NODE], initializer=tf.constant_initializer(0.1))
logit = tf.matmul(fc5, fc6_weights) + fc6_biases
return logit
def simple_rnn(incoming, n_units, activation='sigmoid', bias=True,
weights_init='truncated_normal', return_seq=False,
return_states=False, initial_state=None, trainable=True,
restore=True, name="SimpleRNN"):
""" Simple RNN.
Simple Recurrent Layer.
Input:
3-D Tensor [samples, timesteps, input dim].
Output:
if `return_seq`: 3-D Tensor [samples, timesteps, output dim].
else: 2-D Tensor [samples, output dim].
Arguments:
incoming: `Tensor`. Incoming 3-D Tensor.
n_units: `int`, number of units for this layer.
activation: `str` (name) or `Tensor`. Activation applied to this layer.
(See tflearn.activations). Default: 'sigmoid'.
bias: `bool`. If True, a bias is used.
weights_init: `str` (name) or `Tensor`. Weights initialization.
(See tflearn.initializations) Default: 'truncated_normal'.
return_seq: `bool`. If True, returns the full sequence instead of
last sequence output only.
return_states: `bool`. If True, returns a tuple with output and
states: (output, states).
initial_state: `Tensor`. An initial state for the RNN. This must be
a tensor of appropriate type and shape [batch_size x cell.state_size].
name: `str`. A name for this layer (optional).
"""
input_shape = utils.get_incoming_shape(incoming)
W_init = weights_init
if isinstance(weights_init, str):
W_init = initializations.get(weights_init)()
with tf.name_scope(name) as scope:
cell = BasicRNNCell(n_units, activation, bias, W_init,
trainable, restore)
inference = incoming
# If a tensor given, convert it to a per timestep list
if type(inference) not in [list, np.array]:
ndim = len(input_shape)
assert ndim >= 3, "Input dim should be at least 3."
axes = [1, 0] + list(range(2, ndim))
inference = tf.transpose(inference, (axes))
inference = tf.unpack(inference)
# Track per layer variables
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope,
cell.W)
if bias:
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope,
cell.b)
outputs, states = _rnn(cell, inference, dtype=tf.float32,
initial_state=initial_state, scope=scope[:-1])
# Track activations.
tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, outputs[-1])
o = outputs if return_seq else outputs[-1]
s = states if return_seq else states[-1]
return (o, s) if return_states else o
def vgg16(x, keep_prob):
tensors = []
# build_network
W_conv1_1 = tf.get_variable(
'conv1_1',
shape=[3, 3, 3, 64],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv1_1 = bias_variable("bias1_1", [64])
t_conv1_1 = threshold_variable("tconv1_1", shape=[3, 3, 3, 64])
t_bias1_1 = threshold_variable("tbias1_1", shape=[64])
tensors += [W_conv1_1, b_conv1_1, t_conv1_1, t_bias1_1]
output = tf.nn.relu(
batch_norm(conv2d(x, W_conv1_1 * t_conv1_1) + b_conv1_1 * t_bias1_1))
W_conv1_2 = tf.get_variable(
'conv1_2',
shape=[3, 3, 64, 64],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv1_2 = bias_variable("bias1_2", [64])
t_conv1_2 = threshold_variable("tconv1_2", shape=[3, 3, 64, 64])
t_bias1_2 = threshold_variable("tbias1_2", shape=[64])
tensors += [W_conv1_2, b_conv1_2, t_conv1_2, t_bias1_2]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv1_2 * t_conv1_2) + b_conv1_2 * t_bias1_2))
output = max_pool(output, 2, 2, "pool1")
# out :16
W_conv2_1 = tf.get_variable(
'conv2_1',
shape=[3, 3, 64, 128],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv2_1 = bias_variable("bias2_1", [128])
t_conv2_1 = threshold_variable("tconv2_1", shape=[3, 3, 64, 128])
t_bias2_1 = threshold_variable("tbias2_1", shape=[128])
tensors += [W_conv2_1, b_conv2_1, t_conv2_1, t_bias2_1]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv2_1 * t_conv2_1) + b_conv2_1 * t_bias2_1))
W_conv2_2 = tf.get_variable(
'conv2_2',
shape=[3, 3, 128, 128],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv2_2 = bias_variable("bias2_2", [128])
t_conv2_2 = threshold_variable("tconv2_2", shape=[3, 3, 128, 128])
t_bias2_2 = threshold_variable("tbias2_2", shape=[128])
tensors += [W_conv2_2, b_conv2_2, t_conv2_2, t_bias2_2]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv2_2 * t_conv2_2) + b_conv2_2 * t_bias2_2))
output = max_pool(output, 2, 2, "pool2")
# out :8
W_conv3_1 = tf.get_variable(
'conv3_1',
shape=[3, 3, 128, 256],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv3_1 = bias_variable("bias3_1", [256])
t_conv3_1 = threshold_variable("tconv3_1", shape=[3, 3, 128, 256])
t_bias3_1 = threshold_variable("tbias3_1", shape=[256])
tensors += [W_conv3_1, b_conv3_1, t_conv3_1, t_bias3_1]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv3_1 * t_conv3_1) + b_conv3_1 * t_bias3_1))
# output = tf.nn.relu(conv2d(output, W_conv3_1) + b_conv3_1)
W_conv3_2 = tf.get_variable(
'conv3_2',
shape=[3, 3, 256, 256],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv3_2 = bias_variable("bias3_2", [256])
t_conv3_2 = threshold_variable("tconv3_2", shape=[3, 3, 256, 256])
t_bias3_2 = threshold_variable("tbias3_2", shape=[256])
tensors += [W_conv3_2, b_conv3_2, t_conv3_2, t_bias3_2]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv3_2 * t_conv3_2) + b_conv3_2 * t_bias3_2))
# output = tf.nn.relu(conv2d(output, W_conv3_2) + b_conv3_2)
W_conv3_3 = tf.get_variable(
'conv3_3',
shape=[3, 3, 256, 256],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv3_3 = bias_variable("bias3_3", [256])
t_conv3_3 = threshold_variable("tconv3_3", shape=[3, 3, 256, 256])
t_bias3_3 = threshold_variable("tbias3_3", shape=[256])
tensors += [W_conv3_3, b_conv3_3, t_conv3_3, t_bias3_3]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv3_3 * t_conv3_3) + b_conv3_3 * t_bias3_3))
# output = tf.nn.relu(conv2d(output, W_conv3_3) + b_conv3_3)
output = max_pool(output, 2, 2, "pool3")
# out :4
W_conv4_1 = tf.get_variable(
'conv4_1',
shape=[3, 3, 256, 512],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv4_1 = bias_variable("bias4_1", [512])
t_conv4_1 = threshold_variable("tconv4_1", shape=[3, 3, 256, 512])
t_bias4_1 = threshold_variable("tbias4_1", shape=[512])
tensors += [W_conv4_1, b_conv4_1, t_conv4_1, t_bias4_1]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv4_1 * t_conv4_1) + b_conv4_1 * t_bias4_1))
# output = tf.nn.relu(conv2d(output, W_conv4_1) + b_conv4_1)
W_conv4_2 = tf.get_variable(
'conv4_2',
shape=[3, 3, 512, 512],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv4_2 = bias_variable("bias4_2", [512])
t_conv4_2 = threshold_variable("tconv4_2", shape=[3, 3, 512, 512])
t_bias4_2 = threshold_variable("tbias4_2", shape=[512])
tensors += [W_conv4_2, b_conv4_2, t_conv4_2, t_bias4_2]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv4_2 * t_conv4_2) + b_conv4_2 * t_bias4_2))
# output = tf.nn.relu(conv2d(output, W_conv4_2) + b_conv4_2)
W_conv4_3 = tf.get_variable(
'conv4_3',
shape=[3, 3, 512, 512],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv4_3 = bias_variable("bias4_3", [512])
t_conv4_3 = threshold_variable("tconv4_3", shape=[3, 3, 512, 512])
t_bias4_3 = threshold_variable("tbias4_3", shape=[512])
tensors += [W_conv4_3, b_conv4_3, t_conv4_3, t_bias4_3]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv4_3 * t_conv4_3) + b_conv4_3 * t_bias4_3))
# output = tf.nn.relu(conv2d(output, W_conv4_3) + b_conv4_3)
output = max_pool(output, 2, 2)
# out :2
W_conv5_1 = tf.get_variable(
'conv5_1',
shape=[3, 3, 512, 512],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv5_1 = bias_variable("bias5_1", [512])
t_conv5_1 = threshold_variable("tconv5_1", shape=[3, 3, 512, 512])
t_bias5_1 = threshold_variable("tbias5_1", shape=[512])
tensors += [W_conv5_1, b_conv5_1, t_conv5_1, t_bias5_1]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv5_1 * t_conv5_1) + b_conv5_1 * t_bias5_1))
# output = tf.nn.relu(conv2d(output, W_conv5_1) + b_conv5_1)
W_conv5_2 = tf.get_variable(
'conv5_2',
shape=[3, 3, 512, 512],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv5_2 = bias_variable("bias5_2", [512])
t_conv5_2 = threshold_variable("tconv5_2", shape=[3, 3, 512, 512])
t_bias5_2 = threshold_variable("tbias5_2", shape=[512])
tensors += [W_conv5_2, b_conv5_2, t_conv5_2, t_bias5_2]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv5_2 * t_conv5_2) + b_conv5_2 * t_bias5_2))
# output = tf.nn.relu(conv2d(output, W_conv5_2) + b_conv5_2)
W_conv5_3 = tf.get_variable(
'conv5_3',
shape=[3, 3, 512, 512],
initializer=tf.contrib.keras.initializers.he_normal())
b_conv5_3 = bias_variable("bias5_3", [512])
t_conv5_3 = threshold_variable("tconv5_3", shape=[3, 3, 512, 512])
t_bias5_3 = threshold_variable("tbias5_3", shape=[512])
tensors += [W_conv5_3, b_conv5_3, t_conv5_3, t_bias5_3]
output = tf.nn.relu(
batch_norm(
conv2d(output, W_conv5_3 * t_conv5_3) + b_conv5_3 * t_bias5_3))
# output = tf.nn.relu(conv2d(output, W_conv5_3) + b_conv5_3)
# output = max_pool(output, 2, 2)
# output = tf.contrib.layers.flatten(output)
flatten_output = tf.reshape(output, [-1, 2 * 2 * 512])
W_fc1 = tf.get_variable(
'fc1',
shape=[2048, 4096],
initializer=tf.contrib.keras.initializers.he_normal())
b_fc1 = bias_variable("fc1_b", [4096])
output = tf.nn.relu(batch_norm(tf.matmul(flatten_output, W_fc1) + b_fc1))
# output = tf.nn.relu(tf.matmul(flatten_output, W_fc1) + b_fc1)
output = tf.nn.dropout(output, keep_prob)
W_fc2 = tf.get_variable(
'fc2',
shape=[4096, 4096],
initializer=tf.contrib.keras.initializers.he_normal())
b_fc2 = bias_variable('fc2_b', [4096])
output = tf.nn.relu(batch_norm(tf.matmul(output, W_fc2) + b_fc2))
# output = tf.nn.relu(tf.matmul(output, W_fc2) + b_fc2)
output = tf.nn.dropout(output, keep_prob)
W_fc3 = tf.get_variable(
'fc3',
shape=[4096, 100],
initializer=tf.contrib.keras.initializers.he_normal())
b_fc3 = bias_variable('fc3_b', [100])
output = tf.nn.relu(batch_norm(tf.matmul(output, W_fc3) + b_fc3))
for i in range(12):
#tf.add_to_collection("losses",tf.nn.l2_loss(tensors[4 * i]*tensors[4 * i+2])*weight_decay)
#tf.add_to_collection('losses', groupcrosslasso(tensors[4 * i]*tensors[4 * i+2], tensors[4*(i+1)]*tensors[4*(i+1)+2],weight_decay))
tf.add_to_collection(
'losses',
grouplasso(tensors[4 * i] * tensors[4 * i + 2], weight_decay1,
weight_decay2))
return output, tensors
def train(self):
sess = tf.Session()
if self.load_model_directory == None:
# --Model parameters--
# Encoder
rng = 1.0 / math.sqrt(float(self.input_dim + self.attr_dim))
W = tf.Variable(tf.random_uniform([self.input_dim, self.attr_dim],
minval=-rng,
maxval=rng),
name="W")
# --Data and prior--
# Input image features, shape = [batch_size, input_dim]
X = tf.placeholder(tf.float32, [None, self.input_dim], name="X")
# Text attributes describing the class, shape [batch_size, attr_dim]
T = tf.placeholder(tf.float32, [None, self.attr_dim], name="T")
# Test time, the text features/attributes of a single unseen class
t = tf.placeholder(tf.float32, [self.attr_dim], name="t")
# --Build model--
# Autoencoder---it's more like domain transfer function
T_from_X = tf.matmul(X, W)
X_from_T = tf.matmul(T, tf.transpose(W))
# Match loss
# Frobenious norm of a batch of vector differences
match_loss = tf.reduce_mean(tf.pow(T_from_X - T, 2))
tf.add_to_collection("match_loss", match_loss)
# Reconstruction loss
# L2 norm of vector difference, average over size of batch
recon_loss = tf.reduce_mean(tf.pow(X_from_T - X, 2))
tf.add_to_collection("recon_loss", recon_loss)
# Training objectives
train_step = tf.train.AdagradOptimizer(
self.lrn_rate).minimize(recon_loss +
self.coef_match * match_loss)
tf.add_to_collection("train_step", train_step)
# Test time
dist_from_t = tf.reduce_sum(tf.pow(T_from_X - t, 2), axis=1)
tf.add_to_collection("dist_from_t", dist_from_t)
# --Set up graph--
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
else:
# --Load graph--
saver = tf.train.import_meta_graph(self.load_model_directory +
"/log.meta")
saver.restore(
sess, tf.train.latest_checkpoint(self.load_model_directory))
graph = tf.get_default_graph()
W = graph.get_tensor_by_name("W:0")
X = graph.get_tensor_by_name("X:0")
T = graph.get_tensor_by_name("T:0")
t = graph.get_tensor_by_name("t:0")
match_loss = tf.get_collection("match_loss")[0]
recon_loss = tf.get_collection("recon_loss")[0]
train_step = tf.get_collection("train_step")[0]
dist_from_t = tf.get_collection("dist_from_t")[0]
epoch = 0
log_file = open(self.log_file_name_head + ".txt", "w+")
self.write_log(log_file,
sess,
X,
T,
t,
match_loss,
recon_loss,
dist_from_t,
epoch=epoch,
epoch_time=0.0,
total_time=0.0,
eval_test=True)
total_time_begin = time.time()
for epoch in range(1, self.epoch_max + 1):
epoch_time_begin = time.time()
self.data.initialize_batch("train",
batch_size=self.train_batch_size)
while self.data.has_next_batch():
X_batch, Y_batch, _, _ = self.data.next_batch()
T_batch = self.attr_data.get_batch("test", Y_batch)
feed_dict = {X: X_batch, T: T_batch}
sess.run(train_step, feed_dict=feed_dict)
# End of all mini-batches in an epoch
epoch_time = time.time() - epoch_time_begin
total_time = time.time() - total_time_begin
if epoch % self.save_model_period == 0:
saver.save(sess, self.log_file_name_head)
self.write_log(log_file,
sess,
X,
T,
t,
match_loss,
recon_loss,
dist_from_t,
epoch=epoch,
epoch_time=epoch_time,
total_time=total_time,
eval_test=True)
else:
self.write_log(log_file,
sess,
X,
T,
t,
match_loss,
recon_loss,
dist_from_t,
epoch=epoch,
epoch_time=epoch_time,
total_time=total_time,
eval_test=False)
# End of all epochs
log_file.close()
def generator(x, isTrain=True, reuse=False):
with tf.variable_scope('generator', reuse=reuse):
# 1st hidden layer
conv1 = tf.layers.conv2d_transpose(x,
filter_num[0], [4, 4],
strides=(2, 2),
padding='same',
name='conv1')
relu1 = tf.nn.relu(tf.layers.batch_normalization(conv1,
training=isTrain),
name='relu1')
# 2nd hidden layer
conv2 = tf.layers.conv2d_transpose(relu1,
filter_num[1], [4, 4],
strides=(2, 2),
padding='same',
name='conv2')
relu2 = tf.nn.relu(tf.layers.batch_normalization(conv2,
training=isTrain),
name='relu2')
# 3rd hidden layer
conv3 = tf.layers.conv2d_transpose(relu2,
filter_num[2], [4, 4],
strides=(2, 2),
padding='same',
name='conv3')
relu3 = tf.nn.relu(tf.layers.batch_normalization(conv3,
training=isTrain),
name='relu3')
# 4th hidden layer
conv4 = tf.layers.conv2d_transpose(relu3,
filter_num[3], [4, 4],
strides=(2, 2),
padding='same',
name='conv4')
relu4 = tf.nn.relu(tf.layers.batch_normalization(conv4,
training=isTrain),
name='relu4')
# conv5 = tf.layers.conv2d_transpose(relu4, filter_num[4], [4, 4], strides=(2, 2), padding='same')
# relu5 = relu(tf.layers.batch_normalization(conv5, training=isTrain))
# output layer
conv6 = tf.layers.conv2d_transpose(relu4,
1, [4, 4],
strides=(2, 2),
padding='same',
name='conv6')
o = tf.nn.tanh(conv6, name='output')
tf.add_to_collection('G_conv1', conv1)
tf.add_to_collection('G_relu1', relu1)
tf.add_to_collection('G_conv2', conv2)
tf.add_to_collection('G_relu2', relu2)
tf.add_to_collection('G_conv3', conv3)
tf.add_to_collection('G_relu3', relu3)
tf.add_to_collection('G_conv4', conv4)
tf.add_to_collection('G_relu4', relu4)
tf.add_to_collection('G_conv6', conv6)
tf.add_to_collection('G_z', o)
return o
def weight_variable(shape,wd=0.0):
initial = tf.truncated_normal(shape, stddev=0.1)
var = tf.Variable(initial)
weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
tf.add_to_collection('losses', weight_decay)
return var
def input_fn(self,
mode,
hparams,
data_dir=None,
params=None,
config=None,
force_repeat=False,
prevent_repeat=False,
dataset_kwargs=None):
"""Builds input pipeline for problem.
Args:
mode: tf.estimator.ModeKeys
hparams: HParams, model hparams
data_dir: str, data directory; if None, will use hparams.data_dir
params: dict, may include "batch_size"
config: RunConfig; should have the data_parallelism attribute if not using
TPU
force_repeat: bool, whether to repeat the data even if not training
prevent_repeat: bool, whether to not repeat when in training mode.
Overrides force_repeat.
dataset_kwargs: dict, if passed, will pass as kwargs to self.dataset
method when called
Returns:
(features_dict<str name, Tensor feature>, Tensor targets)
"""
partition_id, num_partitions = self._dataset_partition(mode, config)
is_training = mode == tf.estimator.ModeKeys.TRAIN
if config and config.use_tpu:
num_threads = 64
else:
num_threads = cpu_count() if is_training else 1
if config and hasattr(config,
"data_parallelism") and config.data_parallelism:
num_shards = config.data_parallelism.n
else:
num_shards = 1
max_length = self.max_length(hparams)
mlperf_log.transformer_print(key=mlperf_log.INPUT_MAX_LENGTH,
value=max_length)
def tpu_valid_size(example):
return data_reader.example_valid_size(example, hparams.min_length,
max_length)
def gpu_valid_size(example):
drop_long_sequences = is_training or hparams.eval_drop_long_sequences
max_validate_length = max_length if drop_long_sequences else 10**9
return data_reader.example_valid_size(example, hparams.min_length,
max_validate_length)
def define_shapes(example):
batch_size = config and config.use_tpu and params["batch_size"]
return standardize_shapes(example, batch_size=batch_size)
# Read and preprocess
data_dir = data_dir or (hasattr(hparams, "data_dir")
and hparams.data_dir)
dataset_kwargs = dataset_kwargs or {}
dataset_kwargs.update({
"mode": mode,
"data_dir": data_dir,
"num_threads": num_threads,
"hparams": hparams,
"partition_id": partition_id,
"num_partitions": num_partitions,
})
dataset = self.dataset(**dataset_kwargs)
if (force_repeat or is_training) and not prevent_repeat:
# Repeat and skip a random number of records
dataset = dataset.repeat()
if is_training and self.skip_random_fraction_when_training:
data_files = tf.contrib.slim.parallel_reader.get_data_files(
self.filepattern(data_dir, mode))
# In continuous_train_and_eval when switching between train and
# eval, this input_fn method gets called multiple times and it
# would give you the exact same samples from the last call
# (because the Graph seed is set). So this skip gives you some
# shuffling.
dataset = skip_random_fraction(dataset, data_files[0])
dataset = dataset.map(data_reader.cast_ints_to_int32,
num_parallel_calls=num_threads)
if self.batch_size_means_tokens:
batch_size_means_tokens = True
else:
if _are_shapes_fully_defined(dataset.output_shapes):
batch_size_means_tokens = False
else:
tf.logging.warning(
"Shapes are not fully defined. Assuming batch_size means tokens."
)
batch_size_means_tokens = True
# Batching
if not batch_size_means_tokens:
# Batch size means examples per datashard.
if config and config.use_tpu:
# on TPU, we use params["batch_size"], which specifies the number of
# examples across all datashards
batch_size = params["batch_size"]
dataset = dataset.batch(batch_size, drop_remainder=True)
else:
batch_size = hparams.batch_size * num_shards
dataset = dataset.batch(batch_size)
else:
# batch_size means tokens per datashard
if config and config.use_tpu:
dataset = dataset.filter(tpu_valid_size)
padded_shapes = self._pad_for_tpu(dataset.output_shapes,
hparams)
# on TPU, we use params["batch_size"], which specifies the number of
# examples across all datashards
batch_size = params["batch_size"]
if hparams.pad_batch:
tf.logging.warn(
"Padding the batch to ensure that remainder eval batches are "
"processed. This may lead to incorrect metrics for "
"non-zero-padded features, e.g. images. Use a smaller batch "
"size that has no remainder in that case.")
dataset = dataset.padded_batch(batch_size,
padded_shapes,
drop_remainder=False)
dataset = dataset.map(functools.partial(
pad_batch, batch_multiple=batch_size),
num_parallel_calls=num_threads)
else:
dataset = dataset.padded_batch(batch_size,
padded_shapes,
drop_remainder=True)
else:
# On GPU, bucket by length
dataset = dataset.filter(gpu_valid_size)
batching_scheme = data_reader.hparams_to_batching_scheme(
hparams,
shard_multiplier=num_shards,
length_multiplier=self.get_hparams().batch_size_multiplier)
if hparams.use_fixed_batch_size:
# Here batch_size really means examples per datashard.
batching_scheme["batch_sizes"] = [hparams.batch_size]
batching_scheme["boundaries"] = []
dataset = dataset.apply(
tf.data.experimental.bucket_by_sequence_length(
data_reader.example_length,
batching_scheme["boundaries"],
batching_scheme["batch_sizes"]))
if not is_training:
batch_multiple = num_shards
if hparams.use_fixed_batch_size:
# Make sure the last batch has the same fixed size as the rest.
batch_multiple *= hparams.batch_size
if batch_multiple > 1:
tf.logging.warn(
"Padding the batch to ensure that remainder eval batches have "
"a batch size divisible by the number of data shards. This may "
"lead to incorrect metrics for non-zero-padded features, e.g. "
"images. Use a single datashard (i.e. 1 GPU) in that case."
)
dataset = dataset.map(functools.partial(
pad_batch, batch_multiple=batch_multiple),
num_parallel_calls=num_threads)
dataset = dataset.map(define_shapes, num_parallel_calls=num_threads)
# Add shuffling for training batches. This is necessary along with record
# level shuffling in the dataset generation. Record shuffling will shuffle
# the examples. However, in some cases, it's possible that the shuffle
# buffer size for record shuffling is smaller than the batch size. In such
# cases, adding batch shuffling ensures that the data is in random order
# during training
if (is_training and hasattr(hparams, "batch_shuffle_size")
and hparams.batch_shuffle_size):
dataset = dataset.shuffle(hparams.batch_shuffle_size)
def prepare_for_output(example):
if not config or not config.use_tpu:
_summarize_features(example, num_shards)
if mode == tf.estimator.ModeKeys.PREDICT:
example["infer_targets"] = example.pop("targets")
return example
else:
return example, example["targets"]
dataset = dataset.map(prepare_for_output,
num_parallel_calls=num_threads)
dataset = dataset.prefetch(2)
if mode == tf.estimator.ModeKeys.PREDICT:
# This is because of a bug in the Estimator that short-circuits prediction
# if it doesn't see a QueueRunner. DummyQueueRunner implements the
# minimal expected interface but does nothing.
tf.add_to_collection(tf.GraphKeys.QUEUE_RUNNERS,
data_reader.DummyQueueRunner())
return dataset
def discriminator(x, isTrain=True, reuse=False):
with tf.variable_scope('discriminator', reuse=reuse):
# 1st hidden layer
# conv0 = tf.layers.conv2d(x, filter_num[4], [4, 4], strides=(2, 2), padding='same')
# lrelu0 = lrelu(conv0, 0.2)
conv1 = tf.layers.conv2d(x,
filter_num[3], [4, 4],
strides=(2, 2),
padding='same',
name='conv1')
lrelu1 = lrelu(
tf.layers.batch_normalization(conv1, training=isTrain, name='bn1'),
0.2)
# 2nd hidden layer
conv2 = tf.layers.conv2d(lrelu1,
filter_num[2], [4, 4],
strides=(2, 2),
padding='same',
name='conv2')
lrelu2 = lrelu(
tf.layers.batch_normalization(conv2, training=isTrain, name='bn2'),
0.2)
# 3rd hidden layer
conv3 = tf.layers.conv2d(lrelu2,
filter_num[1], [4, 4],
strides=(2, 2),
padding='same',
name='conv3')
lrelu3 = lrelu(
tf.layers.batch_normalization(conv3, training=isTrain, name='bn3'),
0.2)
# 4th hidden layer
conv4 = tf.layers.conv2d(lrelu3,
filter_num[0], [4, 4],
strides=(2, 2),
padding='same',
name='conv4')
lrelu4 = lrelu(
tf.layers.batch_normalization(conv4, training=isTrain, name='bn4'),
0.2)
# output layer
conv5 = tf.layers.conv2d(lrelu4,
1, [8, 8],
strides=(1, 1),
padding='valid',
name='conv5')
o = tf.nn.sigmoid(conv5, name='output')
tf.add_to_collection('D_conv1', conv1)
tf.add_to_collection('D_lrelu1', lrelu1)
tf.add_to_collection('D_conv2', conv2)
tf.add_to_collection('D_lrelu2', lrelu2)
tf.add_to_collection('D_conv3', conv3)
tf.add_to_collection('D_lrelu3', lrelu3)
tf.add_to_collection('D_conv4', conv4)
tf.add_to_collection('D_lrelu4', lrelu4)
tf.add_to_collection('D_conv5', conv5)
tf.add_to_collection('D_z', o)
return o, conv5
