def create_model_trainable(self):
"""
The model definition.
"""
outputs, reg = self.nn(self.input_placeholder)
self.predictions = outputs
with tf.name_scope('CostFunction'):
with tf.name_scope('Q_Vals'):
self.q_vals = tf.reduce_sum(tf.mul(self.predictions, self.actions_placeholder), 1)
reward = tf.reduce_sum(self.q_vals)
tf.scalar_summary('Q_vals', reward)
with tf.name_scope('Loss'):
loss_non_regularized = tf.reduce_sum(tf.square(self.labels_placeholder - self.q_vals))
self.loss = loss_non_regularized + reg
tf.scalar_summary('Loss (without regularization)', loss_non_regularized)
tf.scalar_summary('Regularization', reg)
with tf.name_scope('Optimizer'):
#optimizer = tf.train.AdamOptimizer(learning_rate=self.lr)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.lr)
# optimizer = tf.train.RMSPropOptimizer(learning_rate = self.lr)
self.train_op = optimizer.minimize(self.loss)
def loc_net_fc(images, batch_size):
images -= 128
images /= 128.
images = tf.image.resize_images(images, 150, 150)
images_flat = tf.reshape(images, [batch_size, -1])
hidden_size = 100
with tf.name_scope('fc1') as scope:
weights = tf.Variable(tf.truncated_normal([150**2*3, hidden_size],
dtype=tf.float32, stddev=1e-3), name='weights')
biases = tf.Variable(tf.constant(0.0, shape=[hidden_size],
dtype=tf.float32), name='biases')
hidden = tf.add(tf.matmul(images_flat, weights), biases, name=scope)
hidden = tf.nn.relu(hidden)
with tf.name_scope('fc2') as scope:
weights = tf.Variable(tf.truncated_normal([hidden_size, 3],
dtype=tf.float32, stddev=1e-3), name='weights')
biases = tf.Variable(tf.constant(0.0, shape=[3], dtype=tf.float32),
name='biases')
theta = tf.add(tf.matmul(hidden, weights), biases, name=scope)
theta = tf.nn.tanh(theta)
return theta
def init_training_graph(self):
with tf.name_scope('Evaluation'):
# self.logits = self.conv_layer_f(self.last, self.logits_weight, strides=[1,1,1,1], scope_name="logits/")
with tf.name_scope("logits/"):
self.logits2 = tf.nn.conv2d(self.last, self.logits_weight, strides=[1,1,1,1], padding="VALID")
self.logits = tf.nn.bias_add(self.logits2, self.logits_biases)
self.predictions = self.logits
#self.predictions = tf.squeeze(self.logits, [3])
#softmax = tf.nn.softmax(self.logits)
#print softmax.get_shape()
#self.predictions = tf.slice(softmax, [0, 0, 0, 0], [-1, -1, -1, 1])
with tf.name_scope('Loss'):
self.loss = tf.reduce_mean(tf.losses.mean_squared_error(self.logits, self.train_labels_node))
#self.loss = tf.reduce_mean(tf.losses.mean_squared_error(self.predictions, self.train_labels_node))
tf.summary.scalar("mean_squared_error", self.loss)
self.predictions = tf.squeeze(self.predictions, [3])
self.train_prediction = self.predictions
self.test_prediction = self.predictions
tf.global_variables_initializer().run()
print('Computational graph initialised')
def bboxes_clip(bbox_ref, bboxes, scope=None):
"""Clip bounding boxes to a reference box.
Batch-compatible if the first dimension of `bbox_ref` and `bboxes`
can be broadcasted.
Args:
bbox_ref: Reference bounding box. Nx4 or 4 shaped-Tensor;
bboxes: Bounding boxes to clip. Nx4 or 4 shaped-Tensor or dictionary.
Return:
Clipped bboxes.
"""
# Bboxes is dictionary.
if isinstance(bboxes, dict):
with tf.name_scope(scope, 'bboxes_clip_dict'):
d_bboxes = {}
for c in bboxes.keys():
d_bboxes[c] = bboxes_clip(bbox_ref, bboxes[c])
return d_bboxes
# Tensors inputs.
with tf.name_scope(scope, 'bboxes_clip'):
# Easier with transposed bboxes. Especially for broadcasting.
bbox_ref = tf.transpose(bbox_ref)
bboxes = tf.transpose(bboxes)
# Intersection bboxes and reference bbox.
ymin = tf.maximum(bboxes[0], bbox_ref[0])
xmin = tf.maximum(bboxes[1], bbox_ref[1])
ymax = tf.minimum(bboxes[2], bbox_ref[2])
xmax = tf.minimum(bboxes[3], bbox_ref[3])
# Double check! Empty boxes when no-intersection.
ymin = tf.minimum(ymin, ymax)
xmin = tf.minimum(xmin, xmax)
bboxes = tf.transpose(tf.stack([ymin, xmin, ymax, xmax], axis=0))
return bboxes
def make_HBF2_model(x,W1,S1,C1,W2,S2,C2,phase_train):
with tf.name_scope("layer1") as scope:
layer1 = ml.get_Gaussian_layer(x,W1,S1,C1,phase_train)
with tf.name_scope("layer2") as scope:
layer2 = ml.get_Gaussian_layer(layer1,W2,S2,C2,phase_train)
y = layer2
return y
def build(self, FLAGS):
"""None
Build the model graph
:return:
"""
with tf.name_scope('G_'):
self.predict_g = self.__G__()
with tf.name_scope('D_'):
self.predict_d, self.predict_d_logits = self.__D__(self.input_d, input_type="Real")
tf.get_variable_scope().reuse_variables()
self.predict_d_for_g, self.predict_d_logits_for_g = self.__D__(self.predict_g, input_type="Gen")
if len(self.regularization_values_d) > 0:
self.regularization_sum_d = sum(self.regularization_values_d)
with tf.name_scope('loss'):
# self.loss_g = self.__loss_g__(predict=self.predict_g, self.labels, reg=self.regularization_sum)
self.__loss__(FLAGS)
with tf.name_scope('training'):
self.train_op_d, self.train_op_g = self.__training__(learning_rate=FLAGS.learning_rate)
with tf.name_scope('evaluation'):
# Calculate accuracy L2 norm
self.evaluation = self.__evaluation__(predict=self.predict_g, labels=self.labels)
def fc_layers(self):
# fc1
with tf.name_scope('fc1') as scope:
shape = int(np.prod(self.pool5.get_shape()[1:]))
fc1w = tf.Variable(tf.truncated_normal([shape, 4096],
dtype=tf.float32,
stddev=1e-1), name='weights')
fc1b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32),
trainable=True, name='biases')
pool5_flat = tf.reshape(self.pool5, [-1, shape])
fc1l = tf.nn.bias_add(tf.matmul(pool5_flat, fc1w), fc1b)
self.fc1 = tf.nn.relu(fc1l)
self.parameters += [fc1w, fc1b]
# fc2
with tf.name_scope('fc2') as scope:
fc2w = tf.Variable(tf.truncated_normal([4096, 4096],
dtype=tf.float32,
stddev=1e-1), name='weights')
fc2b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32),
trainable=True, name='biases')
fc2l = tf.nn.bias_add(tf.matmul(self.fc1, fc2w), fc2b)
self.fc2 = tf.nn.relu(fc2l)
self.parameters += [fc2w, fc2b]
# fc3
with tf.name_scope('fc3') as scope:
fc3w = tf.Variable(tf.truncated_normal([4096, 1000],
dtype=tf.float32,
stddev=1e-1), name='weights')
fc3b = tf.Variable(tf.constant(1.0, shape=[1000], dtype=tf.float32),
trainable=True, name='biases')
self.fc3l = tf.nn.bias_add(tf.matmul(self.fc2, fc3w), fc3b)
self.parameters += [fc3w, fc3b]
def inference(images, hidden1_units):
"""Build the MNIST model up to where it may be used for inference.
Args:
images: Images placeholder, from inputs().
hidden1_units: Size of the first hidden layer.
hidden2_units: Size of the second hidden layer.
Returns:
softmax_linear: Output tensor with the computed logits.
"""
# Hidden 1
with tf.name_scope('hidden1'):
weights = tf.Variable(
tf.truncated_normal([IMAGE_PIXELS, hidden1_units],
stddev=1.0 / math.sqrt(float(IMAGE_PIXELS))),
name='weights')
biases = tf.Variable(tf.zeros([hidden1_units]),
name='biases')
hidden1 = tf.nn.relu(tf.matmul(images, weights) + biases)
# Hidden 2
# Linear
with tf.name_scope('softmax_linear'):
weights = tf.Variable(
tf.truncated_normal([hidden1_units, NUM_CLASSES],
stddev=1.0 / math.sqrt(float(hidden1_units))),
name='weights')
biases = tf.Variable(tf.zeros([NUM_CLASSES]),
name='biases')
logits = tf.matmul(hidden1, weights) + biases
return logits
def nn_conv_layer(input_tensor, patch_size, num_channels,output_depth, layer_name, biases=False,act=None, pool=None):
"""Reusable code for making a simple neural net layer.
"""
# Adding a name scope ensures logical grouping of the layers in the graph.
with tf.name_scope(layer_name):
# This Variable will hold the state of the weights for the layer
with tf.name_scope('weights'):
weights = weight_variable([patch_size,patch_size,num_channels,output_depth])
# print ("weights:%s"%(weights.get_shape()))
variable_summaries(weights, layer_name + '/weights')
if (biases==True):
with tf.name_scope('biases'):
biases = bias_variable([output_depth])
# print("biases:%s" % (biases.get_shape()))
variable_summaries(biases, layer_name + '/biases')
with tf.name_scope('conv2d'):
# print("input:%s" % (input_tensor.get_shape()))
preactivate = tf.nn.conv2d(input_tensor, weights, [1, 1, 1, 1], padding='SAME')
tf.histogram_summary(layer_name + '/pre_activations', preactivate)
print("preactivate:%s" % (preactivate.get_shape()))
if (pool!=None):
max_pool=pool(preactivate,ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],
padding='SAME',name='max_pool')
if (act!=None):
activations = act(max_pool+biases, 'activation')
# tf.histogram_summary(layer_name + '/activations', activations)
return preactivate
def variable_summaries(var, name, collection_key):
"""Attach a lot of summaries to a Tensor (for TensorBoard visualization).
Args:
- var: Tensor for variable from which we want to log.
- name: Variable name.
- collection_key: Collection to save the summary to, can be any key of
`VAR_LOG_LEVELS`.
"""
if collection_key not in VAR_LOG_LEVELS.keys():
raise ValueError('"{}" not in `VAR_LOG_LEVELS`'.format(collection_key))
collections = VAR_LOG_LEVELS[collection_key]
with tf.name_scope(name):
mean = tf.reduce_mean(var)
tf.summary.scalar('mean', mean, collections)
num_params = tf.reduce_prod(tf.shape(var))
tf.summary.scalar('num_params', num_params, collections)
with tf.name_scope('stddev'):
stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
tf.summary.scalar('stddev', stddev, collections)
tf.summary.scalar('max', tf.reduce_max(var), collections)
tf.summary.scalar('min', tf.reduce_min(var), collections)
tf.summary.histogram('histogram', var, collections)
tf.summary.scalar('sparsity', tf.nn.zero_fraction(var), collections)
def ce(model, config, scope, connect, threshold = 1e-5):
with tf.variable_scope(scope), tf.name_scope(scope):
with tf.variable_scope('inputs'), tf.name_scope('inputs'):
model['%s_in0length' %scope] = model['%s_out0length' %connect]
model['%s_in1length' %scope] = model['%s_out1length' %connect]
model['%s_in2length' %scope] = model['%s_out2length' %connect]
model['%s_maxin2length' %scope] = model['%s_maxout2length' %connect]
model['%s_inputs' %scope] = tf.clip_by_value(tf.nn.softmax(model['%s_outputs' %connect]), threshold, 1. - threshold, name = '%s_inputs' %scope)
model['%s_out0length' %scope] = model['%s_in0length' %scope]
model['%s_out1length' %scope] = model['%s_in1length' %scope]
model['%s_out2length' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_out2length' %scope)
model['%s_maxout2length' %scope] = model['%s_maxin2length' %scope]
with tf.variable_scope('labels'), tf.name_scope('labels'):
model['%s_labels_len' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_labels_len' %scope)
model['%s_labels_ind' %scope] = tf.placeholder(tf.int64, [None, 2], '%s_labels_ind' %scope)
model['%s_labels_val' %scope] = tf.placeholder(tf.int32, [None], '%s_labels_val' %scope)
model['%s_labels_collapsed' %scope] = tf.sparse_to_dense(model['%s_labels_ind' %scope], [model['%s_maxin2length' %scope], model['%s_in0length' %scope]], model['%s_labels_val' %scope], -1, name = '%s_labels_collapsed' %scope)
model['%s_labels' %scope] = tf.one_hot(model['%s_labels_collapsed' %scope], model['%s_out1length' %scope], name = '%s_labels' %scope)
with tf.variable_scope('loss'), tf.name_scope('loss'):
model['%s_loss' %scope] = tf.reduce_sum(-tf.multiply(model['%s_labels' %scope], tf.log(model['%s_inputs' %scope])), name = '%s_loss' %scope)
with tf.variable_scope('outputs'), tf.name_scope('outputs'):
model['%s_output' %scope] = model['%s_inputs' %scope]
return model
def add_evaluation_step(result_tensor, ground_truth_tensor):
"""Inserts the operations we need to evaluate the accuracy of our results.
Args:
result_tensor: The new final node that produces results.
ground_truth_tensor: The node we feed ground truth data
into.
Returns:
Nothing.
"""
with tf.name_scope('accuracy'):
with tf.name_scope('correct_prediction'):
# tf.argmax(result_tensor, 1) = return index of maximal value (= 1 in a 1-of-N encoding vector) in each row (axis = 1)
# But we have more ones (indicating multiple labels) in one row of result_tensor due to the multi-label classification
# correct_prediction = tf.equal(tf.argmax(result_tensor, 1), \
# tf.argmax(ground_truth_tensor, 1))
# ground_truth is not a binary tensor, it contains the probabilities of each label = we need to tf.round() it
# to acquire a binary tensor allowing comparison by tf.equal()
# See: http://stackoverflow.com/questions/39219414/in-tensorflow-how-can-i-get-nonzero-values-and-their-indices-from-a-tensor-with
correct_prediction = tf.equal(tf.round(result_tensor), ground_truth_tensor)
with tf.name_scope('accuracy'):
# Mean accuracy over all labels:
# http://stackoverflow.com/questions/37746670/tensorflow-multi-label-accuracy-calculation
evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.scalar_summary('accuracy', evaluation_step)
return evaluation_step
def build(self):
"""None
Build the model graph
:return:
"""
with tf.name_scope('G_'):
self.predict_g = self.__G__()
self.predict_g2 = self.__G2__()
with tf.name_scope('D_'):
# Create reference examples
# Input d holds real&imaginary values. The discriminative decision based on reconstructed image
self.reconstructed_image_reference = self.get_reconstructed_image(real=self.input_d['real'],
imag=self.input_d['imag'], name='Both_gt')
predict_g2_stacked = tf.stack([self.predict_g2['real'][:,0,:,:], self.predict_g2['imag'][:,0,:,:]], axis=1)
self.predict, self.predict_logits = self.__D__([self.reconstructed_image_reference, predict_g2_stacked])
self.predict_d, self.predict_d_for_g = tf.split(value=self.predict, num_or_size_splits=2, axis=0)
self.predict_d_logits, self.predict_d_logits_for_g = tf.split(value=self.predict_logits,
num_or_size_splits=2, axis=0)
self.clip_weights = self.__clip_weights__()
with tf.name_scope('loss'):
# self.loss_g = self.__loss_g__(predict=self.predict_g, self.labels, reg=self.regularization_sum)
self.__loss__()
with tf.name_scope('training'):
self.train_op_d, self.train_op_g = self.__training__(learning_rate=self.FLAGS.learning_rate)
with tf.name_scope('evaluation'):
# Calculate accuracy L2 norm
self.evaluation = self.__evaluation__(predict=self.predict_g, labels=self.labels)
def encode(self, inputs, attention_bias):
"""Generate continuous representation for inputs.
Args:
inputs: int tensor with shape [batch_size, input_length].
attention_bias: float tensor with shape [batch_size, 1, 1, input_length]
Returns:
float tensor with shape [batch_size, input_length, hidden_size]
"""
with tf.name_scope("encode"):
# Prepare inputs to the layer stack by adding positional encodings and
# applying dropout.
embedded_inputs = self.embedding_softmax_layer(inputs)
inputs_padding = model_utils.get_padding(inputs)
with tf.name_scope("add_pos_encoding"):
length = tf.shape(embedded_inputs)[1]
pos_encoding = model_utils.get_position_encoding(
length, self.params.hidden_size)
encoder_inputs = embedded_inputs + pos_encoding
if self.train:
encoder_inputs = tf.nn.dropout(
encoder_inputs, 1 - self.params.layer_postprocess_dropout)
return self.encoder_stack(encoder_inputs, attention_bias, inputs_padding)
def inference(input_tensor,train,regularizer):
#第一层卷积
with tf.variable_scope('layer1-conv1'):
conv1_weights = tf.get_variable("weight",
[CONV1_SIZE,CONV1_SIZE,NUM_CHANNELS,CONV1_DEEP],
initializer=tf.truncated_normal_initializer(stddev=0.1))
conv1_biases = tf.get_variable("biases",[CONV1_DEEP],
initializer=tf.constant_initializer(0.0))
conv1 = tf.nn.conv2d(input_tensor,conv1_weights,
strides=[1,1,1,1],padding='SAME')
relu1 = tf.nn.relu(tf.nn.bias_add(conv1,conv1_biases))
#第二层池化
with tf.name_scope('layer2-pool1'):
pool1 = tf.nn.max_pool(relu1,ksize=[1,2,2,1],
strides=[1,2,2,1],padding='SAME')
#第三层卷积
with tf.variable_scope('layer3-conv2'):
conv2_weights = tf.get_variable("weight",
[CONV2_SIZE,CONV2_SIZE,CONV1_DEEP,CONV2_DEEP],
initializer=tf.truncated_normal_initializer(stddev=0.1))
conv2_biases = tf.get_variable("biases",[CONV2_DEEP],
initializer=tf.constant_initializer(0.0))
conv2 = tf.nn.conv2d(pool1,conv2_weights,
strides=[1,1,1,1],padding='SAME')
relu2 = tf.nn.relu(tf.nn.bias_add(conv2,conv2_biases))
#第四层池化
with tf.name_scope('layer4-pool2'):
pool2 = tf.nn.max_pool(relu2,ksize=[1,2,2,1],
strides=[1,2,2,1],padding='SAME')
pool_shape = pool2.get_shape().as_list()
nodes = pool_shape[1] * pool_shape[2] * pool_shape[3]
reshaped = tf.reshape(pool2,[pool_shape[0],nodes])
#第五层全连接层
with tf.variable_scope('layer5-fc1'):
fc1_weights = tf.get_variable("weight",[nodes,FC_SIZE],
initializer=tf.truncated_normal_initializer(stddev=0.1))
#只有全连接层的权重需要加入正则化
if regularizer != None:
tf.add_to_collection('losses',regularizer(fc1_weights))
fc1_biases = tf.get_variable("bias",[FC_SIZE],
initializer=tf.constant_initializer(0.1))
fc1 = tf.nn.relu(tf.matmul(reshaped,fc1_weights) + fc1_biases)
if train: fc1 = tf.nn.dropout(fc1,0.5)
#第六层全连接层
with tf.variable_scope('layer6-fc2'):
fc2_weights = tf.get_variable("weight",[FC_SIZE,NUM_LABELS],
initializer=tf.truncated_normal_initializer(stddev=0.1))
#只有全连接层的权重需要加入正则化
if regularizer != None:
tf.add_to_collection('losses',regularizer(fc2_weights))
fc2_biases = tf.get_variable("bias",[NUM_LABELS],
initializer=tf.constant_initializer(0.1))
logit = tf.matmul(fc1,fc2_weights) + fc2_biases
return logit
def __init__(self, num_features, num_output, l2_reg_lambda=0.0, neg_output=False):
self.input_x = tf.placeholder(tf.float32, [None, num_features], name="input_x")
self.input_y = tf.placeholder(tf.float32, [None, num_output], name="input_y")
# Keeping track of l2 regularization loss (optional)
l2_loss = tf.constant(0.0)
with tf.name_scope("softmax"):
filter_shape = [num_features, num_output]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1))
b = tf.Variable(tf.constant(0.1, shape=[num_output]))
self.raw_scores = tf.nn.xw_plus_b(self.input_x, W, b, name="scores")
if neg_output:
self.scores = tf.nn.elu(self.raw_scores, name="tanh")
else:
self.scores = tf.nn.relu(self.raw_scores, name="relu")
l2_loss += tf.nn.l2_loss(W)
l2_loss += tf.nn.l2_loss(b)
with tf.name_scope("loss"):
self.losses = tf.square(tf.sub(self.scores, self.input_y))
self.avgloss = tf.reduce_mean(tf.abs(tf.sub(self.scores, self.input_y)))
self.loss = tf.reduce_mean(self.losses) + l2_reg_lambda * l2_loss
def feature_extraction(x, Nx, Ny, Ch, conv_w_1, conv_b_1, conv_w_2, conv_b_2, ff_w_1, ff_b_1, m1s = 4, m2s = 2, No = 512):
""" Creates a convolutional neural network to analyze some world tensor and return features from it. """
# Check that all the sizes are consistent
assert(Nx/m1s == int(Nx/m1s) and (Nx/m1s/m2s == int(Nx/m1s/m2s)))
assert(Ny/m1s == int(Ny/m1s) and (Ny/m1s/m2s == int(Ny/m1s/m2s)))
# First Convolutional Layer
with tf.name_scope("Convolution1"):
conv1_act = tf.nn.conv2d(x, conv_w_1, strides=[1, 1, 1, 1], padding='SAME') + conv_b_1
conv1 = tf.nn.relu(conv1_act, 'relu')
# First Max Pooling Layer
with tf.name_scope("Max1"):
max1 = tf.nn.max_pool(conv1, ksize=[1, m1s, m1s, 1], strides=[1, m1s, m1s, 1], padding='SAME')
# Second Convolutional Layer
with tf.name_scope("Convolution2"):
conv2_act = tf.nn.conv2d(max1, conv_w_2, strides=[1, 1, 1, 1], padding='SAME') + conv_b_2
conv2 = tf.nn.relu(conv2_act, 'relu')
# Second Max Pooling Layer
with tf.name_scope("Max2"):
max2 = tf.nn.max_pool(conv2, ksize=[1, m2s, m2s, 1], strides=[1, m2s, m2s, 1], padding='SAME')
# Reshaping max2 for FF1
max2_rshp = tf.reshape(max2, [-1, 288]) # Layer shape [None, 5, 5, 64] 1600 Total
# First Feed Forward Layer
with tf.name_scope('FF1'):
ff1_act = tf.matmul(max2_rshp, ff_w_1) + ff_b_1
ff1 = tf.nn.relu(ff1_act, 'relu')
return ff1
def get_probabilities(units_value, weights, bias, is_hidden_layer = False):
if is_hidden_layer:
with tf.name_scope("Hidden_Probabilities"):
return tf.nn.sigmoid(tf.matmul(units_value,weights) + bias)
else:
with tf.name_scope("Visible_Probabilities"):
return tf.nn.sigmoid(tf.matmul(units_value,tf.transpose(weights)) + bias)
def nn_graph(activation):
# create graph
x = tf.placeholder(tf.float32, (1, 2, 2, 3), 'x_placeholder')
y = tf.placeholder(tf.int32, name='y_placeholder', shape=[1, 2])
with tf.name_scope('conv1'):
conv_1 = tf.layers.conv2d(
inputs=x,
filters=10,
kernel_size=[2, 2],
padding="same",
activation=activation)
with tf.name_scope('fc2'):
flatten = tf.layers.flatten(conv_1)
top = tf.layers.dense(flatten, _classes)
with tf.name_scope('logits'):
logits = tf.nn.softmax(top)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits)
cross_entropy = tf.reduce_mean(cross_entropy)
return x, y, cross_entropy
def ced(model, config, scope, connect, threshold = 1e-5):
with tf.variable_scope(scope), tf.name_scope(scope):
with tf.variable_scope('inputs'), tf.name_scope('inputs'):
model['%s_in0length' %scope] = model['%s_out0length' %connect]
model['%s_in1length' %scope] = model['%s_out1length' %connect]
model['%s_in2length' %scope] = model['%s_out2length' %connect]
model['%s_maxin2length' %scope] = model['%s_maxout2length' %connect]
model['%s_inputs' %scope] = tf.clip_by_value(model['%s_outputs' %connect], threshold, 1. - threshold, name = '%s_inputs' %scope)
model['%s_out0length' %scope] = model['%s_in0length' %scope]
model['%s_out1length' %scope] = model['%s_in1length' %scope]
model['%s_out2length' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_out2length' %scope)
model['%s_maxout2length' %scope] = model['%s_maxin2length' %scope]
with tf.variable_scope('labels'), tf.name_scope('labels'):
model['%s_labels_len' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_labels_len' %scope)
model['%s_labels_ind' %scope] = tf.placeholder(tf.int64, [None, 3], '%s_labels_ind' %scope)
model['%s_labels_val' %scope] = tf.placeholder(tf.float32, [None], '%s_labels_val' %scope)
model['%s_labels' %scope] = tf.sparse_to_dense(model['%s_labels_ind' %scope], [model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], model['%s_labels_val' %scope], -1, name = '%s_labels' %scope)
with tf.variable_scope('loss'), tf.name_scope('loss'):
model['%s_loss' %scope] = tf.reduce_sum(tf.where(tf.less(model['%s_labels' %scope], tf.zeros([model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], tf.float32)), tf.zeros([model['%s_in0length' %scope], model['%s_maxin2length' %scope], model['%s_maxin2length' %scope]], tf.float32), -tf.add(tf.multiply(model['%s_labels' %scope], tf.log(model['%s_inputs' %scope])), tf.multiply(tf.subtract(1., model['%s_labels' %scope]), tf.log(tf.subtract(1., model['%s_inputs' %scope]))))), name = '%s_loss' %scope)
with tf.variable_scope('outputs'), tf.name_scope('outputs'):
model['%s_output' %scope] = model['%s_inputs' %scope]
return model
def conv_net(x, weights, biases, dropout):
with tf.name_scope('input_czm'):
# Reshape input picture
x = tf.reshape(x, shape=[-1, 28, 28, 1])
with tf.name_scope('first_layer'):
# Convolution Layer
conv1 = conv2d(x, weights['wc1'], biases['bc1'])
# Max Pooling (down-sampling)
conv1 = maxpool2d(conv1, k=2)
with tf.name_scope('sec_layer'):
# Convolution Layer
conv2 = conv2d(conv1, weights['wc2'], biases['bc2'])
# Max Pooling (down-sampling)
conv2 = maxpool2d(conv2, k=2)
with tf.name_scope('full_conn'):
# Fully connected layer
# Reshape conv2 output to fit fully connected layer input
fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]])
fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])
fc1 = tf.nn.relu(fc1)
with tf.name_scope('dropout_ops'):
# Apply Dropout
fc1 = tf.nn.dropout(fc1, dropout)
# Output, class prediction
out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])
return out
def testVarOpScopeReuseParam(self):
with self.test_session():
with tf.variable_scope("outer") as outer:
with tf.variable_op_scope([], "tower", "default"):
self.assertEqual(tf.get_variable("w", []).name,
"outer/tower/w:0")
with tf.name_scope("scope2") as sc2:
self.assertEqual(sc2, "outer/tower/scope2/")
with tf.variable_op_scope([], None, "default"):
self.assertEqual(tf.get_variable("w", []).name,
"outer/default/w:0")
with tf.name_scope("scope2") as sc2:
self.assertEqual(sc2, "outer/default/scope2/")
with tf.variable_scope(outer) as outer:
with tf.variable_op_scope([], "tower", "default", reuse=True):
self.assertEqual(tf.get_variable("w", []).name,
"outer/tower/w:0")
with tf.name_scope("scope2") as sc2:
self.assertEqual(sc2, "outer_1/tower/scope2/")
outer.reuse_variables()
with tf.variable_op_scope([], None, "default"):
self.assertEqual(tf.get_variable("w", []).name,
"outer/default/w:0")
with tf.name_scope("scope2") as sc2:
self.assertEqual(sc2, "outer_1/default/scope2/")
def loss(self, logits, labels, regularization):
"""Adds to the inference model the layers required to generate loss."""
with tf.name_scope('loss'):
with tf.name_scope('var_loss'):
labels = tf.cast(labels, tf.float32)
shape = labels.get_shape()
same_class = tf.boolean_mask(logits, tf.equal(labels, tf.ones(shape)))
diff_class = tf.boolean_mask(logits, tf.not_equal(labels, tf.ones(shape)))
same_mean, same_var = tf.nn.moments(same_class, [0])
diff_mean, diff_var = tf.nn.moments(diff_class, [0])
var_loss = same_var + diff_var
with tf.name_scope('mean_loss'):
mean_loss = self.lamda * tf.where(tf.greater(self.mu - (same_mean - diff_mean), 0),
self.mu - (same_mean - diff_mean), 0)
with tf.name_scope('regularization'):
regularization *= tf.add_n(self.regularizers)
loss = var_loss + mean_loss + regularization
# Summaries for TensorBoard.
tf.summary.scalar('loss/total', loss)
with tf.name_scope('averages'):
averages = tf.train.ExponentialMovingAverage(0.9)
op_averages = averages.apply([var_loss, mean_loss, regularization, loss])
tf.summary.scalar('loss/avg/var_loss', averages.average(var_loss))
tf.summary.scalar('loss/avg/mean_loss', averages.average(mean_loss))
tf.summary.scalar('loss/avg/regularization', averages.average(regularization))
tf.summary.scalar('loss/avg/total', averages.average(loss))
with tf.control_dependencies([op_averages]):
loss_average = tf.identity(averages.average(loss), name='control')
return loss, loss_average
def generate_model(self, model, name=''):
if not model: return self
with tf.name_scope('state'):
self.keep_prob = tf.placeholder(tf.float32) # 1 for testing! else 1 - dropout
self.train_phase = tf.placeholder(tf.bool, name='train_phase')
with tf.device(_cpu): self.global_step = tf.Variable(
0) # dont set, feed or increment global_step, tensorflow will do it automatically
with tf.name_scope('data'):
if len(self.input_shape) == 1:
self.input_width = self.input_shape[0]
elif self.input_shape:
self.x = x = self.input = tf.placeholder(tf.float32, [None, self.input_shape[0], self.input_shape[1]])
# todo [None, self.input_shape]
self.last_layer = x
self.last_shape = x
elif self.input_width:
self.x = x = self.target = tf.placeholder(tf.float32, [None, self.input_width])
self.last_layer = x
else:
raise Exception("need input_shape or input_width by now")
self.y = y = self.target = tf.placeholder(tf.float32, [None, self.output_width])
with tf.name_scope('model'):
model(self)
if (self.last_width != self.output_width):
self.classifier() # 10 classes auto
def loss(self, logits, labels, regularization):
"""Adds to the inference model the layers required to generate loss."""
with tf.name_scope('loss'):
with tf.name_scope('hinge_loss'):
labels = tf.cast(labels, tf.float32)
zeros = tf.zeros(labels.get_shape())
output = tf.ones(labels.get_shape()) - tf.multiply(labels, logits)
hinge_loss = tf.where(tf.greater(output, zeros), output, zeros)
hinge_loss = tf.reduce_mean(hinge_loss)
with tf.name_scope('regularization'):
regularization *= tf.add_n(self.regularizers)
loss = hinge_loss + regularization
# Summaries for TensorBoard.
tf.summary.scalar('loss/hinge_loss', hinge_loss)
tf.summary.scalar('loss/regularization', regularization)
tf.summary.scalar('loss/total', loss)
with tf.name_scope('averages'):
averages = tf.train.ExponentialMovingAverage(0.9)
op_averages = averages.apply([hinge_loss, regularization, loss])
tf.summary.scalar('loss/avg/hinge_loss', averages.average(hinge_loss))
tf.summary.scalar('loss/avg/regularization', averages.average(regularization))
tf.summary.scalar('loss/avg/total', averages.average(loss))
with tf.control_dependencies([op_averages]):
loss_average = tf.identity(averages.average(loss), name='control')
return loss, loss_average
def testSharingWeightsWithDifferentNamescope(self):
num_units = 3
input_size = 5
batch_size = 2
num_proj = 4
with self.test_session(graph=tf.Graph()) as sess:
initializer = tf.random_uniform_initializer(-1, 1, seed=self._seed)
inputs = 10 * [
tf.placeholder(tf.float32, shape=(None, input_size))]
cell = rnn_cell.LSTMCell(
num_units, input_size, use_peepholes=True,
num_proj=num_proj, initializer=initializer)
with tf.name_scope("scope0"):
with tf.variable_scope("share_scope"):
outputs0, _ = rnn.rnn(cell, inputs, dtype=tf.float32)
with tf.name_scope("scope1"):
with tf.variable_scope("share_scope", reuse=True):
outputs1, _ = rnn.rnn(cell, inputs, dtype=tf.float32)
tf.initialize_all_variables().run()
input_value = np.random.randn(batch_size, input_size)
output_values = sess.run(
outputs0 + outputs1, feed_dict={inputs[0]: input_value})
outputs0_values = output_values[:10]
outputs1_values = output_values[10:]
self.assertEqual(len(outputs0_values), len(outputs1_values))
for out0, out1 in zip(outputs0_values, outputs1_values):
self.assertAllEqual(out0, out1)
def build(self):
"""None
Build the model graph
:return:
"""
with tf.name_scope('G_'):
self.predict_g = self.__G__()
with tf.name_scope('D_'):
self.predict, self.predict_logits = self.__D__([self.input_d, self.predict_g], input_type="Real")
self.predict_d, self.predict_d_for_g = tf.split(value=self.predict, num_or_size_splits=2, axis=0)
self.predict_d_logits, self.predict_d_logits_for_g = tf.split(value=self.predict_logits,
num_or_size_splits=2, axis=0)
# self.predict_d, self.predict_d_logits
# with tf.variable_scope(tf.get_variable_scope(), reuse=True):
# self.predict_d_for_g, self.predict_d_logits_for_g = self.__D__(self.predict_g, input_type="Gen")
if len(self.regularization_values_d) > 0:
self.regularization_sum_d = sum(self.regularization_values_d)
with tf.name_scope('loss'):
# self.loss_g = self.__loss_g__(predict=self.predict_g, self.labels, reg=self.regularization_sum)
self.__loss__()
with tf.name_scope('training'):
self.train_op_d, self.train_op_g = self.__training__(learning_rate=self.FLAGS.learning_rate)
with tf.name_scope('evaluation'):
# Calculate accuracy L2 norm
self.evaluation = self.__evaluation__(predict=self.predict_g, labels=self.labels)
def inference(images,hidden1_units,hidden2_units):
"""建立前馈神经网络模型
Args:
images:输入图像数据
hidden1_units:第一个隐藏层的神经元数目
hidden2_units:第二个隐藏层 的神经元数目
returns:
softmax_linear:输出张量为计算后的结果
"""
#隐藏层1
with tf.name_scope('hidden1'):
weights = tf.Variable(tf.truncated_normal([IMAGE_PIXELS,hidden1_units],stddev=1.0/math.sqrt(float(IMAGE_PIXELS))),name='weights')#?
biases = tf.Variable(tf.zeros([hidden1_units]),name='biases')
hidden1 = tf.nn.relu(tf.matmul(images,weights)+biases)
#隐藏层2
with tf.name_scope('hidden2'):
weights = tf.Variable(tf.truncated_normal([hidden1_units,hidden2_units],stddev=1.0/math.sqrt(float(hidden1_units))),name='weights')
biases = tf.Variable(tf.zeros([hidden2_units]),name='biases')
hidden2 = tf.nn.relu(tf.matmul(hidden1,weights)+biases)
#线性输出层
with tf.name_scope('softmax_linear'):
weights = tf.Variable(tf.truncated_normal([hidden2_units,NUM_CLASSES]),name='biases')
biases = tf.Variable(tf.zeros([NUM_CLASSES]),name='biases')
logits = tf.matmul(hidden2,weights) + biases
return logits
def bboxes_resize(bbox_ref, bboxes, name=None):
"""Resize bounding boxes based on a reference bounding box,
assuming that the latter is [0, 0, 1, 1] after transform. Useful for
updating a collection of boxes after cropping an image.
"""
# Bboxes is dictionary.
if isinstance(bboxes, dict):
with tf.name_scope(name, 'bboxes_resize_dict'):
d_bboxes = {}
for c in bboxes.keys():
d_bboxes[c] = bboxes_resize(bbox_ref, bboxes[c])
return d_bboxes
# Tensors inputs.
with tf.name_scope(name, 'bboxes_resize'):
# Translate.
v = tf.stack([bbox_ref[0], bbox_ref[1], bbox_ref[0], bbox_ref[1]])
bboxes = bboxes - v
# Scale.
s = tf.stack([bbox_ref[2] - bbox_ref[0],
bbox_ref[3] - bbox_ref[1],
bbox_ref[2] - bbox_ref[0],
bbox_ref[3] - bbox_ref[1]])
bboxes = bboxes / s
return bboxes
def testVarOpScopeOuterScope(self):
with self.test_session():
with tf.variable_scope("outer") as outer:
pass
with tf.variable_op_scope([], outer, "default"):
self.assertEqual(tf.get_variable("w", []).name,
"outer/w:0")
with tf.name_scope("scope2") as sc2:
self.assertEqual(sc2, "outer_1/scope2/")
with tf.variable_op_scope([], None, "default"):
self.assertEqual(tf.get_variable("w", []).name,
"outer/default/w:0")
with tf.name_scope("scope2") as sc2:
self.assertEqual(sc2, "outer_1/default/scope2/")
with tf.variable_op_scope([], outer, "default", reuse=True):
self.assertEqual(tf.get_variable("w", []).name,
"outer/w:0")
with tf.name_scope("scope2") as sc2:
self.assertEqual(sc2, "outer_2/scope2/")
outer.reuse_variables()
with tf.variable_op_scope([], None, "default"):
self.assertEqual(tf.get_variable("w", []).name,
"outer/default/w:0")
with tf.name_scope("scope2") as sc2:
self.assertEqual(sc2, "outer_2/default/scope2/")
def extract_groups(predicted_boxes,
predicted_scores,
predicted_group_flags=None,
predicted_offsets=None,
mode='train',
verbose=False,
epsilon=1e-8,
**kwargs):
""" Extract crops from the outputs of intermediate stage.
Args:
predicted_boxes: A (batch_size, num_cells, num_cells, num_boxes, 4) array
predicted_scores: A (batch_size, num_cells, num_cells, num_boxes, 1) array
predicted_group_flags: A (batch_size, num_cells, num_cells, num_boxes, 1) array
predicted_offsets: A (batch_size, num_cells, num_cells, num_boxes, 2) array
mode: If test, the boxes are only passed to the next stage if they are worth being refined
(ie groups or unprecise individual)
Kwargs:
{train, test}_patch_confidence_threshold: Minimum confidence threshold to qualify for refinement
patch_nms_threshold: NMS threshold
{train, test}_num_crops: Number of crops to extract
test_patch_strong_confidence_threshold: high confidence threshold
previous_batch_size: Batch size of the previous stage (for which `predicted boxes` where output). Needs
to be statistically known for the NMS loop.
#Returns:
Extracted crops and their confidence scores
"""
if mode == 'train': # train time
(confidence_threshold, nms_threshold, num_outputs) = get_defaults(
kwargs, ['train_patch_confidence_threshold', 'train_patch_nms_threshold', 'train_num_crops'], verbose=verbose)
elif mode in ['val', 'test']: # inference
(confidence_threshold, nms_threshold, num_outputs) = get_defaults(
kwargs, ['test_patch_confidence_threshold', 'test_patch_nms_threshold', 'test_num_crops'], verbose=verbose)
else:
raise ValueError('Unknown mode', mode)
if verbose:
print(' extracting %d crops' % num_outputs)
## Flatten
# predicted_score: (batch, num_boxes, 1)
# predicted_boxes: (batch, num_boxes, 4)
with tf.name_scope('flat_output'):
predicted_boxes = utils.flatten_percell_output(predicted_boxes)
predicted_scores = utils.flatten_percell_output(predicted_scores)
## Filter
kept_out_filter = tf.zeros(tf.shape(predicted_scores)) # default
with tf.name_scope('filter_groups'):
# At test time, we keep out individual confidences with high confidence
# we save these `shortcut` boxes in the `kept_out_filter` Tensor
if mode in ['test', 'val']:
strong_confidence_threshold = get_defaults(
kwargs, ['test_patch_strong_confidence_threshold'], verbose=verbose)[0]
if isinstance(strong_confidence_threshold, tf.Tensor) or strong_confidence_threshold < 1.0:
predicted_boxes, predicted_scores, kept_out_filter = filter_individuals(
predicted_boxes, predicted_scores, predicted_group_flags, strong_confidence_threshold)
# Additionally, we filter out boxes with confidence below the threshold
if isinstance(confidence_threshold, tf.Tensor) or confidence_threshold > 0.:
with tf.name_scope('filter_confidence'):
predicted_boxes, predicted_scores = filter_threshold(
predicted_boxes, predicted_scores, confidence_threshold)
## Rescale remaining boxes with the learned offsets
with tf.name_scope('offsets_rescale_boxes'):
if predicted_offsets is not None:
predicted_boxes = utils.rescale_with_offsets(
predicted_boxes, utils.flatten_percell_output(predicted_offsets), epsilon)
## Extract n best patches
# crop_boxes: (batch, num_crops, 4)
# crop_boxes_confidences: (batch, num_crops)
predicted_scores = tf.squeeze(predicted_scores, axis=-1)
if isinstance(num_outputs, tf.Tensor) or num_outputs > 0:
# Non-Maximum Suppression: outputs the top `num_outputs` boxes after NMS
if (isinstance(nms_threshold, tf.Tensor) or nms_threshold < 1.0) or (isinstance(num_outputs, tf.Tensor)):
batch_size = get_defaults(kwargs, ['previous_batch_size'], verbose=verbose)[0]
current_batch = tf.shape(predicted_boxes)[0]
with tf.name_scope('nms'):
nms_boxes = []
nms_boxes_confidences = []
for i in range(batch_size):
boxes, scores = tf.cond(
i < current_batch, # last batch can be smaller
true_fn=lambda: utils.nms_with_pad(predicted_boxes[i, :, :],
predicted_scores[i, :],
num_outputs,
iou_threshold=nms_threshold),
false_fn=lambda: (tf.zeros((num_outputs, 4)), tf.zeros((num_outputs,)))
)
nms_boxes.append(boxes)
nms_boxes_confidences.append(scores)
# Reshape nms boxes output
predicted_boxes = tf.stack(nms_boxes, axis=0)
predicted_boxes = tf.slice(predicted_boxes, (0, 0, 0), (current_batch, -1, -1))
predicted_boxes = tf.reshape(predicted_boxes, (-1, num_outputs, 4))
# Reshape nms scores output
predicted_scores = tf.stack(nms_boxes_confidences, axis=0)
predicted_scores = tf.slice(predicted_scores, (0, 0), (current_batch, -1))
predicted_scores = tf.reshape(predicted_scores, (-1, num_outputs))
# No NMS: Outputs `num_outputs` boxes with the best confidence scores
# num_outputs need to be defined for tf.nn.top_k
else:
predicted_scores, top_indices = tf.nn.top_k(predicted_scores, k=num_outputs)
batch_indices = tf.range(tf.shape(predicted_boxes)[0])
batch_indices = tf.tile(tf.expand_dims(batch_indices, axis=-1), (1, num_outputs))
gather_indices = tf.stack([batch_indices, top_indices], axis=-1)
predicted_boxes = tf.gather_nd(predicted_boxes, gather_indices)
# No filtering
return predicted_boxes, predicted_scores, kept_out_filter
def convlayers(self):
self.parameters = []
# zero-mean input
with tf.name_scope('preprocess') as scope:
mean = tf.constant([123.68, 116.779, 103.939],
dtype=tf.float32,
shape=[1, 1, 1, 3],
name='img_mean')
images = self.imgs - mean
# conv1_1
with tf.name_scope('conv1_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 3, 64],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[64],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv1_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv1_2
with tf.name_scope('conv1_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 64],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv1_1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0, shape=[64],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv1_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# pool1
self.pool1 = tf.nn.max_pool(self.conv1_2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool1')
# conv2_1
with tf.name_scope('conv2_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 128],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.pool1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[128],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv2_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv2_2
with tf.name_scope('conv2_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 128],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv2_1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[128],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv2_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# pool2
self.pool2 = tf.nn.max_pool(self.conv2_2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool2')
# conv3_1
with tf.name_scope('conv3_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 128, 256],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.pool2,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[256],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv3_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv3_2
with tf.name_scope('conv3_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv3_1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[256],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv3_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv3_3
with tf.name_scope('conv3_3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 256],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv3_2,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[256],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv3_3 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# pool3
self.pool3 = tf.nn.max_pool(self.conv3_3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool3')
# conv4_1
with tf.name_scope('conv4_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 256, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.pool3,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv4_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv4_2
with tf.name_scope('conv4_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv4_1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv4_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv4_3
with tf.name_scope('conv4_3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv4_2,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv4_3 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# pool4
self.pool4 = tf.nn.max_pool(self.conv4_3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool4')
# conv5_1
with tf.name_scope('conv5_1') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.pool4,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv5_1 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv5_2
with tf.name_scope('conv5_2') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv5_1,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv5_2 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# conv5_3
with tf.name_scope('conv5_3') as scope:
kernel = tf.Variable(tf.truncated_normal([3, 3, 512, 512],
dtype=tf.float32,
stddev=1e-1),
name='weights')
conv = tf.nn.conv2d(self.conv5_2,
kernel, [1, 1, 1, 1],
padding='SAME')
biases = tf.Variable(tf.constant(0.0,
shape=[512],
dtype=tf.float32),
trainable=True,
name='biases')
out = tf.nn.bias_add(conv, biases)
self.conv5_3 = tf.nn.relu(out, name=scope)
self.parameters += [kernel, biases]
# pool5
self.pool5 = tf.nn.max_pool(self.conv5_3,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool4')
def __init__(self,
block_fn,
num_blocks: List[int],
num_frames: int,
model_structure: List[Any],
input_specs: layers.InputSpec = layers.InputSpec(
shape=[None, None, None, None, 3]),
model_edge_weights: Optional[List[Any]] = None,
use_object_input: bool = False,
attention_mode: str = 'peer',
bn_decay: float = rf.BATCH_NORM_DECAY,
bn_epsilon: float = rf.BATCH_NORM_EPSILON,
use_sync_bn: bool = False,
**kwargs):
"""Generator for AssembleNet++ models.
Args:
block_fn: `function` for the block to use within the model. Currently only
has `bottleneck_block_interleave as its option`.
num_blocks: list of 4 `int`s denoting the number of blocks to include in
each of the 4 block groups. Each group consists of blocks that take
inputs of the same resolution.
num_frames: the number of frames in the input tensor.
model_structure: AssembleNetPlus model structure in the string format.
input_specs: `tf.keras.layers.InputSpec` specs of the input tensor.
Dimension should be `[batch*time, height, width, channels]`.
model_edge_weights: AssembleNet model structure connection weight in the
string format.
use_object_input : 'bool' values whether using object inputs
attention_mode : 'str' , default = 'self', If we use peer attention 'peer'
bn_decay: `float` batch norm decay parameter to use.
bn_epsilon: `float` batch norm epsilon parameter to use.
use_sync_bn: use synchronized batch norm for TPU.
**kwargs: pass through arguments.
Returns:
Model `function` that takes in `inputs` and `is_training` and returns the
output `Tensor` of the AssembleNetPlus model.
"""
data_format = tf.keras.backend.image_data_format()
# Creation of the model graph.
logging.info('model_structure=%r', model_structure)
logging.info('model_structure=%r', model_structure)
logging.info('model_edge_weights=%r', model_edge_weights)
structure = model_structure
if use_object_input:
original_inputs = tf.keras.Input(shape=input_specs[0].shape[1:])
object_inputs = tf.keras.Input(shape=input_specs[1].shape[1:])
input_specs = input_specs[0]
else:
original_inputs = tf.keras.Input(shape=input_specs.shape[1:])
object_inputs = None
original_num_frames = num_frames
assert num_frames > 0, f'Invalid num_frames {num_frames}'
grouping = {-3: [], -2: [], -1: [], 0: [], 1: [], 2: [], 3: []}
for i in range(len(structure)):
grouping[structure[i][0]].append(i)
stem_count = len(grouping[-3]) + len(grouping[-2]) + len(grouping[-1])
assert stem_count != 0
stem_filters = 128 // stem_count
if len(input_specs.shape) == 5:
first_dim = (input_specs.shape[0] * input_specs.shape[1] if
input_specs.shape[0] and input_specs.shape[1] else -1)
reshape_inputs = tf.reshape(original_inputs,
(first_dim, ) + input_specs.shape[2:])
elif len(input_specs.shape) == 4:
reshape_inputs = original_inputs
else:
raise ValueError(
f'Expect input spec to be 4 or 5 dimensions {input_specs.shape}'
)
if grouping[-2]:
# Instead of loading optical flows as inputs from data pipeline, we are
# applying the "Representation Flow" to RGB frames so that we can compute
# the flow within TPU/GPU on fly. It's essentially optical flow since we
# do it with RGBs.
axis = 3 if data_format == 'channels_last' else 1
flow_inputs = rf.RepresentationFlow(
original_num_frames,
depth=reshape_inputs.shape.as_list()[axis],
num_iter=40,
bottleneck=1)(reshape_inputs)
streams = []
for i in range(len(structure)):
with tf.name_scope('Node_' + str(i)):
if structure[i][0] == -1:
inputs = asn.rgb_conv_stem(
reshape_inputs,
original_num_frames,
stem_filters,
temporal_dilation=structure[i][1],
bn_decay=bn_decay,
bn_epsilon=bn_epsilon,
use_sync_bn=use_sync_bn)
streams.append(inputs)
elif structure[i][0] == -2:
inputs = asn.flow_conv_stem(
flow_inputs,
stem_filters,
temporal_dilation=structure[i][1],
bn_decay=bn_decay,
bn_epsilon=bn_epsilon,
use_sync_bn=use_sync_bn)
streams.append(inputs)
elif structure[i][0] == -3:
# In order to use the object inputs, you need to feed your object
# input tensor here.
inputs = object_conv_stem(object_inputs)
streams.append(inputs)
else:
block_number = structure[i][0]
combined_inputs = [
streams[structure[i][1][j]]
for j in range(0, len(structure[i][1]))
]
logging.info(grouping)
nodes_below = []
for k in range(-3, structure[i][0]):
nodes_below = nodes_below + grouping[k]
peers = []
if attention_mode:
lg_channel = -1
# To show structures for attention we show nodes_below
logging.info(nodes_below)
for k in nodes_below:
logging.info(streams[k].shape)
lg_channel = max(streams[k].shape[3], lg_channel)
for node_index in nodes_below:
attn = tf.reduce_mean(streams[node_index], [1, 2])
attn = tf.keras.layers.Dense(
units=lg_channel,
kernel_initializer=tf.
random_normal_initializer(stddev=.01))(
inputs=attn)
peers.append(attn)
combined_inputs = fusion_with_peer_attention(
combined_inputs,
index=i,
attention_mode=attention_mode,
attention_in=peers,
use_5d_mode=False)
graph = asn.block_group(inputs=combined_inputs,
filters=structure[i][2],
block_fn=block_fn,
blocks=num_blocks[block_number],
strides=structure[i][4],
name='block_group' + str(i),
block_level=structure[i][0],
num_frames=num_frames,
temporal_dilation=structure[i][3])
streams.append(graph)
if use_object_input:
inputs = [original_inputs, object_inputs]
else:
inputs = original_inputs
super(AssembleNetPlus, self).__init__(inputs=inputs,
outputs=streams,
**kwargs)
def fusion_with_peer_attention(inputs: List[tf.Tensor],
index: Optional[int] = None,
attention_mode: Optional[str] = None,
attention_in: Optional[List[tf.Tensor]] = None,
use_5d_mode: bool = False,
model_edge_weights: Optional[List[Any]] = None,
num_object_classes: Optional[int] = None):
"""Weighted summation of multiple tensors, while using peer-attention.
Summation weights are to be learned. Uses spatial max pooling and 1x1 conv.
to match their sizes. Before the summation, each connection (i.e., each input)
itself is scaled with channel-wise peer-attention. Notice that attention is
applied for each connection, conditioned based on attention_in.
Args:
inputs: A list of `Tensors`. Either 4D or 5D, depending of use_5d_mode.
index: `int` index of the block within the AssembleNet architecture. Used
for summation weight initial loading.
attention_mode: `str` specifying mode. If not `peer', does self-attention.
attention_in: A list of `Tensors' of size [batch*time, channels].
use_5d_mode: `bool` indicating whether the inputs are in 5D tensor or 4D.
model_edge_weights: AssembleNet model structure connection weights in the
string format.
num_object_classes: Assemblenet++ structure used object inputs so we should
use what dataset classes you might be use (e.g. ADE-20k 151 classes)
Returns:
The output `Tensor` after concatenation.
"""
if use_5d_mode:
h_channel_loc = 2
conv_function = asn.conv3d_same_padding
else:
h_channel_loc = 1
conv_function = asn.conv2d_fixed_padding
# If only 1 input.
if len(inputs) == 1:
inputs[0] = apply_attention(inputs[0], attention_mode, attention_in,
use_5d_mode)
return inputs[0]
# get smallest spatial size and largest channels
sm_size = [10000, 10000]
lg_channel = 0
for inp in inputs:
# assume batch X height x width x channels
sm_size[0] = min(sm_size[0], inp.shape[h_channel_loc])
sm_size[1] = min(sm_size[1], inp.shape[h_channel_loc + 1])
# Note that, when using object inputs, object channel sizes are usually big.
# Since we do not want the object channel size to increase the number of
# parameters for every fusion, we exclude it when computing lg_channel.
if inp.shape[-1] > lg_channel and inp.shape[-1] != num_object_classes: # pylint: disable=line-too-long
lg_channel = inp.shape[3]
per_channel_inps = _ApplyEdgeWeight(
weights_shape=[len(inputs)],
index=index,
use_5d_mode=use_5d_mode,
model_edge_weights=model_edge_weights)(inputs)
# Implementation of connectivity with peer-attention
if attention_mode:
for key, channel_inps in per_channel_inps.items():
for idx in range(len(channel_inps)):
with tf.name_scope('Connection_' + str(key) + '_' + str(idx)):
channel_inps[idx] = apply_attention(
channel_inps[idx], attention_mode, attention_in,
use_5d_mode)
# Adding 1x1 conv layers (to match channel size) and fusing all inputs.
# We add inputs with the same channels first before applying 1x1 conv to save
# memory.
inps = []
for key, channel_inps in per_channel_inps.items():
if len(channel_inps) < 1:
continue
if len(channel_inps) == 1:
if key == lg_channel:
inp = channel_inps[0]
else:
inp = conv_function(channel_inps[0],
lg_channel,
kernel_size=1,
strides=1)
inps.append(inp)
else:
if key == lg_channel:
inp = tf.add_n(channel_inps)
else:
inp = conv_function(channel_inps[0],
lg_channel,
kernel_size=1,
strides=1)
inps.append(inp)
return tf.add_n(inps)
def __init__(self, sequence_length, num_classes,pos_vocab_size, pos_embedding_size,
text_embedding_size,filter_sizes, num_heads, num_filters, l2_reg_lambda=0.0):
# Placeholders for input, output and dropout
self.text_embedded_chars = tf.placeholder(tf.float32, shape=[None, sequence_length, 768], name='text_embedded_chars')
self.input_p1 = tf.placeholder(tf.int32, shape=[None, sequence_length], name='input_p1')
self.input_p2 = tf.placeholder(tf.int32, shape=[None, sequence_length], name='input_p2')
self.input_y = tf.placeholder(tf.float32, shape=[None, num_classes], name='input_y') #[20 19]
self.dropout_keep_prob = tf.placeholder(tf.float32, name='dropout_keep_prob')
self.emb_dropout_keep_prob = tf.placeholder(tf.float32, name='emb_dropout_keep_prob')
initializer = tf.keras.initializers.glorot_normal
# Embedding layer
# with tf.device('/device:GPU:0'), tf.variable_scope("text-embedding"):
# # self.W_text = tf.Variable(tf.random_uniform([text_vocab_size, text_embedding_size], -0.25, 0.25), name="W_text")
# # self.text_embedded_chars = tf.nn.embedding_lookup(self.W_text, self.input_text) #[800 90 300]
# # self.text_embedded_chars = server_bert.get_sentence_embedding(self.input_text) #[800 90 768]
# # self.text_embedded_chars_trans = transformer.transformerencoder(self.text_embedded_chars)
# self.text_embedded_chars_change = tf.layers.dense(self.text_embedded_chars, units=300,activation=tf.nn.relu,use_bias=True, trainable=True) #[800 90 300]
# print("change:",self.text_embedded_chars_change.get_shape())# (?, 90, 300)
# self.text_embedded_chars_expanded = tf.expand_dims(self.text_embedded_chars_change, -1) #[800 90 300 1]
# print(self.text_embedded_chars_expanded.get_shape())
with tf.device('/cpu:0'), tf.variable_scope("position-embedding"):
self.W_pos = tf.get_variable("W_pos", [pos_vocab_size, pos_embedding_size], initializer=initializer())
self.p1_embedded_chars = tf.nn.embedding_lookup(self.W_pos, self.input_p1)
self.p2_embedded_chars = tf.nn.embedding_lookup(self.W_pos, self.input_p2)
self.p1_embedded_chars_expanded = tf.expand_dims(self.p1_embedded_chars, -1) #[800 90 50 1]
self.p2_embedded_chars_expanded = tf.expand_dims(self.p2_embedded_chars, -1)
# self.embedded_chars_expanded = tf.concat([self.text_embedded_chars_expanded,
# self.p1_embedded_chars_expanded,
# self.p2_embedded_chars_expanded], 2) #[800 90 400 1]
_embedding_size = text_embedding_size
self.text_shape=tf.shape(self.text_embedded_chars)
# self.text_expand_shape=tf.shape(self.text_embedded_chars_expanded)
# self.pos_expand_shape=tf.shape(self.p1_embedded_chars_expanded)
# self.embedd_shape=tf.shape(self.text_embedded_chars_change)
# self.embedding_size_shape=tf.shape(_embedding_size)
# Dropout for Word Embedding
with tf.variable_scope('dropout-embeddings'):
self.embedded_chars = tf.nn.dropout(self.text_embedded_chars, self.emb_dropout_keep_prob)
# self-attention
with tf.variable_scope("self-attention"):
self.self_attn_output, self.self_alphas = multihead_attention(self.embedded_chars, self.embedded_chars,
num_units=768, num_heads=num_heads)
# print("attention shape:", self.self_attn.get_shape) #(?, 90 ,300)
self.self_attn = tf.layers.dense(self.self_attn_output, units=300,
activation=tf.nn.relu, use_bias=True,
trainable=True) # [800 90 300]
print("change:", self.self_attn.get_shape()) # (?, 90, 300)
self.self_atten_change =tf.expand_dims(self.self_attn, -1) #[800 90 300 1]
# Create a convolution + maxpool layer for each filter size
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
with tf.variable_scope("conv-maxpool-%s" % filter_size):
# Convolution Layer
conv = tf.layers.conv2d(self.self_atten_change, num_filters, [filter_size, _embedding_size],
kernel_initializer=initializer(), activation=tf.nn.relu,
name="conv") # num_filter=128,filter_size=2,3,4,5
print(conv.get_shape()) # (?,89,1, 128);(?88,1,128)(?87,1,128)(?86 1 128)
# Maxpooling over the outputs
pooled = tf.nn.max_pool(conv, ksize=[1, sequence_length - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1], padding='VALID', name="pool")
print(pooled.get_shape()) # (?, 1, 1, 128)
pooled_outputs.append(pooled)
# Combine all the pooled features
num_filters_total = num_filters * len(filter_sizes)
# print(pooled_outputs.get_shape())
print(np.array(pooled_outputs).shape) #（4，）
self.h_pool = tf.concat(pooled_outputs, 3)
# print(self.h_pool.get_shape()) #(?,1,1,512)
self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total])
# print(self.h_pool_flat.get_shape())#(?,512)
# Add dropout
with tf.variable_scope("dropout"):
self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob)
# Final scores and predictions
with tf.variable_scope("output"):
self.logits = tf.layers.dense(self.h_drop, num_classes, kernel_initializer=initializer())
print(self.logits.get_shape()) #(?,19)
self.predictions = tf.argmax(self.logits, 1, name="predictions")
# Calculate mean cross-entropy loss
with tf.variable_scope("loss"):
losses = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.input_y)
self.l2 = tf.add_n([tf.nn.l2_loss(v) for v in tf.trainable_variables()])
self.loss = tf.reduce_mean(losses) + l2_reg_lambda * self.l2
# Accuracy
with tf.name_scope("accuracy"):
correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, tf.float32), name="accuracy")
def conv_layer(input, filter, kernel, stride=1, layer_name="conv"):
with tf.name_scope(layer_name):
network = tf.layers.conv2d(inputs=input, filters=filter, kernel_size=kernel, strides=stride, padding='SAME')
return network
def get_next_stage_inputs(inputs,
crop_boxes,
batch_size=None,
image_size=256,
previous_batch_size=None,
full_image_size=1024,
image_folder=None,
image_format=None,
grid_offsets=None,
intersection_ratio_threshold=0.25,
epsilon=1e-8,
use_queue=False,
shuffle_buffer=1,
num_threads=1,
capacity=5000,
verbose=False):
"""
Create input queue for the second - and final - stage.
Args:
inputs, a dictionnary of inputs
crop_boxes, a (batch_size, num_crops, 4) tensor of crops
image_folder: Image directory, used for reloading the full resolution images if needed
batch_size: Batch size for the output of this pipeline
image_size: Size of the images patches in the new dataset
full_image_size: Size of the images to load before applying the croppings
grid_offsets: A (num_cells, num_cells) array
use_queue: Whether to use a queue or directly output the new inputs dictionary
shuffle_buffer: shuffle buffer of the output queue
num_threads: number of readers in the output queue
capacity: Output queue capacity
verbose: verbosity
"""
assert 0. <= intersection_ratio_threshold < 1.
num_crops = tf.shape(crop_boxes)[1]
new_inputs = {}
# new_im_id: (batch_size * num_crops,)
if 'im_id' in inputs:
with tf.name_scope('im_ids'):
new_inputs['im_id'] = tile_and_reshape(inputs['im_id'], num_crops)
# classes: (batch_size * num_crops, num_classes)
if 'class_labels' in inputs:
with tf.name_scope('class_labels'):
new_inputs['class_labels'] = tile_and_reshape(inputs['class_labels'], num_crops)
# new_image: (num_patches, image_size, image_size, 3)
with tf.name_scope('extract_image_patches'):
# Extract patches and resize
# crop_boxes_indices: (batch_size * num_crops,)
# crop_boxes_flat: (batch_size * num_crops, 4)
crop_boxes_indices = tf.ones(tf.shape(crop_boxes)[:2], dtype=tf.int32)
crop_boxes_indices = tf.cumsum(crop_boxes_indices, axis=0, exclusive=True)
crop_boxes_indices = tf.reshape(crop_boxes_indices, (-1,))
crop_boxes_flat = tf.gather(tf.reshape(crop_boxes, (-1, 4)), [1, 0, 3, 2], axis=-1)
new_inputs['image'] = tf.image.crop_and_resize(
inputs['image'], crop_boxes_flat, crop_boxes_indices,
(image_size, image_size), name='extract_groups')
# new_bounding_boxes: (num_patches, max_num_bbs, 4)
# rescale bounding boxes coordinates to the cropped image
if 'bounding_boxes' in inputs:
with tf.name_scope('shift_bbs'):
# bounding_boxes: (batch, num_crops, max_num_bbs, 4)
# crop_boxes: (batch, num_crops, 1, 4)
bounding_boxes = inputs['bounding_boxes']
max_num_bbs = bounding_boxes.get_shape()[1].value
bounding_boxes = tf.expand_dims(bounding_boxes, axis=1)
bounding_boxes = tf.tile(bounding_boxes, (1, num_crops, 1, 1))
crop_boxes = tf.expand_dims(crop_boxes, axis=2)
# Filter out cut bbs
ratios = utils.get_intersection_ratio(tf.split(bounding_boxes, 4, axis=-1), tf.split(crop_boxes, 4, axis=-1))
condition = tf.tile(ratios > intersection_ratio_threshold, (1, 1, 1, 4))
bounding_boxes *= tf.to_float(condition)
# Rescale coordinates to the cropped image
crop_mins, crop_maxs = tf.split(crop_boxes, 2, axis=-1)
bounding_boxes -= tf.tile(crop_mins, (1, 1, 1, 2))
bounding_boxes /= tf.maximum(epsilon, tf.tile(crop_maxs - crop_mins, (1, 1, 1, 2)))
bounding_boxes = tf.clip_by_value(bounding_boxes, 0., 1.)
bounding_boxes = tf.reshape(bounding_boxes, (-1, max_num_bbs, 4))
new_inputs['bounding_boxes'] = bounding_boxes
# number of valid boxes: (num_patches,)
if 'num_boxes' in inputs:
with tf.name_scope('num_boxes'):
valid_boxes = ((bounding_boxes[..., 2] > bounding_boxes[..., 0]) &
(bounding_boxes[..., 3] > bounding_boxes[..., 1]))
num_boxes = tf.to_float(valid_boxes)
new_inputs['num_boxes'] = tf.to_int32(tf.reduce_sum(num_boxes, axis=-1) )
# Compute the box presence in cell mask
# obj_i_mask_bbs: (num_patches, num_cells, num_cells, 1, num_gt)
if 'obj_i_mask_bbs' in inputs:
with tf.name_scope('grid_offsets'):
if grid_offsets is not None:
num_cells = grid_offsets.shape[:2]
grid_offsets_mins = grid_offsets / num_cells
grid_offsets_maxs = (grid_offsets + 1.) / num_cells
bounding_boxes = tf.reshape(bounding_boxes, (-1, 1, 1, max_num_bbs, 4))
mins, maxs = tf.split(bounding_boxes, 2, axis=-1)
inters = tf.maximum(0., tf.minimum(maxs, grid_offsets_maxs) - tf.maximum(mins, grid_offsets_mins))
inters = tf.reduce_prod(inters, axis=-1)
obj_i_mask = tf.expand_dims(tf.to_float(inters > 0.) , axis=-2)
new_inputs['obj_i_mask_bbs'] = obj_i_mask
# During training: enqueue the inputs
if use_queue:
assert batch_size is not None
filter_valid = tf.logical_and(crop_boxes[..., 2] > crop_boxes[..., 0], crop_boxes[..., 3] > crop_boxes[..., 1] )
filter_valid = tf.reshape(filter_valid, (-1,))
# TODO maybe_batch is deprecated
out_ = tf.train.maybe_batch(
new_inputs, filter_valid, batch_size, num_threads=num_threads, enqueue_many=True, capacity=capacity)
# During inference: process crops deterministically
else:
out_ = new_inputs
if verbose == 1:
print('\n'.join(" \033[32m%s\033[0m: shape=%s, dtype=%s" % (key, value.get_shape().as_list(), value.dtype)
for key, value in out_.items() if key != 'batch_size'))
elif verbose > 1:
print('\n'.join(" *%s*: shape=%s, dtype=%s" % (key, value.get_shape().as_list(), value.dtype)
for key, value in out_.items() if key != 'batch_size'))
return out_
def __init__(self, config, batch, word_mat=None, filter_sizes=None, embedding_size=None,num_filters=None,trainable=True, l2_reg_lambda=0.0, keep_prob=0.9, graph=None):
# Placeholders for input, output and dropout
self.config = config
self.graph = graph if graph is not None else tf.Graph()
self.trainable = trainable
if trainable == True:
self.input_x, self.input_x1, self.ch, self.qh, self.input_y, self.qa_id = batch.get_next() # self.y1 is (64, 3)self.alterh batch_size is[batch,3,alternative_len,chara_len]
else:
self.input_x, self.input_x1, self.ch, self.qh = batch.get_next() # self.y1 is (64, 3)self.alterh batch_size is[batch,3,alternative_len,chara_len]
self.dropout_keep_prob =keep_prob
self.global_step = tf.get_variable('global_step', shape=[], dtype=tf.int32,
initializer=tf.constant_initializer(0), trainable=False)
self.dropout = tf.placeholder_with_default(0.5, (), name="dropout")
# Keeping track of l2 regularization loss (optional)
l2_loss = tf.constant(0.0)
# Embedding layer
with tf.device('/cpu:0'), tf.name_scope("embedding"):
self.W = tf.get_variable("word_mat", initializer=tf.constant(word_mat, dtype=tf.float32),
trainable=True)
self.c_mask = tf.cast(self.input_x, tf.bool) # self.c为填充之后的长度是一致的，用0进行填充
self.q_mask = tf.cast(self.input_x1, tf.bool)
if trainable:
self.c_maxlen, self.q_maxlen, = config.para_limit, config.ques_limit,
else:
self.c_maxlen, self.q_maxlen = config.test_para_limit, config.test_ques_limit
self.embedded_chars = tf.nn.embedding_lookup(self.W, self.input_x)
self.embedded_chars1 = tf.nn.embedding_lookup(self.W, self.input_x1)
self.embedded_chars_expanded = tf.expand_dims(self.embedded_chars, -1)
self.embedded_question = tf.expand_dims(self.embedded_chars1, -1)
S = optimized_trilinear_for_attention([self.embedded_chars_expanded, self.embedded_question], self.c_maxlen, self.q_maxlen,
input_keep_prob=1.0 - self.dropout)
print(S,"2222222222222222222")
# Create a convolution + maxpool layer for each filter size
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
with tf.name_scope("conv-maxpool-%s" % filter_size):
# Convolution Layer
filter_shape = [filter_size, embedding_size, 1, num_filters]
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b")
conv = tf.nn.conv2d(
self.embedded_chars_expanded,
W,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
# Apply nonlinearity
h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
# Maxpooling over the outputs
pooled = tf.nn.max_pool(
h,
ksize=[1, config.para_limit - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding='VALID',
name="pool")
pooled_outputs.append(pooled)
# Combine all the pooled features
num_filters_total = num_filters * len(filter_sizes)
self.h_pool = tf.concat(pooled_outputs, 3)
self.h_pool_flat = tf.reshape(self.h_pool, [-1, num_filters_total])
# Add dropout
with tf.name_scope("dropout"):
self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob)
# Final (unnormalized) scores and predictions
with tf.name_scope("output"):
W = tf.get_variable(
"W",
shape=[num_filters_total, 3],
initializer=tf.contrib.layers.xavier_initializer())
b = tf.Variable(tf.constant(0.1, shape=[3]), name="b")
l2_loss += tf.nn.l2_loss(W)
l2_loss += tf.nn.l2_loss(b)
self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name="scores")
self.predictions = tf.argmax(self.scores, 1, name="predictions")
# Calculate mean cross-entropy loss
with tf.name_scope("loss"):
losses = tf.nn.softmax_cross_entropy_with_logits(logits=self.scores, labels=self.input_y)
self.loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss
# Accuracy
with tf.name_scope("accuracy"):
correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")
if config.decay is not None:
self.var_ema = tf.train.ExponentialMovingAverage(config.decay)
ema_op = self.var_ema.apply(tf.trainable_variables())
with tf.control_dependencies([ema_op]):
self.loss = tf.identity(self.loss)
self.assign_vars = []
for var in tf.global_variables():
v = self.var_ema.average(var)
if v:
self.assign_vars.append(tf.assign(var, v))
self.lr = tf.minimum(config.init_lr,
0.001 / tf.log(999.) * tf.log(tf.cast(self.global_step, tf.float32) + 1))
self.opt = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.8, beta2=0.999, epsilon=1e-7)
grads = self.opt.compute_gradients(self.loss)
gradients, variables = zip(*grads)
capped_grads, _ = tf.clip_by_global_norm(
gradients, config.grad_clip)
self.train_op = self.opt.apply_gradients(
zip(capped_grads, variables), global_step=self.global_step)
self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=3)
def body_sharded(self, sharded_features):
# Remove dropout if not training
hparams = self._hparams
dp = self._data_parallelism
if hparams.use_inputs:
decoder_input = dp(tf.squeeze, sharded_features["inputs"], 2)
decoder_self_attention_bias = None
else:
targets = sharded_features["targets"]
targets = dp(tf.squeeze, targets, 2)
(decoder_input, decoder_self_attention_bias, pad_remover) = dp(
attention_lm_moe_prepare_decoder, targets, hparams)
def preprocess(x):
return dp(common_layers.layer_preprocess, x, hparams)
def postprocess(x, y):
return dp(common_layers.layer_postprocess, x, y, hparams)
x = dp(tf.nn.dropout, decoder_input,
1.0 - hparams.layer_prepostprocess_dropout)
extra_loss = 0.0
if not hparams.use_inputs:
# As preprocess and postprocess are called with batch of size one (all
# batches concatenated), we just make sure that batch_norm is not use (
# should not either way)
assert hparams.norm_type != "batch"
tf.logging.info("Applying Padding Remover for the attention experts")
dp_remove_pad = functools.partial(
dp, remove_pad, pad_remover=pad_remover, mode=hparams.mode)
dp_restore_pad = functools.partial(
dp, restore_pad, ref_x=x, pad_remover=pad_remover, mode=hparams.mode)
else:
# Using identity function: No effect
dp_remove_pad = lambda x: x
dp_restore_pad = lambda x: x
if hparams.attention_exp_factor != 0:
tf.logging.info("Expand/compress tokens before sending them to experts")
dp_expand_bc = lambda x: dp( # pylint: disable=g-long-lambda
expand_batch_coordinates,
x,
hparams.attention_exp_factor)
dp_expand_x = lambda x: dp( # pylint: disable=g-long-lambda
common_attention.deconv_elems_1d,
x,
hparams.attention_exp_factor,
hparams.attention_exp_inputdim)
dp_compress_x = lambda x, l: dp( # pylint: disable=g-long-lambda
common_attention.conv_elems_1d,
x,
hparams.attention_exp_factor,
l)
else:
dp_expand_bc = lambda x: x
dp_expand_x = lambda x: x
dp_compress_x = lambda x, l: x
def print_shape(x, suffix, debug=False):
# To help debugging, print the input/output shapes at inference and eval
# Inference for long sequences can take a long time, so that's help to
# see the progression of the generation
if not debug and hparams.mode == ModeKeys.TRAIN:
return x
return tf.Print(x, [tf.shape(x)], "shape_x_{}".format(suffix))
with tf.name_scope("batch_coordinate_preprocess"):
batch_coordinate = dp(get_batch_coordinate, x)
batch_coordinate = dp_remove_pad(batch_coordinate)
batch_coordinate = dp_expand_bc(batch_coordinate)
batch_order = dp(get_batch_coordinate, x, axis=-1)
batch_order = dp_remove_pad(batch_order)
batch_order = dp_expand_bc(batch_order)
x = dp(print_shape, x, "in")
assert hparams.batch_size >= hparams.max_length
num_hidden_layers = (
len(hparams.attention_layers) or hparams.num_hidden_layers)
for layer in range(num_hidden_layers):
with tf.variable_scope("layer_%d" % layer):
# Use the layer type defined in attention_layers
if hparams.attention_layers:
attention_type = LAYER_SYMBOLS[hparams.attention_layers[layer]]
else:
attention_type = hparams.attention_type
with tf.variable_scope(
"attention_{}".format(attention_type)):
if attention_type in [
AttentionType.MULTIHEAD, AttentionType.MULTIHEAD_FULL]:
attention_dot_type = (
"local_mask_right" if hparams.attention_local else
"dot_product")
if attention_type == AttentionType.MULTIHEAD_FULL:
attention_dot_type = "dot_product"
y = dp(
common_attention.multihead_attention,
preprocess(x),
None,
decoder_self_attention_bias,
hparams.attention_key_channels or hparams.hidden_size,
hparams.attention_value_channels or hparams.hidden_size,
hparams.hidden_size,
hparams.num_heads,
hparams.attention_dropout,
attention_type=attention_dot_type,
block_length=hparams.attention_block_length,
name="decoder_self_attention")
elif attention_type == AttentionType.SPARSE_MULTIHEAD:
x_in = preprocess(x)
x_in = dp_remove_pad(x_in)
y, loss_experts = dp(
common_attention.multihead_attention_sparse_dot_prod,
x_in,
None,
None, # Bias is computed inside
hparams.attention_key_channels or hparams.hidden_size,
hparams.attention_value_channels or hparams.hidden_size,
hparams.hidden_size,
hparams.num_heads,
hparams.attention_dropout,
# Additional parameters
bi=[common_attention.BatchInfo(
coordinates=batch_coordinate[i],
order=batch_order[i], # No future mask
) for i in range(dp.n)],
use_map_fn=hparams.lsh_use_map_fn,
experts_params=dict(
nb_hyperplanes=hparams.lsh_num_hyperplanes,
),
)
y = dp_restore_pad(y)
# TODO(avaswani, epot, noam): Do we need to divide by num shards ?
extra_loss += tf.add_n(loss_experts) / dp.n
elif attention_type == AttentionType.SPARSE_MULTIHEAD_TRUNCATED:
x_in = preprocess(x)
y, loss_experts = dp(
common_attention.multihead_attention_sparse_truncated,
x_in,
None,
None, # Bias is computed inside
hparams.attention_key_channels or hparams.hidden_size,
hparams.attention_value_channels or hparams.hidden_size,
hparams.hidden_size,
hparams.num_heads,
hparams.attention_dropout,
# Additional parameters
bi=[common_attention.BatchInfo(
coordinates=batch_coordinate[i],
order=batch_order[i], # No future mask
) for i in range(dp.n)],
mask_right=True,
experts_params=dict(
nb_hyperplanes=hparams.lsh_num_hyperplanes,
),
)
# TODO(avaswani, epot, noam): Do we need to divide by num shards ?
extra_loss += tf.add_n(loss_experts) / dp.n
elif attention_type == AttentionType.MEMORY_EFFICIENT:
assert hparams.layer_preprocess_sequence == "n"
y = dp(
common_attention.multihead_self_attention_memory_efficient,
x,
decoder_self_attention_bias,
hparams.num_heads,
name="decoder_self_attention")
elif attention_type == AttentionType.MULTIHEAD_REDUCED:
y = dp(
common_attention.multihead_self_attention_reduced,
preprocess(x),
factor=hparams.attention_red_factor,
reduction_type=hparams.attention_reduction_type,
nonlinearity=hparams.attention_nonlinearity,
multihead_params=dict(
total_key_depth=
hparams.attention_key_channels or hparams.hidden_size,
total_value_depth=
hparams.attention_value_channels or hparams.hidden_size,
num_heads=hparams.num_heads,
dropout_rate=hparams.attention_dropout,
))
elif attention_type == AttentionType.LOCAL_EXPERTS:
x_in = preprocess(x)
x_in = dp_remove_pad(x_in)
x_in = dp_expand_x(x_in)
y, loss = dp(
common_attention.local_expert_attention,
x_in,
k=hparams.attention_moe_k,
loss_coef=hparams.attention_load_balance,
attention_num_experts=hparams.attention_num_experts,
train=hparams.mode == ModeKeys.TRAIN,
batch_coordinate=batch_coordinate,
mask_right=not hparams.use_inputs,
split_batch=bool(hparams.attention_split_batch),
attention_num_head=hparams.attention_num_head,
attention_kq_size=hparams.attention_kq_size,
attention_v_size=hparams.attention_v_size)
y = dp_compress_x(y, x[0].get_shape().as_list()[-1])
y = dp_restore_pad(y)
# TODO(avaswani, epot, noam): Do we need to divide by num shards ?
extra_loss += tf.add_n(loss) / dp.n
else:
raise ValueError("Only {} supported for now.".format(
AttentionType.get_choices()))
x = postprocess(x, y)
with tf.variable_scope("ffn"):
if hparams.memory_efficient_ffn:
assert hparams.layer_preprocess_sequence == "n"
y = dp(
common_layers.conv_hidden_relu_memory_efficient,
x,
hparams.filter_size)
else:
additional_conv_params = dict()
if hparams.use_sepconv:
additional_conv_params = dict(
padding="LEFT",
# Parameters copied from the transformer model
kernel_size=(3, 1),
second_kernel_size=(31, 1),
)
y = dp(
common_layers.conv_hidden_relu,
preprocess(x),
hparams.filter_size,
hparams.hidden_size,
dropout=hparams.relu_dropout,
**additional_conv_params
)
x = postprocess(x, y)
x = preprocess(x)
decoder_output = dp(tf.expand_dims, x, 2)
return decoder_output, extra_loss
def get_tf_dataset(tfrecords_file,
record_keys,
image_format,
max_num_bbs,
with_groups=True,
grouping_method='intersect',
grid_offsets=None,
with_classes=False,
num_classes=None,
batch_size=1,
drop_remainder=False,
num_epochs=1,
image_size=1024,
image_folder='',
data_augmentation_threshold=0.5,
num_devices=1,
num_threads=4,
shuffle_buffer=1,
prefetch_capacity=1,
make_initializable_iterator=False,
verbose=1):
"""Parse and load inputs from the given TFRecords as a tf.data.Dataset.
Args:
tfrecords_file: Path to the TFRecords file containing the data.
record_keys: Feature keys present in the TFrecords. Loaded from the metadata file
max_num_bbs: Maximum number of bounding boxes in the dataset. Used for reshaping the `bounding_boxes` records.
num_classes: Number of classes in the dataset. Only used if with_classes is True
with_classes: wheter to use class information
with_groups: whether to pre-compute grouped instances ground-truth
grid_offsets: Precomputed grid offsets
batch_size: Batch size.
num_epochs: Number of epochs to repeat.
image_size: The square size which to resize images to.
image_folder: path to the directory containing the images in the dataset.
data_augmentation_threshold: Data augmentation probabilitiy (in [0, 1])
num_devices: Number of devices
num_threads: Number of readers for the batch queue.
shuffle_buffer: Size of the shuffling buffer.
prefetch_capacity: Buffer size for prefetching.
make_initializable_iterator: if True, make an initializable and add its initializer to the collection `iterator_init`
verbose: Verbosity level
Returns:
A tf.Data.dataset iterator (and its initializer if initializable_iterator)
"""
assert grouping_method in ['intersect', 'intersect_with_density', 'unique_intersect']
assert not (with_classes and num_classes is None)
assert len(record_keys)
assert batch_size > 0
assert image_size > 0
assert 0. <= data_augmentation_threshold <= 1.
if grid_offsets is not None:
num_cells = grid_offsets.shape[:2]
assert num_devices > 0
assert num_threads > 0
assert shuffle_buffer > 0
if verbose == 2:
print(' \033[31m> load_inputs\033[0m')
elif verbose == 1:
print(' > load_inputs')
# Normalize grid cells offsets
if grid_offsets is not None:
grid_offsets_mins = grid_offsets / num_cells
grid_offsets_maxs = (grid_offsets + 1.) / num_cells
# Create TFRecords feature
features = read_tfrecords(record_keys, max_num_bbs=max_num_bbs)
def parsing_function(example_proto):
# Basic features
parsed_features = tf.parse_single_example(example_proto, features)
output = parse_basic_feature(parsed_features, image_folder, image_format, image_size=image_size)
bounding_boxes = output['bounding_boxes']
# Empty/active cells mask
# obj_i_mask_bbs: (num_cells, num_cells, 1, num_bbs)
mins, maxs = tf.split(bounding_boxes, 2, axis=-1) # (num_bbs, 2)
inters = tf.maximum(0., tf.minimum(maxs, grid_offsets_maxs) - tf.maximum(mins, grid_offsets_mins))
inters = tf.reduce_prod(inters, axis=-1)
obj_i_mask = tf.expand_dims(tf.to_float(inters > 0.) , axis=-2)
output["obj_i_mask_bbs"] = obj_i_mask
# Grouped instances
# group_bounding_boxes_per_cell: (num_cells, num_cells, 1, 4), cell bounding box after grouping
# group_flags: (num_cells, num_cells, 1, 1), whether a cell contains a group or not
# num_group_boxes: (), number of bounding boxes after grouping
if with_groups:
## Define group_mask: (num_cells, num_cells, num_bbs, 1)
## Maps each gt bounding box to a grid cell to be merged into a group
if grouping_method == 'intersect_with_density':
obj_i_mask = tf.expand_dims(tf.to_float(inters > 0.) , axis=-2)
obj_i_mask *= tf.expand_dims(tf.to_float(inters < 1. / (num_cells[0] * num_cells[1])) , axis=-2)
group_mask = tf.transpose(obj_i_mask, (0, 1, 3, 2)) # (num_cells, num_cells, num_bbs, 1)
elif grouping_method == 'unique_intersect':
# weight 1: Intersection between gt boxes and cells
# Upper bounded by 1
# (num_cells, num_cells, num_bbs)
w1 = inters * num_cells[0] * num_cells[1]
# weight 2: Opposite of How many objects coocurs in each cells
# Upper bounded by 1
# (num_cells, num_cells, 1)
w2 = 1. - tf.reduce_sum(obj_i_mask, axis=-1) / tf.to_float(output['num_boxes'])
# Assign each ground-truth to one unique group
group_mask = w1 * w2
group_mask = tf.to_float(group_mask > 0.) * tf.to_float(group_mask >= tf.reduce_max(group_mask, axis=(0, 1), keep_dims=True))
group_mask = tf.expand_dims(group_mask, axis=-1)
elif grouping_method == 'intersect':
group_mask = tf.transpose(obj_i_mask, (0, 1, 3, 2)) # (num_cells, num_cells, num_bbs, 1)
## Merge bbs coocurring in the same cell to form groups
mins = mins + 1. - group_mask
mins = tf.reduce_min(mins, axis=2, keep_dims=True) # (num_cells, num_cells, 1, 2)
maxs = maxs * group_mask
maxs = tf.reduce_max(maxs, axis=2, keep_dims=True)
group_bounding_boxes_per_cell = tf.concat([mins, maxs], axis=-1)
group_bounding_boxes_per_cell = tf.clip_by_value(group_bounding_boxes_per_cell, 0., 1.)
output["group_bounding_boxes_per_cell"] = group_bounding_boxes_per_cell
num_bbs_per_cell = tf.reduce_sum(group_mask, axis=2, keep_dims=True)
num_group_boxes = tf.reduce_sum(tf.to_int32(num_bbs_per_cell > 0))
output["num_group_boxes"] = num_group_boxes
group_flags = tf.maximum(tf.minimum(num_bbs_per_cell, 2.) - 1., 0.)
output["group_flags"] = group_flags
# is_flipped flag: (), indicates whether the image has been flipped during data augmentation
output["is_flipped"] = tf.constant(0.)
# Optional : add classes
if with_classes:
class_labels = tf.one_hot(parsed_features['classes'], num_classes,
axis=-1, on_value=1, off_value=0, dtype=tf.int32)
output['class_labels'] = class_labels
# Group classes (majority vote) # (num_cells, num_cells, 1, num_classes)
if with_groups:
percell_class_labels = tf.expand_dims(tf.expand_dims(class_labels, axis=0), axis=0)
percell_class_labels = group_mask * tf.to_float(percell_class_labels)
percell_class_labels = tf.reduce_sum(percell_class_labels, axis=2, keep_dims=True)
group_class_labels = tf.argmax(percell_class_labels, axis=-1)
group_class_labels = tf.one_hot(group_class_labels, num_classes,
axis=-1, on_value=1, off_value=0, dtype=tf.int32)
group_class_labels = tf.to_int32(percell_class_labels * tf.to_float(group_class_labels))
output["group_class_labels"] = group_class_labels
return output
## Create the dataset
with tf.name_scope('load_dataset'):
# Parse data
dataset = tf.data.TFRecordDataset(tfrecords_file)
# Map
dataset = dataset.shuffle(buffer_size=shuffle_buffer)
dataset = dataset.map(parsing_function, num_parallel_calls=num_threads)
# Repeat
if num_epochs > 1:
dataset = dataset.repeat(num_epochs)
# Batch
if tf.__version__ == '1.4.0':
dataset = dataset.batch(batch_size * num_devices)
else:
dataset = dataset.batch(batch_size * num_devices, drop_remainder=drop_remainder)
# Prefetch
if prefetch_capacity > 0:
dataset = dataset.prefetch(prefetch_capacity)
# Iterator
if make_initializable_iterator:
iterator = dataset.make_initializable_iterator()
iterator_init = iterator.initializer
tf.add_to_collection('iterator_init', iterator_init)
else:
iterator = dataset.make_one_shot_iterator()
iterator_init = None
batch = iterator.get_next()
## Apply data augmentation
with tf.name_scope('data_augmentation'):
if data_augmentation_threshold > 0.:
batch = apply_data_augmentation(batch, data_augmentation_threshold)
## Split across device
slice_dims = [0] * num_devices
unpadded_batch = tf.to_int32(tf.shape(batch['im_id'])[0])
for i in range(num_devices):
slice_dims[i] = tf.maximum(0, tf.minimum(batch_size, unpadded_batch))
unpadded_batch -= batch_size
inputs = [{} for _ in range(num_devices)]
for key, value in batch.items():
for i, split_value in enumerate(tf.split(value, slice_dims, axis=0)):
inputs[i][key] = split_value
## Verbose log
if verbose == 2:
print('\n'.join(" \033[32m%s\033[0m: shape=%s, dtype=%s" % (key, value.get_shape().as_list(), value.dtype)
for key, value in inputs[0].items()))
elif verbose == 1:
print('\n'.join(" *%s*: shape=%s, dtype=%s" % (key, value.get_shape().as_list(), value.dtype)
for key, value in inputs[0].items()))
return inputs, iterator_init
def predict(logits, labels):
with tf.name_scope('Accuracy') as scope:
pred = tf.nn.softmax(logits)
correct_prediction = tf.equal(tf.argmax(pred, 1), labels, name='accuracy_pre_example')
evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy_rate')
return evaluation_step
def create_model(self,in_data,one_hot=False,training=True):
with tf.variable_scope('scope_model',reuse=tf.AUTO_REUSE):
if one_hot: # not using embedding layer
embed = in_data[0]
else: # using embedding layer
with tf.name_scope('embed'):
if not config.PRETRAIN_EMBD_TAG:
self.embedding_size = config.EMBEDDING_SIZE
self.vocab_size = config.VOCAB_SIZE
embed_matrix = tf.get_variable('embed_matrix',
shape=[self.vocab_size, self.embedding_size],
initializer=tf.random_uniform_initializer())
else:
embed_matrix = tf.Variable(self.pretrain_embd,
trainable=config.PRETRAIN_EMBD_TRAINABLE,name='embed_matrix')
'''
#make sure the pretrain embd is load correctly
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
print(sess.run(embed_matrix))
'''
embed = tf.nn.embedding_lookup(embed_matrix, in_data[0], name='embedding')
print(embed_matrix.name)
print(embed.name)
self.create_actual_model(embed,training)
self.get_logits()
self.logits = tf.layers.batch_normalization(self.logits,name='Batch_Normalization')
self.logits = tf.nn.relu(self.logits,name='RELU')
labels = tf.argmax(input=in_data[1], axis=2)
predictions = tf.argmax(input=self.logits, axis=1)
print(labels)
print(predictions)
_, self.acc_op = tf.metrics.accuracy(labels=labels, predictions=predictions,name = 'my_metrics')
if training:
loss = tf.nn.softmax_cross_entropy_with_logits(logits=self.logits,
labels=in_data[1])
self.loss += loss
self.loss = tf.reduce_sum(self.loss)
params = tf.trainable_variables()
optimizer = tf.train.AdamOptimizer(config.LR)
grad_and_vars = tf.gradients(self.loss,params)
clipped_gradients , _= tf.clip_by_global_norm(grad_and_vars,0.5)
self.opt = optimizer.apply_gradients(zip(clipped_gradients,params),global_step = self.gstep)
def __init__(self, dim_a, dim_s, action_bound):
self.dim_s, self.dim_a, self.action_bound = dim_s, dim_a, action_bound
self.pointer = 0
self.memory = deque(maxlen=MEMORY_SIZE)
self.history_loss = []
self.history_TD_error = []
self.history_reward = []
self.sess = tf.Session()
# variable
self.state = tf.placeholder(tf.float32, [None, dim_s], name="state")
self.state_ = tf.placeholder(tf.float32, [None, dim_s], name="state_")
# self.action = tf.placeholder(tf.float32, [None, 1], name="action")
self.reward = tf.placeholder(tf.float32, [None, 1], name="reward")
# parameters
self.ae_param = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="actor/eval")
self.at_param = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="actor/target")
self.ce_param = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="critic/eval")
self.ct_param = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="critic/target")
#
with tf.variable_scope("actor"):
self.action = self._build_actor(self.state,
scope="eval",
trainable=True)
self.action_ = self._build_actor(self.state_,
scope="target",
trainable=False)
with tf.variable_scope("critic"):
self.q_value = self._build_critic(self.state,
self.action,
scope="eval",
trainable=True)
self.q_value_ = self._build_critic(self.state_,
self.action_,
scope="target",
trainable=False)
# loss
with tf.name_scope('loss'):
q_target = self.reward + GAMMA * self.q_value_
self.TD_error = tf.losses.mean_squared_error(labels=self.q_value,
predictions=q_target)
self.loss = -tf.reduce_mean(self.q_value)
with tf.variable_scope("summaries"):
tf.summary.scalar('TD', self.TD_error)
tf.summary.scalar('Loss', self.loss)
tf.summary.scalar('Reward', self.history_reward)
# train actor
with tf.name_scope('train'):
self.actor_train = tf.train.AdamOptimizer(LR_A).minimize(self.loss)
self.critic_train = tf.train.AdamOptimizer(LR_C).minimize(
self.TD_error)
self.sess.run(tf.global_variables_initializer())
# tensorboard
# merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter("./logs/", self.sess.graph)
print("Wrote Tensorboard Logs")
writer.close()
def __init__(self, sequence_length, num_classes, vocab_size,
embedding_size, filter_sizes, num_filters, num_domains,
l2_reg_lambda):
self.input_x = tf.placeholder(tf.int32, [None, sequence_length],
name="input_x")
self.input_y = tf.placeholder(tf.int32, [None, num_classes],
name="input_y")
self.input_d = tf.placeholder(tf.int32, [None, num_domains],
name="input_d")
l2_loss = tf.constant(0.0)
with tf.variable_scope("embedding"):
self.emb_W = tf.get_variable(
name="lookup_emb",
shape=[vocab_size, embedding_size],
initializer=tf.random_uniform_initializer(minval=-1.0,
maxval=1.0),
trainable=False)
embedded_chars = tf.nn.embedding_lookup(self.emb_W, self.input_x)
self.embedded_chars_expanded = tf.expand_dims(embedded_chars, -1)
#cnn+pooling
num_filters_total = num_filters * len(filter_sizes)
#shared
self.pub_h_pool = self.cnn("shared-public",
self.embedded_chars_expanded,
sequence_length, embedding_size,
filter_sizes, num_filters)
#private
self.pri_h_pool = self.cnn("shared-private",
self.embedded_chars_expanded,
sequence_length, embedding_size,
filter_sizes, num_filters)
#final representation
self.h_pool = tf.concat([self.pri_h_pool, self.pub_h_pool], axis=1)
input_dim = num_filters_total * 2
# Add dropout
with tf.name_scope("dropout"):
self.dropout_keep_prob = tf.placeholder(tf.float32,
name="dropout_keep_prob")
self.h_drop = tf.nn.dropout(self.h_pool, self.dropout_keep_prob)
self.pub_h_drop = tf.nn.dropout(self.pub_h_pool,
self.dropout_keep_prob)
self.pri_h_drop = tf.nn.dropout(self.pri_h_pool,
self.dropout_keep_prob)
hidden_size = 300
with tf.variable_scope("label"):
h1 = self.wx_plus_b(scope_name="h1",
x=self.h_drop,
size=[input_dim, hidden_size])
self.y_scores = self.wx_plus_b(scope_name='score',
x=h1,
size=[hidden_size, num_classes])
# CalculateMean cross-entropy loss
with tf.name_scope("loss"):
losses = tf.nn.softmax_cross_entropy_with_logits(
logits=self.y_scores,
labels=self.input_y,
)
self.y_loss = tf.reduce_mean(losses, name="task_loss")
with tf.name_scope("accuracy"):
self.y_pred = tf.argmax(self.y_scores, 1, name="predictions")
cor_pred = tf.cast(
tf.equal(self.y_pred, tf.argmax(self.input_y, 1)), "float")
self.y_accuracy = tf.reduce_mean(cor_pred, name="accuracy")
with tf.variable_scope("domain"):
h1 = self.wx_plus_b(scope_name="h1",
x=self.pub_h_drop,
size=[num_filters_total, hidden_size])
self.domain_scores = self.wx_plus_b(
scope_name="score", x=h1, size=[hidden_size, num_domains])
with tf.name_scope("loss"):
losses = tf.nn.softmax_cross_entropy_with_logits(
logits=self.domain_scores,
labels=self.input_d,
)
self.domain_loss = tf.reduce_mean(losses)
with tf.name_scope("accuracy"):
self.domain_pred = tf.argmax(self.domain_scores,
1,
name="predictions")
cor_pred = tf.cast(
tf.equal(self.domain_pred, tf.argmax(self.input_d, 1)),
"float")
self.domain_accuracy = tf.reduce_mean(cor_pred, name="acc")
with tf.variable_scope("gen"):
h1 = self.wx_plus_b(scope_name="h1",
x=self.pri_h_drop,
size=[num_filters_total, hidden_size])
self.gen_scores = self.wx_plus_b(scope_name="score",
x=h1,
size=[hidden_size, num_domains])
with tf.name_scope("loss"):
losses = tf.nn.softmax_cross_entropy_with_logits(
logits=self.gen_scores,
labels=self.input_d,
)
self.gen_loss = tf.reduce_mean(losses)
with tf.name_scope("accuracy"):
self.gen_pred = tf.argmax(self.gen_scores,
1,
name="predictions")
cor_pred = tf.cast(
tf.equal(self.gen_pred, tf.argmax(self.input_d, 1)),
"float")
self.gen_accuracy = tf.reduce_mean(cor_pred, name="acc")
def main():
train_log_file = misc_utils.train_log_file_dir()
ckpt_dir = misc_utils.ckpt_dir()
hparam_file = misc_utils.hparams_file_dir()
if not train_log_file.parent.exists():
os.makedirs(str(train_log_file.parent))
if not ckpt_dir.exists():
os.mkdir(str(ckpt_dir))
misc_utils.save_hparams(str(hparam_file))
g = tf.Graph()
with g.as_default():
with tf.name_scope("inputs"):
noisy_trainset_wav = misc_utils.datasets_dir().joinpath(PARAM.train_noisy_set)
clean_trainset_wav = misc_utils.datasets_dir().joinpath(PARAM.train_clean_set)
noisy_valset_wav = misc_utils.datasets_dir().joinpath(PARAM.validation_noisy_set)
clean_valset_wav = misc_utils.datasets_dir().joinpath(PARAM.validation_clean_set)
train_inputs = dataloader.get_batch_inputs_from_nosiyCleanDataset(noisy_trainset_wav,
clean_trainset_wav)
val_inputs = dataloader.get_batch_inputs_from_nosiyCleanDataset(noisy_valset_wav,
clean_valset_wav)
test_noisy_ph = tf.compat.v1.placeholder(tf.float32, shape=[1, None], name='mixed_batch')
ModelC, G, D = model_builder.get_model_class_and_var()
generator = G()
discriminator = D()
train_model = ModelC(PARAM.MODEL_TRAIN_KEY, generator, discriminator, train_inputs.mixed, train_inputs.clean)
# tf.compat.v1.get_variable_scope().reuse_variables()
val_model = ModelC(PARAM.MODEL_VALIDATE_KEY, generator, discriminator, val_inputs.mixed,val_inputs.clean)
test_model = ModelC(PARAM.MODEL_INFER_KEY, generator, discriminator, test_noisy_ph)
init = tf.group(tf.compat.v1.global_variables_initializer(),
tf.compat.v1.local_variables_initializer())
# misc_utils.show_variables(train_model.save_variables)
# misc_utils.show_variables(val_model.save_variables)
g.finalize()
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = PARAM.GPU_RAM_ALLOW_GROWTH
# config.gpu_options.per_process_gpu_memory_fraction = PARAM.GPU_PARTION
config.allow_soft_placement = False
sess = tf.compat.v1.Session(config=config, graph=g)
sess.run(init)
ckpts = tf.train.get_checkpoint_state(str(misc_utils.ckpt_dir()))
if ckpts is not None:
ckpt = ckpts.model_checkpoint_path
train_model.saver.restore(sess, ckpt)
else:
# region validation before training
misc_utils.print_log("\n\n", train_log_file)
misc_utils.print_log("sum_losses_G: "+str(PARAM.sum_losses_G)+"\n", train_log_file)
misc_utils.print_log("sum_losses_D: "+str(PARAM.sum_losses_D)+"\n", train_log_file)
misc_utils.print_log("show losses: "+str(PARAM.show_losses)+"\n", train_log_file)
evalOutputs_prev = eval_one_epoch(sess, val_model, val_inputs.initializer)
val_msg = "PRERUN.val> sum_loss:[G %.4F, D %.4f], show_losses:%s, Time:%.2Fs.\n\n\n" % (
evalOutputs_prev.avg_sum_loss_G,
evalOutputs_prev.avg_sum_loss_D,
evalOutputs_prev.avg_show_losses,
evalOutputs_prev.cost_time)
misc_utils.print_log(val_msg, train_log_file)
# endregion
avg_sum_loss_G = None
avg_sum_loss_D = None
avg_show_losses = None
save_time = time.time()
init_inputs_times = 1
sess.run(train_inputs.initializer)
while True:
try:
one_batch_time = time.time()
(sum_loss_G, sum_loss_D, show_losses, _,
lr, u_D, u_G
) = sess.run([train_model.losses.sum_loss_G,
train_model.losses.sum_loss_D,
train_model.losses.show_losses,
train_model.train_op,
train_model.lr,
train_model.discriminator._f_u,
train_model.generator._f_u,
# train_model.adam_p[:2]
])
global_step = sess.run(train_model._global_step_increase)
# print(adam_p)
if global_step > PARAM.max_step:
sess.close()
misc_utils.print_log("\n", train_log_file, no_time=True)
msg = '################### Training Done. ###################\n'
misc_utils.print_log(msg, train_log_file)
print('initial inputs %d times' % init_inputs_times)
break
if avg_sum_loss_G is None:
avg_sum_loss_G = sum_loss_G
avg_sum_loss_D = sum_loss_D
avg_show_losses = show_losses
else:
avg_sum_loss_G += sum_loss_G
avg_sum_loss_D += sum_loss_D
avg_show_losses += show_losses
u_str = "#u: [D %.2e, G %.2e]" % (u_D, u_G)
print("\rtrain step: %d/%d, cost %.2fs, sum_loss[G %.2f, D %.2f], show_losses %s, lr %.2e, %s " % (
global_step, PARAM.max_step, time.time()-one_batch_time, sum_loss_G, sum_loss_D,
str(show_losses), lr, u_str),
flush=True, end="")
one_batch_time = time.time()
# print(global_step)
if global_step % PARAM.step_to_save == 0:
print("\r "
" "
" \r", end="")
avg_sum_loss_G /= PARAM.step_to_save
avg_sum_loss_D /= PARAM.step_to_save
avg_show_losses /= PARAM.step_to_save
misc_utils.print_log(" Save step %03d:\n" % global_step, train_log_file)
misc_utils.print_log(" sum_losses_G: "+str(PARAM.sum_losses_G)+"\n", train_log_file)
misc_utils.print_log(" sum_losses_D: "+str(PARAM.sum_losses_D)+"\n", train_log_file)
misc_utils.print_log(" show losses : "+str(PARAM.show_losses)+"\n", train_log_file)
misc_utils.print_log(" %s\n" % u_str, train_log_file)
misc_utils.print_log(" Train > sum_loss:[G %.4f, D %.4f], show_losses:%s, lr:%.2e, Time:%ds. \n" % (
avg_sum_loss_G, avg_sum_loss_D, str(avg_show_losses), lr, time.time()-save_time),
train_log_file)
# val
evalOutputs = eval_one_epoch(sess, val_model, val_inputs.initializer)
misc_utils.print_log(" Validation> sum_loss:[G %.4f, D %.4f], show_losses:%s, Time:%ds. \n" % (
evalOutputs.avg_sum_loss_G, evalOutputs.avg_sum_loss_D,
str(evalOutputs.avg_show_losses), evalOutputs.cost_time),
train_log_file)
# test
if global_step > 2500:
testOutputs = test_one_epoch(sess, test_model)
misc_utils.print_log(" Test > Csig: %.3f, Cbak: %.3f, Covl: %.3f, pesq: %.3f,"
" ssnr: %.4f, lsd: %.4f, estoi: %.4f Time:%ds. \n" % (
testOutputs.csig, testOutputs.cbak, testOutputs.covl, testOutputs.pesq,
testOutputs.ssnr, testOutputs.lsd, testOutputs.estoi, testOutputs.cost_time),
train_log_file)
# save ckpt
ckpt_name = PARAM().config_name()+('_step%06d_trloss%.4f_valloss%.4f_lr%.2e_duration%ds' % (
global_step, avg_sum_loss_G, evalOutputs.avg_sum_loss_G, lr,
time.time()-save_time))
train_model.saver.save(sess, str(ckpt_dir.joinpath(ckpt_name)))
misc_utils.print_log("\n", train_log_file, no_time=True)
# init var
avg_sum_loss_G = None
avg_sum_loss_D = None
avg_show_losses = None
save_time = time.time()
except tf.errors.OutOfRangeError:
sess.run(train_inputs.initializer)
init_inputs_times += 1
def forward_propagation(images,
mnist,
nummatches,
batch_size,
maxlen,
train=False,
dropout=False):
audio_network = stack_layers([
conv_layer(10, 23, 64, name='audio-conv1-layer', padding='VALID'),
pool_layer(4, 1, 2, 1, name="audio-max-pool1-layer", padding='VALID'),
conv_layer(25, 1, 512, name='audio-conv2-layer', padding='VALID'),
mean_pool_layer(name="audio-mean-pool-layer", padding='VALID')
])
image_network = stack_layers([
conv_layer(5, 5, 32, name='image-conv1-layer'),
pool_layer(2, 2, 2, 2, name="image-max-pool1-layer"),
conv_layer(5, 5, 64, name='image-conv2-layer'),
pool_layer(2, 2, 2, 2, name="image-max-pool2-layer"),
flatten(),
fully_connected_layer(512,
keep_prob=0.5 if train and dropout else 1.0,
name="image-local1-layer"),
fully_connected_layer(512, keep_prob=1.0, name="image-local2-layer")
])
image_joint_network = stack_layers([
fully_connected_layer(512, keep_prob=1.0, name="joint-local-layer"),
])
classification_network = stack_layers([
fully_connected_layer(256, keep_prob=1.0, name="class-local2-layer"),
softmax_layer(8, name="class-softmax-layer")
])
specs = tf.concat([images] * COPY, 0)
mnistset = tf.unstack(mnist, axis=1)
m1 = image_network(mnistset[0])
m2 = image_network(mnistset[1])
m3 = image_network(mnistset[2])
m4 = image_network(mnistset[3])
m5 = image_network(mnistset[4])
m6 = image_network(mnistset[5])
m7 = image_network(mnistset[6])
m8 = image_network(mnistset[7])
m9 = image_network(mnistset[8])
tmp = tf.concat([m1, m2, m3, m4, m5, m6, m7, m8, m9], 1)
t1 = image_joint_network(tmp)
# t1 = image_network(mnist)[0]
tmp = audio_network(specs)
t2 = tf.reshape(tmp, [batch_size * COPY, 512])
embeddings = tf.concat([t1, t2], 1)
# print("embeddings",embeddings.get_shape())
_, logits, proba, prediction = classification_network(embeddings)
with tf.name_scope('accuracy'):
with tf.name_scope('accuracy'):
actual = nummatches
with tf.name_scope('num_correct'):
correct = tf.reduce_sum(
tf.to_int32(tf.equal(prediction, actual)))
with tf.name_scope('loss'):
labels_one_hot = tf.one_hot(nummatches, 8, on_value=1.0, off_value=0.0)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=labels_one_hot))
tf.summary.scalar('batch_loss', loss)
return correct, loss, proba, prediction
def __init__(self,
num_actions,
num_features,
num_feature_ids,
embedding_sizes,
hidden_layer_sizes,
seed=None,
gate_gradients=False,
use_locking=False,
embedding_init=1.0,
relu_init=1e-4,
bias_init=0.2,
softmax_init=1e-4,
averaging_decay=0.9999,
use_averaging=True,
check_parameters=True,
check_every=1,
allow_feature_weights=False,
only_train='',
arg_prefix=None,
**unused_kwargs):
"""Initialize the graph builder with parameters defining the network.
Args:
num_actions: int size of the set of parser actions
num_features: int list of dimensions of the feature vectors
num_feature_ids: int list of same length as num_features corresponding to
the sizes of the input feature spaces
embedding_sizes: int list of same length as num_features of the desired
embedding layer sizes
hidden_layer_sizes: int list of desired relu layer sizes; may be empty
seed: optional random initializer seed to enable reproducibility
gate_gradients: if True, gradient updates are computed synchronously,
ensuring consistency and reproducibility
use_locking: if True, use locking to avoid read-write contention when
updating Variables
embedding_init: sets the std dev of normal initializer of embeddings to
embedding_init / embedding_size ** .5
relu_init: sets the std dev of normal initializer of relu weights
to relu_init
bias_init: sets constant initializer of relu bias to bias_init
softmax_init: sets the std dev of normal initializer of softmax init
to softmax_init
averaging_decay: decay for exponential moving average when computing
averaged parameters, set to 1 to do vanilla averaging
use_averaging: whether to use moving averages of parameters during evals
check_parameters: whether to check for NaN/Inf parameters during
training
check_every: checks numerics every check_every steps.
allow_feature_weights: whether feature weights are allowed.
only_train: the comma separated set of parameter names to train. If empty,
all model parameters will be trained.
arg_prefix: prefix for context parameters.
"""
self._num_actions = num_actions
self._num_features = num_features
self._num_feature_ids = num_feature_ids
self._embedding_sizes = embedding_sizes
self._hidden_layer_sizes = hidden_layer_sizes
self._seed = seed
self._gate_gradients = gate_gradients
self._use_locking = use_locking
self._use_averaging = use_averaging
self._check_parameters = check_parameters
self._check_every = check_every
self._allow_feature_weights = allow_feature_weights
self._only_train = set(only_train.split(',')) if only_train else None
self._feature_size = len(embedding_sizes)
self._embedding_init = embedding_init
self._relu_init = relu_init
self._softmax_init = softmax_init
self._arg_prefix = arg_prefix
# Parameters of the network with respect to which training is done.
self.params = {}
# Other variables, with respect to which no training is done, but which we
# nonetheless need to save in order to capture the state of the graph.
self.variables = {}
# Operations to initialize any nodes that require initialization.
self.inits = {}
# Training- and eval-related nodes.
self.training = {}
self.evaluation = {}
self.saver = None
# Nodes to compute moving averages of parameters, called every train step.
self._averaging = {}
self._averaging_decay = averaging_decay
# After the following 'with' statement, we'll be able to re-enter the
# 'params' scope by re-using the self._param_scope member variable. See for
# instance _AddParam.
self.input = tf.placeholder(dtype=tf.string)
self.labels = tf.placeholder(dtype=tf.int32)
self.dropout = tf.placeholder(tf.float32)
self.input_type_indices = tf.placeholder(tf.int32, [None], name="input_type_indices")
self.input_mention_length = tf.placeholder(tf.int32, [None], name="input_mention_length")
self.input_mention_indices = tf.placeholder(tf.int32, [None, None], name="input_mention_indices")
with tf.name_scope('params') as self._param_scope:
self._relu_bias_init = tf.constant_initializer(bias_init)
self.training.update(self._BuildNetwork(self.input,
return_average=False))
features = sparse_to_tuple(features.tocoo())
num_features = features[2][1]
features_nonzero = features[1].shape[0]
# Create model
model = None
if model_str == 'gcn_ae':
model = GCNModelAE(placeholders, num_features, features_nonzero)
elif model_str == 'gcn_Dir':
model = GCNModelDir2(placeholders, num_features, num_nodes, features_nonzero)
pos_weight = float(adj.shape[0] * adj.shape[0] - adj.sum()) / adj.sum()
norm = adj.shape[0] * adj.shape[0] / float((adj.shape[0] * adj.shape[0] - adj.sum()) * 2)
# Optimizer
with tf.name_scope('optimizer'):
if model_str == 'gcn_ae':
opt = OptimizerAE(preds=model.reconstructions,
labels=tf.reshape(tf.sparse_tensor_to_dense(placeholders['adj_orig'],
validate_indices=False), [-1]),
pos_weight=pos_weight,
norm=norm)
elif model_str == 'gcn_Dir':
opt = OptimizerDir2(
model=model,
label_1=placeholders['labels_1'], label_2=placeholders['labels_2'],
mask=placeholders['labels_mask'], label_3=placeholders['labels_3'], )
# Initialize session
sess = tf.Session()
def global_step(self):
with tf.name_scope('global'):
step = tf.Variable(1, name='global_step', dtype=tf.int32, trainable=False)
return step
def _ReluWeightInitializer(self):
with tf.name_scope(self._param_scope):
return tf.random_normal_initializer(stddev=self._relu_init,
seed=self._seed)
def training_loop(
G_args = {}, # Options for generator network.
D_args = {}, # Options for discriminator network.
G_opt_args = {}, # Options for generator optimizer.
D_opt_args = {}, # Options for discriminator optimizer.
G_loss_args = {}, # Options for generator loss.
D_loss_args = {}, # Options for discriminator loss.
dataset_args = {}, # Options for dataset.load_dataset().
sched_args = {}, # Options for train.TrainingSchedule.
grid_args = {}, # Options for train.setup_snapshot_image_grid().
metric_arg_list = [], # Options for MetricGroup.
tf_config = {}, # Options for tflib.init_tf().
data_dir = None, # Directory to load datasets from.
G_smoothing_kimg = 10.0, # Half-life of the running average of generator weights.
minibatch_repeats = 4, # Number of minibatches to run before adjusting training parameters.
lazy_regularization = True, # Perform regularization as a separate training step?
G_reg_interval = 4, # How often the perform regularization for G? Ignored if lazy_regularization=False.
D_reg_interval = 16, # How often the perform regularization for D? Ignored if lazy_regularization=False.
reset_opt_for_new_lod = True, # Reset optimizer internal state (e.g. Adam moments) when new layers are introduced?
total_kimg = 25000, # Total length of the training, measured in thousands of real images.
mirror_augment = False, # Enable mirror augment?
drange_net = [0,1], # Dynamic range used when feeding image data to the networks.
image_snapshot_ticks = 50, # How often to save image snapshots? None = only save 'reals.png' and 'fakes-init.png'.
network_snapshot_ticks = 50, # How often to save network snapshots? None = only save 'networks-final.pkl'.
save_tf_graph = False, # Include full TensorFlow computation graph in the tfevents file?
save_weight_histograms = False, # Include weight histograms in the tfevents file?
resume_pkl = None, # Network pickle to resume training from, None = train from scratch.
resume_kimg = 0.0, # Assumed training progress at the beginning. Affects reporting and training schedule.
resume_time = 0.0, # Assumed wallclock time at the beginning. Affects reporting.
resume_with_new_nets = False): # Construct new networks according to G_args and D_args before resuming training?
# Initialize dnnlib and TensorFlow.
tflib.init_tf(tf_config)
num_gpus = dnnlib.submit_config.num_gpus
# Load training set.
training_set = dataset.load_dataset(data_dir=dnnlib.convert_path(data_dir), verbose=True, **dataset_args)
grid_size, grid_reals, grid_labels = misc.setup_snapshot_image_grid(training_set, **grid_args)
misc.save_image_grid(grid_reals, dnnlib.make_run_dir_path('reals.png'), drange=training_set.dynamic_range, grid_size=grid_size)
# Construct or load networks.
training_set.configure(minibatch_size=sched_args.batch_size)
with tf.device('/gpu:0'):
if resume_pkl is None or resume_with_new_nets:
print('Constructing networks...')
G = tflib.Network('G', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, **G_args)
D = tflib.Network('D', num_channels=training_set.shape[0], resolution=training_set.shape[1], label_size=training_set.label_size, **D_args)
Gs = G.clone('Gs')
start = 0
if resume_pkl is not None:
print('Loading networks from "%s"...' % resume_pkl)
rG, rD, rGs = misc.load_pkl(resume_pkl)
if resume_with_new_nets: G.copy_vars_from(rG); D.copy_vars_from(rD); Gs.copy_vars_from(rGs)
else: G = rG; D = rD; Gs = rGs
start = int(resume_pkl.split('-')[-1].split('.')[0]) // sched_args.batch_size
# Print layers and generate initial image snapshot.
G.print_layers(); D.print_layers()
grid_latents = np.random.randn(np.prod(grid_size), *G.input_shape[1:])
grid_fakes = G.run(grid_latents, grid_labels, is_validation=True, minibatch_size=sched_args.batch_size)
misc.save_image_grid(grid_fakes, dnnlib.make_run_dir_path('fakes_init.png'), drange=drange_net, grid_size=grid_size)
global_step = tf.Variable(start, trainable=False, name='learning_rate_step')
learning_rate = tf.train.exponential_decay(sched_args.lr, global_step, sched_args.decay_step,
sched_args.decay_rate, staircase=sched_args.stair)
add_global = global_step.assign_add(1)
D_opt = tflib.Optimizer(name='TrainD', learning_rate=learning_rate, **D_opt_args)
G_opt = tflib.Optimizer(name='TrainG', learning_rate=learning_rate, **G_opt_args)
for gpu in range(num_gpus):
print('build graph on gpu %s' % str(gpu))
with tf.name_scope('GPU%d' % gpu), tf.device('/gpu:%d' % gpu):
# Create GPU-specific shadow copies of G and D.
G_gpu = G if gpu == 0 else G.clone(G.name + '_shadow')
D_gpu = D if gpu == 0 else D.clone(D.name + '_shadow')
with tf.name_scope('DataFetch'):
reals_read, labels_read = training_set.get_minibatch_tf()
reals_read, labels_read = process_reals(reals_read, labels_read, mirror_augment,
training_set.dynamic_range, drange_net)
with tf.name_scope('Loss'), tf.control_dependencies(None):
loss, reg = dnnlib.util.call_func_by_name(G=G_gpu, D=D_gpu, opt=D_opt, training_set=training_set,
minibatch_size=sched_args.batch_size, reals=reals_read,
labels=labels_read, **D_loss_args)
with tf.control_dependencies([add_global]):
G_opt.register_gradients(loss, G_gpu.trainables)
D_opt.register_gradients(loss, D_gpu.trainables)
G_train_op = G_opt.apply_updates()
D_train_op = D_opt.apply_updates()
# Finalize graph.
with tf.device('/gpu:0'):
try:
peak_gpu_mem_op = tf.contrib.memory_stats.MaxBytesInUse()
except tf.errors.NotFoundError:
peak_gpu_mem_op = tf.constant(0)
tflib.init_uninitialized_vars()
print('Initializing logs...')
summary_log = tf.summary.FileWriter(dnnlib.make_run_dir_path())
if save_tf_graph:
summary_log.add_graph(tf.get_default_graph())
if save_weight_histograms:
G.setup_weight_histograms(); D.setup_weight_histograms()
metrics = metric_base.MetricGroup(metric_arg_list)
print('Training for %d kimg...\n' % total_kimg)
dnnlib.RunContext.get().update('', cur_epoch=resume_kimg, max_epoch=total_kimg)
maintenance_time = dnnlib.RunContext.get().get_last_update_interval()
cur_nimg = int(resume_kimg * 1000)
cur_tick = -1
tick_start_nimg = cur_nimg
prev_lod = -1.0
running_mb_counter = 0
while cur_nimg < total_kimg * 1000:
if dnnlib.RunContext.get().should_stop(): break
loss_, _, _, lr_ = tflib.run([loss, G_train_op, D_train_op, learning_rate])
cur_nimg += sched_args.batch_size * num_gpus
done = (cur_nimg >= total_kimg * 1000)
if cur_tick < 0 or cur_nimg >= tick_start_nimg + sched_args.tick_kimg * 1000 or done:
cur_tick += 1
tick_kimg = (cur_nimg - tick_start_nimg) / 1000.0
tick_start_nimg = cur_nimg
tick_time = dnnlib.RunContext.get().get_time_since_last_update()
total_time = dnnlib.RunContext.get().get_time_since_start() + resume_time
# Report progress.
print(
'tick %-5d kimg %-8.1f minibatch %-4d time %-12s sec/tick %-7.1f '
'sec/kimg %-7.2f maintenance %-6.1f gpumem %.1f loss %-8.1f lr %-2.5f' % (
autosummary('Progress/tick', cur_tick),
autosummary('Progress/kimg', cur_nimg / 1000.0),
autosummary('Progress/minibatch', sched_args.batch_size),
dnnlib.util.format_time(autosummary('Timing/total_sec', total_time)),
autosummary('Timing/sec_per_tick', tick_time),
autosummary('Timing/sec_per_kimg', tick_time / tick_kimg),
autosummary('Timing/maintenance_sec', maintenance_time),
autosummary('Resources/peak_gpu_mem_gb', peak_gpu_mem_op.eval() / 2 ** 30),
autosummary('loss', loss_),
autosummary('lr', lr_)))
autosummary('Timing/total_hours', total_time / (60.0 * 60.0))
autosummary('Timing/total_days', total_time / (24.0 * 60.0 * 60.0))
# Save snapshots.
if image_snapshot_ticks is not None and (cur_tick % image_snapshot_ticks == 0 or done):
grid_fakes = G.run(grid_latents, grid_labels, is_validation=True, minibatch_size=sched_args.batch_size)
misc.save_image_grid(grid_fakes, dnnlib.make_run_dir_path('fakes%06d.png' % (cur_nimg // 1000)),
drange=drange_net, grid_size=grid_size)
if network_snapshot_ticks is not None and (cur_tick % network_snapshot_ticks == 0 or done):
pkl = dnnlib.make_run_dir_path('network-snapshot-%06d.pkl' % (cur_nimg // 1000))
misc.save_pkl((G, D, Gs), pkl)
metrics.run(pkl, run_dir=dnnlib.make_run_dir_path(), data_dir=dnnlib.convert_path(data_dir),
num_gpus=num_gpus, tf_config=tf_config)
# Update summaries and RunContext.
metrics.update_autosummaries()
tflib.autosummary.save_summaries(summary_log, cur_nimg)
dnnlib.RunContext.get().update('%.2f' % 0.0, cur_epoch=cur_nimg // 1000, max_epoch=total_kimg)
maintenance_time = dnnlib.RunContext.get().get_last_update_interval() - tick_time
# Save final snapshot.
misc.save_pkl((G, D, Gs), dnnlib.make_run_dir_path('network-final.pkl'))
# All done.
summary_log.close()
training_set.close()
def __compute_stress(self, damage, effective_stress):
with tf.name_scope("Apply1MinusD"):
stress_vector = tf.multiply(
tf.subtract(1.0, tf.expand_dims(damage, 1)), effective_stress)
return stress_vector
cfg = Config
graph = tf.Graph()
bsize = cfg.bsize
with graph.as_default():
# tf Graph input
tf.set_random_seed(1)
x = tf.placeholder(tf.int32, shape=(cfg.bsize, cfg.max_seq_len), name="x")
y = tf.placeholder(tf.float32, shape=(cfg.bsize, 6), name="y")
em = tf.placeholder(tf.float32, shape=(embedding_matrix.shape[0], embedding_matrix.shape[1]), name="em")
keep_prob = tf.placeholder(dtype=tf.float32, name="keep_prob")
with tf.name_scope("Embedding"):
#embedding = tf.get_variable("embedding", [embedding_matrix.shape[0], embedding_matrix.shape[1]],
# dtype=tf.float32, initializer=tf.constant_initializer(embedding_matrix),
# trainable=False)
embedded_input = tf.nn.embedding_lookup(em, x, name="embedded_input")
with tf.variable_scope('forward'):
fw_cell1 = tf.contrib.rnn.UGRNNCell(64, activation=tf.nn.elu)
fw_cell1 = tf.nn.rnn_cell.DropoutWrapper(fw_cell1, output_keep_prob=keep_prob)
fw_cell2 = tf.contrib.rnn.UGRNNCell(64, activation=tf.nn.elu)
# fw_cell2 = tf.nn.rnn_cell.DropoutWrapper(fw_cell2, output_keep_prob=keep_prob)
stacked_fw_rnn = [fw_cell1, fw_cell2]
fw_multi_cell = tf.contrib.rnn.MultiRNNCell(cells=stacked_fw_rnn, state_is_tuple=True)
with tf.variable_scope('backward'):
bw_cell1 = tf.contrib.rnn.UGRNNCell(64, activation=tf.nn.elu)
def _shadownet_fun(features, labels, mode, params):
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
tower_features = features
tower_labels = labels
tower_losses = []
tower_gradvars = []
tower_preds = []
tower_tensor_dict = []
tower_seq_len = []
num_devices = num_gpus
device_type = 'gpu'
tower_batch_size = int(params.batch_size / num_devices)
for i in range(num_devices):
worker_device = '/{}:{}'.format(device_type, i)
device_setter = local_device_setter(worker_device=worker_device)
with tf.variable_scope('shadownet', reuse=bool(i != 0)):
with tf.name_scope('tower_%d' % i) as name_scope:
with tf.device(device_setter):
loss, gradvars, preds, tensor_dict, seq_len = _tower_fn(
is_training, tower_features[i], tower_labels[i], tower_batch_size, params.l_size)
tower_losses.append(loss)
tower_gradvars.append(gradvars)
tower_preds.append(preds)
tower_tensor_dict.append(tensor_dict)
tower_seq_len.append(seq_len)
if i == 0:
# Only trigger batch_norm moving mean and variance update from
# the 1st tower. Ideally, we should grab the updates from all
# towers but these stats accumulate extremely fast so we can
# ignore the other stats from the other towers without
# significant detriment.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
name_scope)
# Now compute global loss and gradients.
gradvars = []
with tf.name_scope('gradient_averaging'):
all_grads = {}
for grad, var in itertools.chain(*tower_gradvars):
if grad is not None:
all_grads.setdefault(var, []).append(grad)
for var, grads in six.iteritems(all_grads):
# Average gradients on the same device as the variables
with tf.device(var.device):
if len(grads) == 1:
avg_grad = grads[0]
else:
avg_grad = tf.multiply(tf.add_n(grads), 1. / len(grads))
gradvars.append((avg_grad, var))
# Device that runs the ops to apply global gradient updates.
consolidation_device = '/gpu:0' if variable_strategy == 'GPU' else '/cpu:0'
with tf.device(consolidation_device):
global_step = tf.train.get_global_step()
starter_learning_rate = params.learning_rate
learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step,
params.decay_steps, params.decay_rate,
staircase=True)
loss = tf.reduce_mean(tower_losses, name='loss')
decoded, log_prob = tf.nn.ctc_beam_search_decoder(tower_preds[0],
tower_seq_len[0] * np.ones(tower_batch_size),
merge_repeated=False)
sequence_dist = tf.reduce_mean(tf.edit_distance(tf.cast(decoded[0], tf.int32), tower_labels[0]))
sequence_lengths_pred = tf.bincount(tf.cast(decoded[0].indices[:, 0], tf.int32),
minlength=tf.shape(tower_labels[0])[1])
label_lengths_pred = tf.bincount(tf.cast(labels[0].indices[:, 0], tf.int32),
minlength=tf.shape(tower_labels[0])[1])
tensors_to_log = {'global_step': global_step, 'learning_rate': learning_rate, 'loss': loss}
'''
dist_to_log = {'global_step': global_step,
'learning_rate': learning_rate,
'loss': loss,
'train_seq_dist': sequence_dist,
'sequence_lengths_pred': sequence_lengths_pred,
'label_lengths_pred': label_lengths_pred}
'''
logging_hook = tf.train.LoggingTensorHook(
tensors=tensors_to_log, every_n_iter=10)
train_hooks = [logging_hook]
# dist_hook = tf.train.LoggingTensorHook(
# tensors=dist_to_log, every_n_iter=2000)
# train_hooks = [logging_hook, dist_hook]
optimizer = tf.train.AdadeltaOptimizer(learning_rate=learning_rate)
# optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate)
# optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
if params.sync:
optimizer = tf.train.SyncReplicasOptimizer(
optimizer, replicas_to_aggregate=num_workers)
sync_replicas_hook = optimizer.make_session_run_hook(params.is_chief)
train_hooks.append(sync_replicas_hook)
# Create single grouped train op
train_op = [
optimizer.apply_gradients(
gradvars, global_step=tf.train.get_global_step())
]
train_op.extend(update_ops)
train_op = tf.group(*train_op)
return tf.estimator.EstimatorSpec(
mode=mode,
loss=loss,
train_op=train_op,
training_hooks=train_hooks)
def ComputePredictions(self, theta, source_encs, source_paddings, targets,
src_segment_id):
"""Decodes `targets` given encoded source.
Args:
theta: A `.NestedMap` object containing weights' values of this layer and
its children layers.
source_encs: source encoding, of shape [time, batch, depth].
source_paddings: source encoding's padding, of shape [time, batch].
targets: A dict of string to tensors representing the targets one try to
predict. Each tensor in targets is of shape [batch, time].
src_segment_id: source segment id, of shape [time, batch].
Returns:
A Tensor with shape [time, batch, params.softmax.input_dim].
"""
p = self.params
time, batch = py_utils.GetShape(source_paddings, 2)
source_encs = py_utils.HasShape(source_encs, [time, batch, p.source_dim])
with tf.name_scope(p.name):
target_ids = tf.transpose(targets.ids)
target_paddings = py_utils.HasRank(targets.paddings, 2)
target_paddings = tf.expand_dims(tf.transpose(target_paddings), 2)
if p.packed_input:
target_segment_id = tf.expand_dims(tf.transpose(targets.segment_ids), 2)
else:
target_segment_id = tf.zeros_like(target_paddings)
if py_utils.use_tpu():
emb_device = cluster_factory.Current().WorkerDeviceInModelSplit(0)
else:
emb_device = ''
with tf.device(emb_device):
inputs = self.emb.EmbLookup(theta.emb, target_ids)
inputs = self.ApplyClipping(theta, inputs)
summary_utils.histogram(p, 'input_emb', inputs)
inputs = self.ApplyDropout(inputs)
self._emb_out = inputs
# Layer 0 interwines with attention.
(atten_ctxs, xs, atten_probs, _) = self.frnn_with_atten.FProp(
theta.frnn_with_atten,
source_encs,
source_paddings,
inputs,
target_paddings,
src_segment_id=src_segment_id,
segment_id=target_segment_id)
if p.add_summary:
self._AddAttenProbsSummary(source_paddings, targets, [atten_probs])
atten_ctxs = self.ApplyClipping(theta, atten_ctxs)
summary_utils.histogram(p, 'atten_ctxs', atten_ctxs)
for i, (layer, layer_theta) in enumerate(zip(self.frnn, theta.frnn)):
# Forward through Layer-(i + 1) because Layer-0 handled before.
ys, _ = layer.FProp(
layer_theta,
tf.concat([xs, atten_ctxs], 2),
target_paddings,
segment_id=target_segment_id)
ys = self.ApplyDropout(ys)
if 1 + i >= p.residual_start:
xs += ys # Residual skip
xs = self.ApplyClipping(theta, xs)
else:
xs = ys
summary_utils.histogram(p, 'layer_out_%s' % i, xs)
if p.feed_attention_context_vec_to_softmax:
xs = tf.concat([xs, atten_ctxs], 2)
return xs
def _make_graph(self):
self.logger.info("Generating training graph on {} GPUs ...".format(self.cfg.nr_gpus))
weights_initializer = slim.xavier_initializer()
biases_initializer = tf.constant_initializer(0.)
biases_regularizer = tf.no_regularizer
weights_regularizer = tf.contrib.layers.l2_regularizer(self.cfg.weight_decay)
tower_grads = []
with tf.variable_scope(tf.get_variable_scope()):
for i in range(self.cfg.nr_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as name_scope:
# Force all Variables to reside on the CPU.
with slim.arg_scope([slim.model_variable, slim.variable], device='/device:CPU:0'):
with slim.arg_scope([slim.conv2d, slim.conv2d_in_plane, \
slim.conv2d_transpose, slim.separable_conv2d,
slim.fully_connected],
weights_regularizer=weights_regularizer,
biases_regularizer=biases_regularizer,
weights_initializer=weights_initializer,
biases_initializer=biases_initializer):
# loss over single GPU
self.net.make_network(is_train=True)
if i == self.cfg.nr_gpus - 1:
loss = self.net.get_loss(include_wd=True)
else:
loss = self.net.get_loss()
self._input_list.append( self.net.get_inputs() )
tf.get_variable_scope().reuse_variables()
if i == 0:
if self.cfg.nr_gpus > 1 and self.cfg.bn_train is True:
self.logger.warning("BN is calculated only on single GPU.")
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, name_scope)
with tf.control_dependencies(extra_update_ops):
grads = self._optimizer.compute_gradients(loss)
else:
grads = self._optimizer.compute_gradients(loss)
final_grads = []
with tf.variable_scope('Gradient_Mult') as scope:
for grad, var in grads:
scale = 1.
if self.cfg.double_bias and '/biases:' in var.name:
scale *= 2.
if not np.allclose(scale, 1.):
grad = tf.multiply(grad, scale)
final_grads.append((grad, var))
tower_grads.append(final_grads)
if len(tower_grads) > 1:
grads = sum_gradients(tower_grads)
else:
grads = tower_grads[0]
if False:
variable_averages = tf.train.ExponentialMovingAverage(0.9999)
variables_to_average = (tf.trainable_variables() + tf.moving_average_variables())
variables_averages_op = variable_averages.apply(variables_to_average)
apply_gradient_op = self._optimizer.apply_gradients(grads)
train_op = tf.group(apply_gradient_op, variables_averages_op, *extra_update_ops)
else:
apply_gradient_op = self._optimizer.apply_gradients(grads)
train_op = tf.group(apply_gradient_op, *extra_update_ops)
return train_op
def getembedding(wordsetsize, embedding_size):
with tf.name_scope("embedding"):
embedding_dict = tf.Variable(tf.random.uniform(
[wordsetsize + 1, embedding_size], -1.0, 1.0),
name="dict")
return embedding_dict
def _FProp(self, theta, source_encs, source_paddings, targets,
src_segment_id):
"""Decodes `targets` given encoded source.
Args:
theta: A `.NestedMap` object containing weights' values of this layer and
its children layers.
source_encs: source encoding. When `p.is_transparent` is False, it is a
tensor of shape [time, batch, depth]. When `p.is_transparent` is True,
it is a tensor of shape [time, batch, depth, num_trans_layers] if
`p.is_eval` is True, and a list of `num_trans_layers` tensors of shape
[time, batch, depth] if `p.is_eval` is False.
source_paddings: source encoding's padding, of shape [time, batch].
targets: A dict of string to tensors representing the targets one try to
predict. Each tensor in targets is of shape [batch, time].
src_segment_id: source segment id, of shape [time, batch].
Returns:
Output of last decoder layer, [target_time, target_batch, source_dim].
"""
p = self.params
time, batch = py_utils.GetShape(source_paddings, 2)
if p.is_transparent:
if p.is_eval:
source_encs = py_utils.HasShape(
source_encs, [time, batch, p.source_dim, p.num_trans_layers])
source_encs = tf.unstack(source_encs, axis=3)
else:
assert isinstance(source_encs, list)
assert len(source_encs) == p.num_trans_layers
for i in range(p.num_trans_layers):
source_encs[i] = py_utils.HasShape(source_encs[i],
[time, batch, p.source_dim])
else:
source_encs = py_utils.HasShape(source_encs, [time, batch, p.source_dim])
source_encs = [source_encs] * p.num_trans_layers
with tf.name_scope(p.name):
# [batch, time]
target_ids = targets.ids
# [time, batch]
target_paddings = tf.transpose(targets.paddings)
target_segment_pos = None
target_segment_id = None
if p.packed_input:
target_segment_id = tf.transpose(targets.segment_ids)
target_segment_pos = targets.segment_pos
assert src_segment_id is not None, ('Need to provide src_segment_id '
'for packed input.')
# Embedding layer
# [batch, time, model_dim]
token_embs = self.token_emb.EmbLookup(theta.token_emb, target_ids)
target_time = py_utils.GetShape(target_ids)[1]
# [1, time, model_dim]
if p.packed_input:
posit_embs = self.position_emb.FPropWithPosition(
theta.position_emb, target_segment_pos)
else:
posit_embs = tf.expand_dims(
self.position_emb.FProp(theta.position_emb, target_time), 0)
# [time, batch, model_dim]
input_embs = token_embs + posit_embs
if p.model_dim != p.token_emb.embedding_dim:
input_embs = self.emb_proj.FProp(theta.emb_proj, input_embs)
input_embs = tf.transpose(input_embs, [1, 0, 2])
input_embs = self.input_dropout.FProp(theta.input_dropout, input_embs)
atten_probs = []
layer_in = input_embs
for i, (layer, layer_theta) in enumerate(zip(self.trans, theta.trans)):
# [time, batch, model_dim]
layer_out, probs = layer.FProp(
layer_theta,
layer_in,
target_paddings,
source_encs[i],
source_paddings,
source_segment_id=target_segment_id,
aux_segment_id=src_segment_id)
layer_in = layer_out
atten_probs.append(probs)
if p.add_summary:
self._AddAttenProbsSummary(source_paddings, targets, atten_probs)
return layer_out
def _scope_all(scope, default_scope=None):
with tf.variable_scope(scope, default_name=default_scope) as s,\
tf.name_scope(s.original_name_scope):
yield s
def _initialize(self):
input_var = tf.compat.v1.placeholder(tf.float32,
shape=(None, ) +
self._input_shape)
ys_var = tf.compat.v1.placeholder(dtype=tf.float32,
name='ys',
shape=(None, self._output_dim))
old_means_var = tf.compat.v1.placeholder(dtype=tf.float32,
name='old_means',
shape=(None,
self._output_dim))
old_log_stds_var = tf.compat.v1.placeholder(dtype=tf.float32,
name='old_log_stds',
shape=(None,
self._output_dim))
(_, means, log_stds, _, norm_means, norm_log_stds, self._x_mean,
self._x_std, self._y_mean, self._y_std,
dist) = self.build(input_var).outputs
normalized_ys_var = (ys_var - self._y_mean) / self._y_std
normalized_old_means_var = (old_means_var - self._y_mean) / self._y_std
normalized_old_log_stds_var = (old_log_stds_var -
tf.math.log(self._y_std))
normalized_dist_info_vars = dict(mean=norm_means,
log_std=norm_log_stds)
mean_kl = tf.reduce_mean(
dist.kl_sym(
dict(mean=normalized_old_means_var,
log_std=normalized_old_log_stds_var),
normalized_dist_info_vars,
))
loss = -tf.reduce_mean(
dist.log_likelihood_sym(normalized_ys_var,
normalized_dist_info_vars))
self._f_predict = tensor_utils.compile_function([input_var], means)
self._f_pdists = tensor_utils.compile_function([input_var],
[means, log_stds])
optimizer_args = dict(
loss=loss,
target=self,
network_outputs=[norm_means, norm_log_stds],
)
if self._use_trust_region:
optimizer_args['leq_constraint'] = (mean_kl, self._max_kl_step)
optimizer_args['inputs'] = [
input_var, ys_var, old_means_var, old_log_stds_var
]
else:
optimizer_args['inputs'] = [input_var, ys_var]
with tf.name_scope('update_opt'):
self._optimizer.update_opt(**optimizer_args)
