def luong_score(attention_dim, h_j, s_i):
h_proj = tf.get_variable(
'W_1', [attention_dim, tf.get_shape(h_j)[0]], dtype=tf.float32)
s_proj = tf.get_variable(
'W_2', [attention_dim, tf.get_shape(s_i)[0]], dtype=tf.float32)
score = dot_prod(h_proj, s_proj)
return score
def luong_score(attention_dim, h_j, s_i):
h_proj = tf.get_variable('W_1',
[attention_dim, tf.get_shape(h_j)[0]],
dtype=tf.float32)
s_proj = tf.get_variable('W_2',
[attention_dim, tf.get_shape(s_i)[0]],
dtype=tf.float32)
score = dot_prod(h_proj, s_proj)
return score
def make_nll_loss(logits,
targets,
logdetmap,
likefunc=mixture_likelihoods,
min_like=None,
input_is_set=False):
if (input_is_set):
# Here we're treating everything as independent
lshape = tf.get_shape(logits).as_list()
logits = tf.reshape(logits, [lshape[0], -1, lshape[-1]])
tshape = tf.get_shape(targets).as_list()
targets = tf.reshape(targets, [tshape[0], -1, tshape[-1]])
return make_nll_loss_orig(logits, targets, logdetmap, likefunc, min_like)
def position_embedding(self,inputs,maxlen,mask=True,scope='postion_embedding'):
'''
:param inputs: (N,T,E)
:param maxlen: must be >= T
:return:
'''
with tf.variable_scope(scope):
E = inputs.get_shape().as_list()[-1]
N, T = tf.get_shape(inputs)[:2]
position_ind = tf.tile(tf.expand_dims(tf.range(T),0),[N,1])
position_enc = np.array(
[pos / np.power(10000,(i - i % 2) / E) for i in range(E)]
for pos in range(maxlen)
)
position_enc[:,0::2] = np.sin(position_enc[:,0::2])
position_enc[:,1::2] = np.cos(position_enc[:,1::2])
position_enc = tf.convert_to_tensor(position_enc,tf.float32)
position_embed = tf.nn.embedding_lookup(position_enc,position_ind)
if mask:
position_embed = tf.where(tf.equal(inputs,0),inputs,position_embed)
return position_embed
def forward(self, X):
Z = X
for i in self.convpool_layer:
Z = i.forward(Z)
Z_shape = tf.get_shape().as_list()
Z = tf.reshape(Z, [-1, np.prod(Z_shape[1:])])
return tf.matmul(Z, self.W) + self.b
def fc(input, out_dim, activation, name = None):
input_shape = tf.get_shape(input)
with tf.variable_scope(name):
w = tf.get_variable(name='w1', shape=input_shape, initializer=tf.random_normal_initializer(mean=0, stddev=1))
b = tf.get_variable(name='b1', shape=[out_dim], initializer=tf.constant_initializer(0.1))
net = tf.matmul(w, input) + b
return net
def lightcnn(self, input, index, softmax=True):
with tf.name_scope("cnn" + str(index)):
B, L, D = tf.get_shape(input)
d = D / self.config.H
split_x = tf.transpose(
tf.reshape(input, shape=[B, L, d, self.config.H]),
[3, 0, 2, 1]) #[H,B,d,L]
output = tf.zeros_likes(split_x)
with tf.variable_scope("cnn" + str(index), reuse=True):
weight = tf.get_variable(
name='filters',
shape=[self.config.H, 1, 5],
dtype=tf.float32,
trainable=True,
initializer=tf.random_normal_initializer())
if softmax:
weight = tf.nn.softmax(weight, axis=-1)
for i in range(self.config.H):
output[i] = tf.layers.conv1d(inputs=split_x[i],
filters=1,
kernel_size=5,
strides=1,
padding='SAME',
kernel_initializer=weight,
name='conv' + str(i))
return tf.reshape(output, shape=[B, L, -1])
def bahdanau_score(attention_dim, h_j, s_i):
state_size = tf.get_shape(h_j)[0]
h_proj = tf.get_variable('W_1', [attention_dim, state_size],
dtype=tf.float32)
s_proj = tf.get_variable('W_2', [attention_dim, state_size],
dtype=tf.float32)
v = tf.get_variable('v', [attention_dim, state_size], dtype=tf.float32)
score = dot_prod(v, tf.tanh(h_proj + s_proj))
return score
def tf_ssd_bboxes_select_layer(predictions_layer,
localizations_layer,
select_threshold=None,
num_classes=21,
ignore_class=0,
scope=None):
"""Extract classes, scores and bounding boxes from features in one layer.
Batch-compatible: inputs are supposed to have batch-type shapes.
Args:
predictions_layer: A SSD prediction layer;
localizations_layer: A SSD localization layer;
select_threshold: Classification threshold for selecting a box. All boxes
under the threshold are set to 'zero'. If None, no threshold applied.
Return:
d_scores, d_bboxes: Dictionary of scores and bboxes Tensors of
size Batches X N x 1 | 4. Each key corresponding to a class.
"""
select_threshold = 0.0 if select_threshold is None else select_threshold
with tf.name_scope(scope, 'ssd_bboxes_select_layer',
[predictions_layer, localizations_layer]):
# Reshape features: Batches x N x N_labels | 4
p_shape = tf.get_shape(predictions_layer)
predictions_layer = tf.reshape(predictions_layer,
tf.stack([p_shape[0], -1, p_shape[-1]]))
l_shape = tf.get_shape(localizations_layer)
localizations_layer = tf.reshape(
localizations_layer, tf.stack([l_shape[0], -1, l_shape[-1]]))
d_scores = {}
d_bboxes = {}
for c in range(0, num_classes):
if c != ignore_class:
# Remove boxes under the threshold.
scores = predictions_layer[:, :, c]
fmask = tf.cast(tf.greater_equal(scores, select_threshold),
scores.dtype)
scores = scores * fmask
bboxes = localizations_layer * tf.expand_dims(fmask, axis=-1)
# Append to dictionary.
d_scores[c] = scores
d_bboxes[c] = bboxes
return d_scores, d_bboxes
def attention(attention_states, queries, num_heads=1):
"""Put attention masks on hidden using hidden_features and query."""
"""
@:param attention_states: states to be attentioned, shape=[batch_size, attn_len, attn_size]
@:param queries: current feature to calculate the attention, shape=[batch_size, attn_size, feature_len]
@:param num_heads: Number of attention heads that read from attention_states.
@:return A tuple of the form (ds, att_weights), where:
output: list of weights feature, [[shape=[batch_size, feature_len], ...]]
att_weights: list of attention weights, [[shape=[batch_size, attn_len]]
"""
batch_size, attn_len, attn_size = tf.get_shape(
attention_states)[0].value, tf.shape(
attention_states)[1].value, tf.shape(attention_states)[2].value
hidden = tf.reshape(attention_states, [-1, attn_len, 1, attn_size])
attention_vec_size = attn_size
hidden_features, v = [], []
for a in range(num_heads):
k = tf.get_variable("AttnW_%d" % a,
[1, 1, attn_size, attention_vec_size])
hidden_features.append(tf.nn.conv2d(hidden, k, [1, 1, 1, 1], "SAME"))
v.append(tf.get_variable("AttnV_%d" % a, [attention_vec_size]))
def sub_attention(query):
ds = [] # Results of attention reads will be stored here.
att_weights = [] # attention weights given specific query
if nest.is_sequence(query): # If the query is a tuple, flatten it.
query_list = nest.flatten(query)
for q in query_list: # Check that ndims == 2 if specified.
ndims = q.get_shape().ndims
if ndims:
assert ndims == 2
query = tf.concat(query_list, 1)
for a in range(num_heads):
with tf.variable_scope("Attention_%d" % a):
y = linear(query, attention_vec_size, True)
y = tf.reshape(y, [-1, 1, 1, attention_vec_size])
# Attention mask is a softmax of v^T * tanh(...).
s = tf.reduce_sum(v[a] * tf.tanh(hidden_features[a] + y),
[2, 3])
a = tf.nn.softmax(s)
att_weights.append(a)
# Now calculate the attention-weighted vector d.
d = tf.reduce_sum(
tf.reshape(a, [-1, attn_len, 1, 1]) * hidden, [1, 2])
ds.append(tf.reshape(d, [-1, attn_size]))
return ds, att_weights
query_list = tf.unstack(queries, axis=0)
outputs, attn_weights = [], []
for query in query_list:
output, att = sub_attention(query)
outputs.append(output)
attn_weights.append(att)
return outputs, attn_weights
def bahdanau_score(attention_dim, h_j, s_i):
state_size = tf.get_shape(h_j)[0]
h_proj = tf.get_variable('W_1',
[attention_dim, state_size],
dtype=tf.float32)
s_proj = tf.get_variable('W_2',
[attention_dim, state_size],
dtype=tf.float32)
v = tf.get_variable('v',
[attention_dim, state_size],
dtype=tf.float32)
score = dot_prod(v, tf.tanh(h_proj + s_proj))
return score
def replaceZi(Z, i, shape=None):
'''
take Z (NxK) and make K copies where the i'th element is replaced
consecutively by all K-length one-hot vectors. Returns KxNxK
'''
if shape is None:
N, K = [s.value for s in tf.get_shape(Z)]
else:
N, K = shape
one_hot = tf.one_hot(i, N, dtype=dtype)
replace = tf.eye(K, dtype=dtype)-tf.expand_dims(Z[i], 0)
return tf.expand_dims(Z, 0) + tf.einsum('i,kl->kil',
one_hot, replace)
def tf_ssd_bboxes_select_layer_all_classes(predictions_layer,
localizations_layer,
select_threshold=None):
"""Extract classes, scores and bounding boxes from features in one layer.
Batch-compatible: inputs are supposed to have batch-type shapes.
Args:
predictions_layer: A SSD prediction layer;
localizations_layer: A SSD localization layer;
select_threshold: Classification threshold for selecting a box. If None,
select boxes whose classification score is higher than 'no class'.
Return:
classes, scores, bboxes: Input Tensors.
"""
# Reshape features: Batches x N x N_labels | 4
p_shape = tf.get_shape(predictions_layer)
predictions_layer = tf.reshape(predictions_layer,
tf.stack([p_shape[0], -1, p_shape[-1]]))
l_shape = tf.get_shape(localizations_layer)
localizations_layer = tf.reshape(localizations_layer,
tf.stack([l_shape[0], -1, l_shape[-1]]))
# Boxes selection: use threshold or score > no-label criteria.
if select_threshold is None or select_threshold == 0:
# Class prediction and scores: assign 0. to 0-class
classes = tf.argmax(predictions_layer, axis=2)
scores = tf.reduce_max(predictions_layer, axis=2)
scores = scores * tf.cast(classes > 0, scores.dtype)
else:
sub_predictions = predictions_layer[:, :, 1:]
classes = tf.argmax(sub_predictions, axis=2) + 1
scores = tf.reduce_max(sub_predictions, axis=2)
# Only keep predictions higher than threshold.
mask = tf.greater(scores, select_threshold)
classes = classes * tf.cast(mask, classes.dtype)
scores = scores * tf.cast(mask, scores.dtype)
# Assume localization layer already decoded.
bboxes = localizations_layer
return classes, scores, bboxes
def conv3_br(x,
C=None,
L=None,
kernel_size=3,
strides=(1,1,1,1,1),
padding='SAME',
data_format='NDHWC',
dilations=(1,1,1,1,1),
variables_dict=None,
training=True,
do_bn=True,
do_relu=True):
"""
Wrapper around conv3d_transpose, batch_normalization, relu
"""
assert data_format == 'NDHWC'
ks = kernel_size
if C is None:
C = tf.get_shape(x).as_list()[data_format.index('C')]
if L is None:
L = C
w_xavier = tf.initializers.truncated_normal(0, stddev=np.sqrt(2. / (ks*ks*ks*C)), seed=0, dtype=tf.float32)
w = tf.get_variable(name='w', shape=[ks, ks, ks, C, L], dtype=tf.float32, initializer=w_xavier)
b = tf.get_variable(name='b', shape=[L], dtype=tf.float32, initializer=tf.initializers.zeros(tf.float32))
print(w, b)
x_conv = tf.nn.conv3d(x, w, strides=strides, padding=padding, data_format=data_format, dilations=dilations, name=None) + b
if do_bn:
bn_layer = tf.layers.BatchNormalization() # container object that allows us to access moving mean, moving variance
x_bn = bn_layer.apply(x_conv, training=training)
if do_relu:
x_relu = tf.nn.relu(x_bn)
if variables_dict is not None:
variables_dict['w'] = w
variables_dict['b'] = b
if do_bn:
variables_dict['bn_layer'] = bn_layer
variables_dict['gamma'] = bn_layer.gamma
variables_dict['beta'] = bn_layer.beta
variables_dict['moving_mean'] = bn_layer.moving_mean
variables_dict['moving_variance'] = bn_layer.moving_variance
return x_relu
def conv(x, filter_height, filter_width, num_filters, stride_y, stride_x, name, groups = 1, padding = 'SMAE'):
'''
Create a Convolutional Layer
'''
# The Channel of Input
input_channels = int(tf.get_shape(x)[-1])
# Create lambda funtion for convolution
convolve = lambda i, k: tf.nn.con2d(i, k, strides = [1, stride_y, stride_x, 1], padding = padding)
with tf.variable_scope(name) as scope:
# Create tf Variables for weights and biases of conv layer
weights = tf.get_Variable('weights', shape = [filter_height, filter_width, input_channels/groups, num_filters])
biases = tf.get_Variable('biases', shape = [num_filters])
if groups == 1:
conv = convolve(x, weights)
# In the cases of multible groups, split input and weights
else:
input_groups = tf.split(value = x, num_or_size_split = groups, axis = 3)
weights_groups = tf.split(value = weights, num_or_size_split = grooups, axis = 3)
output_groups = [convolve(i, j) for i, j in zip(input_groups, weights_groups)]
# Concat
conv = tf.concat(values = output_groups, axis = 3)
# add bias
bias = tf.reshape(tf.nn.bias_add(conv, biases), shape = tf.shape(conv))
# add relu
relu = tf.nn.relu(bias, name = scope.name)
return relu
def sorted_loss( prediction, ground_truth, weight_map=None):
ground_truth = tf.to_int64(ground_truth) #(865280,)
prediction = tf.cast(prediction, tf.float32)
prediction = tf.nn.softmax(prediction) #(865280, 14)
ids = tf.range(tf.to_int64(tf.shape(ground_truth)[0]), dtype=tf.int64)
ids = tf.stack([ids, ground_truth], axis=1) #(865280, 2)
one_hot = tf.SparseTensor( #(865280, 14)
indices=ids,
values=tf.ones_like(ground_truth, dtype=tf.float32),
dense_shape=tf.to_int64(tf.shape(prediction)))
if ground_truth is not None:
ground_truth = tf.reshape(ground_truth, [-1])
if not isinstance(prediction, (list, tuple)):
prediction = [prediction]
print("GROUND TRUTH SHAPE: ", tf.get_shape(ground_truth))
#writing ground_truth
file = open(ground_truth.txt,"w")
print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@file opened")
kernel_size = 3
stride = 1
padding = "SAME"
# x = tf.placeholder(tf.float32, [1, 9, 9, 32])
mask = tf.ones(shape = [1, 9, 9, 2])
weight_maskup = tf.ones(shape = [kernel_size, kernel_size, 2, 1])
slide_winsize = kernel_size * kernel_size * 2
slide_winsize = 18
def PartialConv(x, mask, channels, kernel_size, stride = 1, padding = "SAME", use_bias = True, use_sn = False, multi_channel = False, return_mask = True, scope = "partial_conv")
ch_in = tf.get_shape()[-1]
ch_out = channels
with tf.variable_scope(scope):
with tf.variable_scope("mask"):
if multi_channel:
weight_maskupdater = tf.ones(shape = [kernel_size, kernel_size, ch_in, ch_out])
slide_winsize = kernel_size * kernel_size * ch_in
else:
weight_maskupdater = tf.ones(shape = [kernel_size, kernel_size, 1, 1])
slide_winsize = kernel_size * kernel_size
update_mask = tf.nn.conv2d(mask, weight_maskup, [1, stride, stride, 1], padding = padding)
mask_ratio = slide_winsize/(update_mask)
def ssd_losses(logits,
localisations,
gclasses,
glocalisations,
gscores,
match_threshold=0.5,
negative_ratio=3.,
alpha=1.,
label_smoothing=0.,
device='/cpu:0',
scope=None):
with tf.name_scope(scope, 'ssd_losses'):
lshape = tf.get_shape(logits[0], 5)
num_classes = lshape[-1]
batch_size = lshape[0]
# Flatten out all vectors!
flogits = []
fgclasses = []
fgscores = []
flocalisations = []
fglocalisations = []
for i in range(len(logits)):
flogits.append(tf.reshape(logits[i], [-1, num_classes]))
fgclasses.append(tf.reshape(gclasses[i], [-1]))
fgscores.append(tf.reshape(gscores[i], [-1]))
flocalisations.append(tf.reshape(localisations[i], [-1, 4]))
fglocalisations.append(tf.reshape(glocalisations[i], [-1, 4]))
# And concat the crap!
logits = tf.concat(flogits, axis=0)
gclasses = tf.concat(fgclasses, axis=0)
gscores = tf.concat(fgscores, axis=0)
localisations = tf.concat(flocalisations, axis=0)
glocalisations = tf.concat(fglocalisations, axis=0)
dtype = logits.dtype
# Compute positive matching mask...
pmask = gscores > match_threshold
fpmask = tf.cast(pmask, dtype)
n_positives = tf.reduce_sum(fpmask)
# Hard negative mining...
no_classes = tf.cast(pmask, tf.int32)
predictions = slim.softmax(logits)
nmask = tf.logical_and(tf.logical_not(pmask), gscores > -0.5)
fnmask = tf.cast(nmask, dtype)
nvalues = tf.where(nmask, predictions[:, 0], 1. - fnmask)
nvalues_flat = tf.reshape(nvalues, [-1])
# Number of negative entries to select.
max_neg_entries = tf.cast(tf.reduce_sum(fnmask), tf.int32)
n_neg = tf.cast(negative_ratio * n_positives, tf.int32) + batch_size
n_neg = tf.minimum(n_neg, max_neg_entries)
val, idxes = tf.nn.top_k(-nvalues_flat, k=n_neg)
max_hard_pred = -val[-1]
# Final negative mask.
nmask = tf.logical_and(nmask, nvalues < max_hard_pred)
fnmask = tf.cast(nmask, dtype)
# Add cross-entropy loss.
with tf.name_scope('cross_entropy_pos'):
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=gclasses)
loss = tf.div(tf.reduce_sum(loss * fpmask),
batch_size,
name='value')
tf.losses.add_loss(loss)
with tf.name_scope('cross_entropy_neg'):
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=no_classes)
loss = tf.div(tf.reduce_sum(loss * fnmask),
batch_size,
name='value')
tf.losses.add_loss(loss)
# Add localization loss: smooth L1, L2, ...
with tf.name_scope('localization'):
# Weights Tensor: positive mask + random negative.
weights = tf.expand_dims(alpha * fpmask, axis=-1)
loss = custom_layers.abs_smooth(localisations - glocalisations)
loss = tf.div(tf.reduce_sum(loss * weights),
batch_size,
name='value')
tf.losses.add_loss(loss)
def ssd_losses_old(logits,
localisations,
gclasses,
glocalisations,
gscores,
match_threshold=0.5,
negative_ratio=3.,
alpha=1.,
label_smoothing=0.,
device='/cpu:0',
scope=None):
"""Loss functions for training the SSD 300 VGG network.
This function defines the different loss components of the SSD, and
adds them to the TF loss collection.
Arguments:
logits: (list of) predictions logits Tensors;
localisations: (list of) localisations Tensors;
gclasses: (list of) groundtruth labels Tensors;
glocalisations: (list of) groundtruth localisations Tensors;
gscores: (list of) groundtruth score Tensors;
"""
with tf.device(device):
with tf.name_scope(scope, 'ssd_losses'):
l_cross_pos = []
l_cross_neg = []
l_loc = []
for i in range(len(logits)):
dtype = logits[i].dtype
with tf.name_scope('block_%i' % i):
# Sizing weight...
wsize = tf.get_shape(logits[i], rank=5)
wsize = wsize[1] * wsize[2] * wsize[3]
# Positive mask.
pmask = gscores[i] > match_threshold
fpmask = tf.cast(pmask, dtype)
n_positives = tf.reduce_sum(fpmask)
# Select some random negative entries.
# n_entries = np.prod(gclasses[i].get_shape().as_list())
# r_positive = n_positives / n_entries
# r_negative = negative_ratio * n_positives / (n_entries - n_positives)
# Negative mask.
no_classes = tf.cast(pmask, tf.int32)
predictions = slim.softmax(logits[i])
nmask = tf.logical_and(tf.logical_not(pmask),
gscores[i] > -0.5)
fnmask = tf.cast(nmask, dtype)
nvalues = tf.where(nmask, predictions[:, :, :, :, 0],
1. - fnmask)
nvalues_flat = tf.reshape(nvalues, [-1])
# Number of negative entries to select.
n_neg = tf.cast(negative_ratio * n_positives, tf.int32)
n_neg = tf.maximum(n_neg, tf.size(nvalues_flat) // 8)
n_neg = tf.maximum(n_neg, tf.shape(nvalues)[0] * 4)
max_neg_entries = 1 + tf.cast(tf.reduce_sum(fnmask),
tf.int32)
n_neg = tf.minimum(n_neg, max_neg_entries)
val, idxes = tf.nn.top_k(-nvalues_flat, k=n_neg)
max_hard_pred = -val[-1]
# Final negative mask.
nmask = tf.logical_and(nmask, nvalues < max_hard_pred)
fnmask = tf.cast(nmask, dtype)
# Add cross-entropy loss.
with tf.name_scope('cross_entropy_pos'):
fpmask = wsize * fpmask
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits[i], labels=gclasses[i])
loss = tf.losses.compute_weighted_loss(loss, fpmask)
l_cross_pos.append(loss)
with tf.name_scope('cross_entropy_neg'):
fnmask = wsize * fnmask
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits[i], labels=no_classes)
loss = tf.losses.compute_weighted_loss(loss, fnmask)
l_cross_neg.append(loss)
# Add localization loss: smooth L1, L2, ...
with tf.name_scope('localization'):
# Weights Tensor: positive mask + random negative.
weights = tf.expand_dims(alpha * fpmask, axis=-1)
loss = custom_layers.abs_smooth(localisations[i] -
glocalisations[i])
loss = tf.losses.compute_weighted_loss(loss, weights)
l_loc.append(loss)
# Additional total losses...
with tf.name_scope('total'):
total_cross_pos = tf.add_n(l_cross_pos, 'cross_entropy_pos')
total_cross_neg = tf.add_n(l_cross_neg, 'cross_entropy_neg')
total_cross = tf.add(total_cross_pos, total_cross_neg,
'cross_entropy')
total_loc = tf.add_n(l_loc, 'localization')
# Add to EXTRA LOSSES TF.collection
tf.add_to_collection('EXTRA_LOSSES', total_cross_pos)
tf.add_to_collection('EXTRA_LOSSES', total_cross_neg)
tf.add_to_collection('EXTRA_LOSSES', total_cross)
tf.add_to_collection('EXTRA_LOSSES', total_loc)
def get_num_latent(self): return tf.get_shape(self.a)
def optimize_graph(self, loss, freeze_vars=None, train_vars=None):
"""Build the graph to optimize the loss function."""
# Global step
self.global_step = tf.Variable(0, name="global_step")
lr = self.learning_rate * tf.exp(
-tf.cast(self.global_step, tf.float32) * self.lr_decay)
# Instead of running optimizer.minimize directly, call compute gradients
# and process returned gradients
optimizer = tf.train.AdagradOptimizer(lr)
grads_and_vars = optimizer.compute_gradients(loss)
# Remove frozen indices from gradients
processes_grads_and_vars = []
for (g, v) in grads_and_vars:
if freeze_vars and (v in freeze_vars):
freeze_indices = freeze_vars[v]
# Remove all gradients for this variable
if freeze_indices == True:
g = None
# Process dense gradients
elif isinstance(g, tf.Tensor):
print("Freezing {} indicies of variable '{}' [D]".format(
len(freeze_indices), v.name))
update_shape = [len(freeze_indices)] + list(
g.get_shape()[1:])
gradient_mask = tf.zeros(update_shape, dtype=g.dtype)
g = tf.scatter_mul(g, freeze_indices, gradient_mask)
# Process sparse gradients
elif isinstance(g, tf.IndexedSlices):
print("Freezing {} indicies of variable '{}' [S]".format(
len(freeze_indices), v.name))
# Remove frozen indices from gradient
g = tf.sparse_mask(g, freeze_indices)
if train_vars and (v in train_vars):
trainable_indices = train_vars[v]
# Process dense gradients
if isinstance(g, tf.Tensor):
print("Training only on {} indicies of variable '{}' [D]".
format(len(freeze_indices), v.name))
gradient_mask = tf.scatter_nd(
tf.reshape(trainable_indices, [-1, 1]),
tf.ones(tf.get_shape(trainable_indices)),
[g.get_shape()[0], 1])
g = tf.multiply(g, gradient_mask)
# Process sparse gradients
elif isinstance(g, tf.IndexedSlices):
print("Training only on {} indicies of variable '{}' [S]".
format(len(freeze_indices), v.name))
raise RuntimeError
processes_grads_and_vars.append((g, v))
train = optimizer.apply_gradients(processes_grads_and_vars,
global_step=self.global_step,
name="train")
tf.summary.scalar("Learning rate", lr)
return train
"""6.第一个卷积层和池化层""" W_conv1 = weight_variable([5, 5, 3, 64], stddev=5e-2, w1=0.0) b_conv1 = bias_variable(0.0, [64]) h_conv1 = tf.nn.relu(conv2d(images_ph, W_conv1) + b_conv1) h_pool1 = max_pool_2x2(h_conv1) norm1 = tf.nn.lrn(h_pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75) """7.第二个卷积层和池化层""" W_conv2 = weight_variable([5, 5, 64, 64], stddev=5e-2, w1=0.0) b_conv2 = bias_variable(0.0, [64]) h_conv2 = tf.nn.relu(conv2d(norm1, W_conv2) + b_conv2) norm2 = tf.nn.lrn(h_conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75) h_pool2 = max_pool_2x2(norm2) """8.全连接层""" reshape = tf.reshape(h_pool2, [batch_size, -1]) dim = tf.get_shape(reshape)[1].value W_fc1 = weight_variable([dim, 384], stddev=0.04, w1=0.004) b_fc1 = bias_variable(0.1, [384]) h_fc1 = tf.nn.relu(tf.matmul(reshape, W_fc1) + b_fc1) """9.全连接层""" W_fc2 = weight_variable([184, 192], stddev=0.04, w1=0.004) b_fc2 = bias_variable(0.1, [192]) h_fc2 = tf.nn.relu(tf.matmul(h_fc1, W_fc2) + b_fc2) """10.模型inference输出结果""" W_rs = weight_variable([192, 10], stddev=1 / 192.0, w1=0.0) b_rs = bias_variable(0.0, [10]) logits = tf.add(tf.matmul(h_fc2, W_rs), b_rs) """11.计算CNN的损失""" def loss(logits, labels):
