def top_k_logits(logits, k):
if k == 0:
# no truncation
return logits
def _top_k():
values, _ = tf.nn.top_k(logits, k=k)
min_values = values[:, -1, tf.newaxis]
return tf.where(
logits < min_values,
tf.ones_like(logits, dtype=logits.dtype) * -1e10,
logits,
)
return tf.cond(
tf.equal(k, 0),
lambda: logits,
lambda: _top_k(),
)
def predicting(self, rate):
hidden = tf.nn.relu(
tf.matmul(self.concatInput, self.weights["MLP1"]) +
self.biases["MLP1"])
logits = tf.matmul(hidden, self.weights["MLP2"]) + self.biases["MLP2"]
predictPossibility = tf.nn.sigmoid(logits)
accuracy = tf.reduce_mean(
tf.cast(
tf.equal(tf.cast(predictPossibility > 0.5, tf.float32),
self.y), tf.float32))
loss = tf.reduce_mean(
tf.nn.weighted_cross_entropy_with_logits(targets=self.y,
logits=logits,
pos_weight=rate))
tv = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
'training_variable')
l2_loss = self.l2_para * tf.reduce_sum([tf.nn.l2_loss(v) for v in tv])
loss += l2_loss
return loss, accuracy, predictPossibility
def __init__(self,
input_width=227,
input_height=227,
input_channels=3,
num_classes=1000,
learning_rate=0.01,
momentum=0.9,
keep_prob=0.5):
# From article: The learning rate was initialized at 0.01.
# From article: We trained our models using stochastic gradient descent with a batch size of 128 examples,
# momentum of 0.9, and weight decay of 0.0005
# From article: We initialized the weights in each layer from a zero-mean Gaussian distribution with standard
# deviation 0.01.
self.input_width = input_width
self.input_height = input_height
self.input_channels = input_channels
self.num_classes = num_classes
self.learning_rate = learning_rate
self.momentum = momentum
self.keep_prob = keep_prob
self.random_mean = 0
self.random_stddev = 0.01
# ----------------------------------------------------------------------------------------------------
# From article: We initialized the neuron biases in the second, fourth, and fifth convolutional layers, as well
# as in the fully-connected hidden layers, with the constant 1. ... We initialized the neuron biases in the
# remaining layers with the constant 0.
# Input: 227x227x3.
with tf.name_scope('input'):
self.X = tf.placeholder(dtype=tf.float32,
shape=[
None, self.input_height,
self.input_width, self.input_channels
],
name='X')
# Labels: 1000.
with tf.name_scope('labels'):
self.Y = tf.placeholder(dtype=tf.float32,
shape=[None, self.num_classes],
name='Y')
# Dropout keep prob.
with tf.name_scope('dropout'):
self.dropout_keep_prob = tf.placeholder(dtype=tf.float32,
shape=(),
name='dropout_keep_prob')
# Layer 1.
# [Input] ==> 227x227x3
# --> 227x227x3 ==> [Convolution: size=(11x11x3)x96, strides=4, padding=valid] ==> 55x55x96
# --> 55x55x96 ==> [ReLU] ==> 55x55x96
# --> 55x55x96 ==> [Local Response Normalization] ==> 55x55x96
# --> 55x55x96 ==> [Max-Pool: size=3x3, strides=2, padding=valid] ==> 27x27x96
# --> [Output] ==> 27x27x96
# Note: 48*2=96, One GPU runs the layer-parts at the top while the other runs the layer-parts at the bottom.
with tf.name_scope('layer1'):
layer1_activations = self.__conv(
input=self.X,
filter_width=11,
filter_height=11,
filters_count=96,
stride_x=4,
stride_y=4,
padding='VALID',
init_biases_with_the_constant_1=False)
layer1_lrn = self.__local_response_normalization(
input=layer1_activations)
layer1_pool = self.__max_pool(input=layer1_lrn,
filter_width=3,
filter_height=3,
stride_x=2,
stride_y=2,
padding='VALID')
# Layer 2.
# [Input] ==> 27x27x96
# --> 27x27x96 ==> [Convolution: size=(5x5x96)x256, strides=1, padding=same] ==> 27x27x256
# --> 27x27x256 ==> [ReLU] ==> 27x27x256
# --> 27x27x256 ==> [Local Response Normalization] ==> 27x27x256
# --> 27x27x256 ==> [Max-Pool: size=3x3, strides=2, padding=valid] ==> 13x13x256
# --> [Output] ==> 13x13x256
# Note: 128*2=256, One GPU runs the layer-parts at the top while the other runs the layer-parts at the bottom.
with tf.name_scope('layer2'):
layer2_activations = self.__conv(
input=layer1_pool,
filter_width=5,
filter_height=5,
filters_count=256,
stride_x=1,
stride_y=1,
padding='SAME',
init_biases_with_the_constant_1=True)
layer2_lrn = self.__local_response_normalization(
input=layer2_activations)
layer2_pool = self.__max_pool(input=layer2_lrn,
filter_width=3,
filter_height=3,
stride_x=2,
stride_y=2,
padding='VALID')
# Layer 3.
# [Input] ==> 13x13x256
# --> 13x13x256 ==> [Convolution: size=(3x3x256)x384, strides=1, padding=same] ==> 13x13x384
# --> 13x13x384 ==> [ReLU] ==> 13x13x384
# --> [Output] ==> 13x13x384
# Note: 192*2=384, One GPU runs the layer-parts at the top while the other runs the layer-parts at the bottom.
with tf.name_scope('layer3'):
layer3_activations = self.__conv(
input=layer2_pool,
filter_width=3,
filter_height=3,
filters_count=384,
stride_x=1,
stride_y=1,
padding='SAME',
init_biases_with_the_constant_1=False)
# Layer 4.
# [Input] ==> 13x13x384
# --> 13x13x384 ==> [Convolution: size=(3x3x384)x384, strides=1, padding=same] ==> 13x13x384
# --> 13x13x384 ==> [ReLU] ==> 13x13x384
# --> [Output] ==> 13x13x384
# Note: 192*2=384, One GPU runs the layer-parts at the top while the other runs the layer-parts at the bottom.
with tf.name_scope('layer4'):
layer4_activations = self.__conv(
input=layer3_activations,
filter_width=3,
filter_height=3,
filters_count=384,
stride_x=1,
stride_y=1,
padding='SAME',
init_biases_with_the_constant_1=True)
# Layer 5.
# [Input] ==> 13x13x384
# --> 13x13x384 ==> [Convolution: size=(3x3x384)x256, strides=1, padding=same] ==> 13x13x256
# --> 13x13x256 ==> [ReLU] ==> 13x13x256
# --> 13x13x256 ==> [Max-Pool: size=3x3, strides=2, padding=valid] ==> 6x6x256
# --> [Output] ==> 6x6x256
# Note: 128*2=256, One GPU runs the layer-parts at the top while the other runs the layer-parts at the bottom.
with tf.name_scope('layer5'):
layer5_activations = self.__conv(
input=layer4_activations,
filter_width=3,
filter_height=3,
filters_count=256,
stride_x=1,
stride_y=1,
padding='SAME',
init_biases_with_the_constant_1=True)
layer5_pool = self.__max_pool(input=layer5_activations,
filter_width=3,
filter_height=3,
stride_x=2,
stride_y=2,
padding='VALID')
# Layer 6.
# [Input] ==> 6x6x256=9216
# --> 9216 ==> [Fully Connected: neurons=4096] ==> 4096
# --> 4096 ==> [ReLU] ==> 4096
# --> 4096 ==> [Dropout] ==> 4096
# --> [Output] ==> 4096
# Note: 2048*2=4096, One GPU runs the layer-parts at the top while the other runs the layer-parts at the bottom.
with tf.name_scope('layer6'):
pool5_shape = layer5_pool.get_shape().as_list()
flattened_input_size = pool5_shape[1] * pool5_shape[
2] * pool5_shape[3]
layer6_fc = self.__fully_connected(
input=tf.reshape(layer5_pool, shape=[-1,
flattened_input_size]),
inputs_count=flattened_input_size,
outputs_count=4096,
relu=True,
init_biases_with_the_constant_1=True)
layer6_dropout = self.__dropout(input=layer6_fc)
# Layer 7.
# [Input] ==> 4096
# --> 4096 ==> [Fully Connected: neurons=4096] ==> 4096
# --> 4096 ==> [ReLU] ==> 4096
# --> 4096 ==> [Dropout] ==> 4096
# --> [Output] ==> 4096
# Note: 2048*2=4096, One GPU runs the layer-parts at the top while the other runs the layer-parts at the bottom.
with tf.name_scope('layer7'):
layer7_fc = self.__fully_connected(
input=layer6_dropout,
inputs_count=4096,
outputs_count=4096,
relu=True,
init_biases_with_the_constant_1=True)
layer7_dropout = self.__dropout(input=layer7_fc)
# Layer 8.
# [Input] ==> 4096
# --> 4096 ==> [Logits: neurons=1000] ==> 1000
# --> [Output] ==> 1000
with tf.name_scope('layer8'):
layer8_logits = self.__fully_connected(
input=layer7_dropout,
inputs_count=4096,
outputs_count=self.num_classes,
relu=False,
name='logits')
# Cross Entropy.
with tf.name_scope('cross_entropy'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(
logits=layer8_logits, labels=self.Y, name='cross_entropy')
self.__variable_summaries(cross_entropy)
# Training.
with tf.name_scope('training'):
loss_operation = tf.reduce_mean(cross_entropy,
name='loss_operation')
tf.summary.scalar(name='loss', tensor=loss_operation)
optimizer = tf.train.MomentumOptimizer(
learning_rate=self.learning_rate, momentum=self.momentum)
# self.training_operation = optimizer.minimize(loss_operation, name='training_operation')
grads_and_vars = optimizer.compute_gradients(loss_operation)
self.training_operation = optimizer.apply_gradients(
grads_and_vars, name='training_operation')
for grad, var in grads_and_vars:
if grad is not None:
with tf.name_scope(var.op.name + '/gradients'):
self.__variable_summaries(grad)
# Accuracy.
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(layer8_logits, 1),
tf.argmax(self.Y, 1),
name='correct_prediction')
self.accuracy_operation = tf.reduce_mean(tf.cast(
correct_prediction, tf.float32),
name='accuracy_operation')
tf.summary.scalar(name='accuracy', tensor=self.accuracy_operation)
def _build_outputs(self, images, labels, mode):
is_training = mode == mode_keys.TRAIN
model_outputs = {}
if "anchor_boxes" in labels:
anchor_boxes = labels["anchor_boxes"]
else:
anchor_boxes = anchor.Anchor(
self._params.architecture.min_level,
self._params.architecture.max_level,
self._params.anchor.num_scales,
self._params.anchor.aspect_ratios,
self._params.anchor.anchor_size,
images.get_shape().as_list()[1:3],
).multilevel_boxes
batch_size = tf.shape(input=images)[0]
for level in anchor_boxes:
anchor_boxes[level] = tf.tile(
tf.expand_dims(anchor_boxes[level], 0), [batch_size, 1, 1])
backbone_features = self._backbone_fn(images, is_training)
fpn_features = self._fpn_fn(backbone_features, is_training)
rpn_score_outputs, rpn_box_outputs = self._rpn_head_fn(
fpn_features, is_training)
model_outputs.update({
"rpn_score_outputs": rpn_score_outputs,
"rpn_box_outputs": rpn_box_outputs,
})
rpn_rois, _ = self._generate_rois_fn(
rpn_box_outputs,
rpn_score_outputs,
anchor_boxes,
labels["image_info"][:, 1, :],
is_training,
)
if is_training:
rpn_rois = tf.stop_gradient(rpn_rois)
# Sample proposals.
(
rpn_rois,
matched_gt_boxes,
matched_gt_classes,
matched_gt_indices,
) = self._sample_rois_fn(rpn_rois, labels["gt_boxes"],
labels["gt_classes"])
# Create bounding box training targets.
box_targets = box_utils.encode_boxes(
matched_gt_boxes, rpn_rois, weights=[10.0, 10.0, 5.0, 5.0])
# If the target is background, the box target is set to all 0s.
box_targets = tf.compat.v1.where(
tf.tile(
tf.expand_dims(tf.equal(matched_gt_classes, 0), axis=-1),
[1, 1, 4]),
tf.zeros_like(box_targets),
box_targets,
)
model_outputs.update({
"class_targets": matched_gt_classes,
"box_targets": box_targets,
})
roi_features = spatial_transform_ops.multilevel_crop_and_resize(
fpn_features, rpn_rois, output_size=7)
class_outputs, box_outputs = self._frcnn_head_fn(
roi_features, is_training)
model_outputs.update({
"class_outputs": class_outputs,
"box_outputs": box_outputs,
})
if not is_training:
detection_results = self._generate_detections_fn(
box_outputs, class_outputs, rpn_rois,
labels["image_info"][:, 1:2, :])
model_outputs.update(detection_results)
if not self._include_mask:
return model_outputs
if is_training:
(
rpn_rois,
classes,
mask_targets,
gather_nd_gt_indices,
) = self._sample_masks_fn(
rpn_rois,
matched_gt_boxes,
matched_gt_classes,
matched_gt_indices,
labels["gt_masks"],
)
mask_targets = tf.stop_gradient(mask_targets)
classes = tf.cast(classes, dtype=tf.int32)
model_outputs.update({
"mask_targets": mask_targets,
"sampled_class_targets": classes,
})
else:
rpn_rois = detection_results["detection_boxes"]
classes = tf.cast(detection_results["detection_classes"],
dtype=tf.int32)
mask_roi_features = spatial_transform_ops.multilevel_crop_and_resize(
fpn_features, rpn_rois, output_size=14)
mask_outputs = self._mrcnn_head_fn(mask_roi_features, classes,
is_training)
if is_training:
model_outputs.update({
"mask_outputs": mask_outputs,
})
else:
model_outputs.update(
{"detection_masks": tf.nn.sigmoid(mask_outputs)})
if not self._include_attributes:
return model_outputs
attribute_outputs = self._attributes_head_fn(mask_roi_features,
is_training)
if is_training:
attribute_targets = tf.gather_nd(
labels["gt_attributes"],
gather_nd_gt_indices) # [batch, K, num_attributes]
model_outputs.update({
"attribute_outputs": attribute_outputs,
"attribute_targets": attribute_targets,
})
else:
model_outputs["detection_attributes"] = tf.nn.sigmoid(
attribute_outputs)
return model_outputs
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(
cross_entropy)
sess = tf.Session()
# Train
init = tf.initialize_all_variables()
sess.run(init)
for i in range(1000):
batch_xs, batch_ys = mnist.train.next_batch(100)
train_step.run({x: batch_xs, y_: batch_ys}, sess)
# Test trained model
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print(accuracy.eval({x: mnist.test.images, y_: mnist.test.labels}, sess))
# Store variable
_W = W.eval(sess)
_b = b.eval(sess)
sess.close()
# Create new graph for exporting
g_2 = tf.Graph()
with g_2.as_default():
# Reconstruct graph
x_2 = tf.placeholder("float", [None, 784], name="input")
def main(trainModel=True,
buildConfusionMatrix=True,
restore=False,
buildClassifiedMatrix=True):
tf.disable_v2_behavior()
input_images = tf.placeholder(tf.float32, [None, 28, 28], name="Input")
real = tf.placeholder(tf.float32, [None, CLASSES], name="real_classes")
layer1 = create_conv_layer(tf.reshape(input_images, [-1, 28, 28, 1]),
1,
28, [5, 5], [2, 2],
name="conv_no_pool")
layer2 = create_conv_layer(layer1,
28,
56, [5, 5], [2, 2],
name='conv_with_pool')
conv_result = tf.reshape(layer2, [-1, 7 * 7 * 56])
relu_layer_weight = tf.Variable(tf.truncated_normal([7 * 7 * 56, 1000],
stddev=STDDEV * 2),
name='relu_layer_weight')
rely_layer_bias = tf.Variable(tf.truncated_normal([1000],
stddev=STDDEV / 2),
name='rely_layer_bias')
relu_layer = tf.matmul(conv_result, relu_layer_weight) + rely_layer_bias
relu_layer = tf.nn.relu(relu_layer)
relu_layer = tf.nn.dropout(relu_layer, DROPOUT)
final_layer_weight = tf.Variable(tf.truncated_normal([1000, CLASSES],
stddev=STDDEV * 2),
name='final_layer_weight')
final_layer_bias = tf.Variable(tf.truncated_normal([CLASSES],
stddev=STDDEV / 2),
name='final_layer_bias')
final_layer = tf.matmul(relu_layer, final_layer_weight) + final_layer_bias
predicts = tf.nn.softmax(final_layer)
predicts_for_log = tf.clip_by_value(predicts, 1e-9, 0.999999999)
#crossEntropy = -tf.reduce_mean(tf.reduce_sum(y * tf.log(y_clipped) + (1 - y) * tf.log(1 - y_clipped), axis=1))
loss = -tf.reduce_mean(
tf.reduce_sum(real * tf.log(predicts_for_log) +
(1 - real) * tf.log(1 - predicts_for_log),
axis=1),
axis=0)
#test = tf.reduce_sum(real * tf.log(predicts_for_log) + (1 - real) * tf.log(1 - predicts_for_log), axis=1)
#loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=final_layer, labels=real))
optimiser = tf.train.GradientDescentOptimizer(
learning_rate=LEARNING_RATE).minimize(loss)
correct_prediction = tf.equal(tf.argmax(real, axis=1),
tf.argmax(predicts, axis=1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
confusion_matrix = tf.confusion_matrix(labels=tf.argmax(real, axis=1),
predictions=tf.argmax(predicts,
axis=1),
num_classes=CLASSES)
saver = tf.train.Saver()
# dataset = get_mnist_dataset()
dataset = get_fashion_dataset()
with tf.Session() as session:
session.run(tf.global_variables_initializer())
if restore:
saver.restore(session, SAVE_PATH)
if trainModel:
train(input_images, real, session, optimiser, loss, accuracy,
saver, dataset)
if buildConfusionMatrix:
test_cm = session.run(confusion_matrix,
feed_dict={
input_images: dataset.test_x,
real: dataset.test_y
})
draw_confusion_matrix(test_cm)
if buildClassifiedMatrix:
all_probs = session.run(predicts,
feed_dict={
input_images: dataset.test_x,
real: dataset.test_y
})
max_failure_picture_index = [[(-1, -1.0)] * CLASSES
for _ in range(CLASSES)]
for i in range(len(all_probs)):
real = np.argmax(dataset.test_y[i])
for j in range(CLASSES):
if max_failure_picture_index[real][j][1] < all_probs[i][j]:
max_failure_picture_index[real][j] = (i,
all_probs[i][j])
draw_max_failure_pictures(dataset.test_x,
max_failure_picture_index)