def testDivisibleBy(self):
tf.reset_default_graph()
mobilenet_v2.mobilenet(
tf.placeholder(tf.float32, (10, 224, 224, 16)),
conv_defs=mobilenet_v2.V2_DEF,
divisible_by=16,
min_depth=32)
s = [op.outputs[0].get_shape().as_list()[-1] for op in find_ops('Conv2D')]
s = set(s)
self.assertSameElements([32, 64, 96, 160, 192, 320, 384, 576, 960, 1280,
1001], s)
def testDivisibleByWithArgScope(self):
tf.reset_default_graph()
# Verifies that depth_multiplier arg scope actually works
# if no default min_depth is provided.
with slim.arg_scope((mobilenet.depth_multiplier,), min_depth=32):
mobilenet_v2.mobilenet(
tf.placeholder(tf.float32, (10, 224, 224, 2)),
conv_defs=mobilenet_v2.V2_DEF, depth_multiplier=0.1)
s = [op.outputs[0].get_shape().as_list()[-1] for op in find_ops('Conv2D')]
s = set(s)
self.assertSameElements(s, [32, 192, 128, 1001])
def testFineGrained(self):
tf.reset_default_graph()
# Verifies that depth_multiplier arg scope actually works
# if no default min_depth is provided.
mobilenet_v2.mobilenet(
tf.placeholder(tf.float32, (10, 224, 224, 2)),
conv_defs=mobilenet_v2.V2_DEF, depth_multiplier=0.01,
finegrain_classification_mode=True)
s = [op.outputs[0].get_shape().as_list()[-1] for op in find_ops('Conv2D')]
s = set(s)
# All convolutions will be 8->48, except for the last one.
self.assertSameElements(s, [8, 48, 1001, 1280])
def testImageSizes(self):
for input_size, output_size in [(224, 7), (192, 6), (160, 5),
(128, 4), (96, 3)]:
tf.reset_default_graph()
_, ep = mobilenet_v2.mobilenet(
tf.placeholder(tf.float32, (10, input_size, input_size, 3)))
self.assertEqual(ep['layer_18/output'].get_shape().as_list()[1:3],
[output_size] * 2)
def build_model(inputs, depth_multiplier, num_classes, is_training):
with tf.variable_scope("Model"):
logits, _ = mobilenet_v2.mobilenet(inputs,
is_training=is_training,
depth_multiplier=depth_multiplier,
num_classes=num_classes)
#global_pool=True)
emo_logits = logits[:, :helpers.num_emotions]
sent_logits = logits[:, helpers.num_emotions:helpers.num_emotions +
helpers.num_sentiments]
return emo_logits, sent_logits
def net(inputs,
data_format='channels_last',
depth_mult=1.0,
VGG_PARAMS_FILE=None,
is_train=False):
if data_format != "channels_last":
print('only works for channels last now')
return None
with tf.contrib.slim.arg_scope(
mobilenet_v2.training_scope(is_training=is_train)):
logits, endpoints = mobilenet_v2.mobilenet(inputs,
base_only=True,
reuse=tf.AUTO_REUSE,
final_endpoint="layer_19",
depth_multiplier=depth_mult)
l15e = endpoints['layer_15/expansion_output']
l19 = endpoints['layer_19']
return [l15e, l19], endpoints
for i in range(num_channels):
channels[i] -= img_mean[i]
return tf.concat(axis=2, values=channels)
image_raw_data = tf.placeholder(tf.string, None)
img_data = tf.image.decode_jpeg(image_raw_data)
image = tf.image.convert_image_dtype(img_data, dtype=tf.uint8)
img_show = image
image.set_shape([img_size, img_size, 3])
image = tf.to_float(image)
image = _mean_image_subtraction(image)
image = tf.expand_dims(image, [0])
with tf.contrib.slim.arg_scope(mnv2.training_scope(is_training=False)):
logits, endpoint = mnv2.mobilenet(image, num_classes=4)
#logits=tf.Print(logits,[logits],'logits: ',summarize=32)
#logits=tf.sigmoid(logits)
#logits=tf.nn.softmax(logits)
#logits = tf.Print(logits, [logits], 'softmax: ', summarize=32)
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, './output/chamo_3000.000000_0.002084/chamo.ckpt')
while True:
img, cur_id = get_sample(0)
image_raw_data_jpg = tf.gfile.FastGFile('./re/chamo.jpg', 'rb').read()
re = sess.run([logits, img_show],
feed_dict={image_raw_data: image_raw_data_jpg})
print(re[0][0])
show_img = re[1]
plt.imshow(show_img)
def model_fn(features, labels, mode):
feature_dict = features
labels = tf.reshape(feature_dict["labels"], [-1])
if mode == tf.estimator.ModeKeys.TRAIN:
is_training = True
else:
is_training = False
if FLAGS.model == 'mobilenet':
scope = mobilenet_v2.training_scope(is_training=is_training)
with tf.contrib.slim.arg_scope(scope):
net, end_points = mobilenet_v2.mobilenet(
feature_dict["features"],
is_training=is_training,
num_classes=FLAGS.num_classes)
end_points['embedding'] = end_points['global_pool']
elif FLAGS.model == 'mobilefacenet':
scope = mobilenet_v2.training_scope(is_training=is_training)
with tf.contrib.slim.arg_scope(scope):
net, end_points = mobilefacenet.mobilefacenet(
feature_dict["features"],
is_training=is_training,
num_classes=FLAGS.num_classes)
elif FLAGS.model == 'metric_learning':
with slim.arg_scope(
inception_v1.inception_v1_arg_scope(weight_decay=0.0)):
net, end_points = proxy_metric_learning.metric_learning(
feature_dict["features"],
is_training=is_training,
num_classes=FLAGS.num_classes)
elif FLAGS.model == 'faceresnet':
net, end_points = mobilefacenet.faceresnet(
feature_dict["features"],
is_training=is_training,
num_classes=FLAGS.num_classes)
elif FLAGS.model == 'cifar100':
net, end_points = cifar.nin(feature_dict["features"],
labels,
is_training=is_training,
num_classes=FLAGS.num_classes)
else:
raise ValueError("Unknown model %s" % FLAGS.model)
small_embeddings = tf.squeeze(end_points['embedding'])
logits = end_points['Logits']
predictions = tf.cast(tf.argmax(logits, 1), dtype=tf.int32)
if FLAGS.final_activation in ['soft_thresh', 'none']:
abs_embeddings = tf.abs(small_embeddings)
else:
abs_embeddings = small_embeddings
nnz_small_embeddings = tf.cast(tf.less(FLAGS.zero_threshold,
abs_embeddings),
dtype=tf.float32)
small_sparsity = tf.reduce_sum(nnz_small_embeddings, axis=1)
small_sparsity = tf.reduce_mean(small_sparsity)
mean_nnz_col = tf.reduce_mean(nnz_small_embeddings, axis=0)
sum_nnz_col = tf.reduce_sum(nnz_small_embeddings, axis=0)
mean_flops_ub = tf.reduce_sum(sum_nnz_col *
(sum_nnz_col - 1)) / (FLAGS.batch_size *
(FLAGS.batch_size - 1))
mean_flops = tf.reduce_sum(mean_nnz_col * mean_nnz_col)
l1_norm_row = tf.reduce_sum(abs_embeddings, axis=1)
l1_norm_col = tf.reduce_mean(abs_embeddings, axis=0)
mean_l1_norm = tf.reduce_mean(l1_norm_row)
mean_flops_sur = tf.reduce_sum(l1_norm_col * l1_norm_col)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions_dict = {
'predictions': predictions,
'true_labels': feature_dict["labels"],
'true_label_texts': feature_dict["label_texts"],
'small_embeddings': small_embeddings,
'sparsity/small': small_sparsity,
'sparsity/flops': mean_flops_ub,
'filename': features["filename"]
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions_dict)
if FLAGS.model == 'mobilenet':
cr_ent_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels)
elif 'face' in FLAGS.model:
cr_ent_loss = mobilefacenet.arcface_loss(logits=logits,
labels=labels,
out_num=FLAGS.num_classes)
elif FLAGS.model == 'metric_learning':
cr_ent_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=labels)
# cr_ent_loss = mobilefacenet.arcface_loss(logits=logits, labels=labels, out_num=FLAGS.num_classes)
elif FLAGS.model == 'cifar100':
# cr_ent_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels)
cr_ent_loss = tf.contrib.losses.metric_learning.triplet_semihard_loss(
labels, small_embeddings, margin=0.1)
# cr_ent_loss = mobilefacenet.arcface_loss(logits=logits,
# labels=labels, out_num=FLAGS.num_classes, m=0.0)
# cr_ent_loss = triplet_semihard_loss(labels=labels, embeddings=small_embeddings,
# pairwise_distance=lambda embed: pairwise_distance_euclid(embed, squared=True), margin=0.3)
else:
raise ValueError('Unknown model %s' % FLAGS.model)
cr_ent_loss = tf.reduce_mean(
cr_ent_loss) + tf.losses.get_regularization_loss()
ema = tf.train.ExponentialMovingAverage(decay=0.99)
ema_op = ema.apply([mean_l1_norm])
moving_l1_norm = ema.average(mean_l1_norm)
global_step = tf.train.get_or_create_global_step()
all_ops = [ema_op]
l1_weight = tf.Variable(0.0, name='l1_weight', trainable=False)
if FLAGS.l1_weighing_scheme == 'constant':
l1_weight = FLAGS.l1_parameter
elif FLAGS.l1_weighing_scheme == 'dynamic_1':
l1_weight = FLAGS.l1_parameter / moving_l1_norm
l1_weight = tf.stop_gradient(l1_weight)
l1_weight = tf.train.piecewise_constant(x=global_step,
boundaries=[5],
values=[0.0, l1_weight])
elif FLAGS.l1_weighing_scheme == 'dynamic_2':
if FLAGS.sparsity_type == "flops_sur":
update_lr = 1e-5
else:
update_lr = 1e-4
update = update_lr * (FLAGS.l1_parameter - cr_ent_loss)
assign_op = tf.assign(l1_weight, tf.nn.relu(l1_weight + update))
all_ops.append(assign_op)
elif FLAGS.l1_weighing_scheme == 'dynamic_3':
update_lr = 1e-4
global_step = tf.train.get_or_create_global_step()
upper_bound = FLAGS.l1_parameter - (
FLAGS.l1_parameter - 12.0) * tf.cast(global_step, tf.float32) / 5e5
upper_bound = tf.cast(upper_bound, tf.float32)
update = update_lr * tf.sign(upper_bound - cr_ent_loss)
assign_op = tf.assign(l1_weight, l1_weight + tf.nn.relu(update))
all_ops.append(assign_op)
elif FLAGS.l1_weighing_scheme == 'dynamic_4':
l1_weight = FLAGS.l1_parameter * tf.minimum(
1.0, (tf.cast(global_step, tf.float32) / FLAGS.l1_p_steps)**2)
elif FLAGS.l1_weighing_scheme is None:
l1_weight = 0.0
else:
raise ValueError('Unknown l1_weighing_scheme %s' %
FLAGS.l1_weighing_scheme)
if FLAGS.sparsity_type == 'l1_norm':
sparsity_loss = l1_weight * mean_l1_norm
elif FLAGS.sparsity_type == 'flops_sur':
sparsity_loss = l1_weight * mean_flops_sur
elif FLAGS.sparsity_type is None:
sparsity_loss = 0.0
else:
raise ValueError("Unknown sparsity_type %d" % FLAGS.sparsity_type)
total_loss = cr_ent_loss + sparsity_loss
accuracy = tf.reduce_mean(
tf.cast(tf.equal(predictions, labels), dtype=tf.float32))
if mode == tf.estimator.ModeKeys.EVAL:
global_accuracy = tf.metrics.mean(accuracy)
global_small_sparsity = tf.metrics.mean(small_sparsity)
global_flops = tf.metrics.mean(mean_flops_ub)
global_l1_norm = tf.metrics.mean(mean_l1_norm)
global_mean_flops_sur = tf.metrics.mean(mean_flops_sur)
global_cr_ent_loss = tf.metrics.mean(cr_ent_loss)
metrics = {
'accuracy': global_accuracy,
'sparsity/small': global_small_sparsity,
'sparsity/flops': global_flops,
'l1_norm': global_l1_norm,
'l1_norm/flops_sur': global_mean_flops_sur,
'loss/cr_ent_loss': global_cr_ent_loss,
}
return tf.estimator.EstimatorSpec(mode,
loss=total_loss,
eval_metric_ops=metrics)
if mode == tf.estimator.ModeKeys.TRAIN:
base_lrate = FLAGS.learning_rate
learning_rate = tf.train.piecewise_constant(
x=global_step,
boundaries=[
FLAGS.decay_step if FLAGS.decay_step is not None else int(1e7)
],
values=[base_lrate, base_lrate / 10.0])
tf.summary.image("input", feature_dict["features"], max_outputs=1)
tf.summary.scalar('sparsity/small', small_sparsity)
tf.summary.scalar('sparsity/flops', mean_flops_ub)
tf.summary.scalar('accuracy', accuracy)
tf.summary.scalar('l1_norm', mean_l1_norm)
tf.summary.scalar('l1_norm/ema',
moving_l1_norm) # Comment this for mom
tf.summary.scalar('l1_norm/l1_weight', l1_weight)
tf.summary.scalar('l1_norm/flops_sur', mean_flops_sur)
tf.summary.scalar('learning_rate', learning_rate)
tf.summary.scalar('loss/cr_ent_loss', cr_ent_loss)
tf.summary.scalar(
'sparsity/ratio', mean_flops * FLAGS.embedding_size /
(small_sparsity * small_sparsity))
try:
tf.summary.scalar('loss/upper_bound', upper_bound)
except NameError:
print("Skipping 'upper_bound' summary")
for variable in tf.trainable_variables():
if 'soft_thresh' in variable.name:
print('Adding summary for lambda')
tf.summary.scalar('lambda', variable)
# Histogram summaries
# gamma = tf.get_default_graph().get_tensor_by_name('MobilenetV2/Conv_2/BatchNorm/gamma:0')
# pre_relu = tf.get_default_graph().get_tensor_by_name(
# 'MobilenetV2/Conv_2/BatchNorm/FusedBatchNorm:0')
# pre_relu = tf.squeeze(pre_relu)
# tf.summary.histogram('gamma', gamma)
# tf.summary.histogram('pre_relu', pre_relu[:, 237])
# tf.summary.histogram('small_activations', nnz_small_embeddings)
# tf.summary.histogram('small_activations/log', tf.log(nnz_small_embeddings + 1e-10))
# fl_sur_ratio = (mean_nnz_col * mean_nnz_col) / (l1_norm_col * l1_norm_col)
# tf.summary.histogram('fl_sur_ratio', fl_sur_ratio)
# tf.summary.histogram('fl_sur_ratio/log', tf.log(fl_sur_ratio + 1e-10))
# l1_sur_ratio = (mean_nnz_col * mean_nnz_col) / l1_norm_col
# tf.summary.histogram('l1_sur_ratio', l1_sur_ratio)
# tf.summary.histogram('l1_sur_ratio/log', tf.log(l1_sur_ratio + 1e-10))
if FLAGS.optimizer == 'mom':
optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=FLAGS.momentum)
elif FLAGS.optimizer == 'sgd':
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=learning_rate)
elif FLAGS.optimizer == 'adam':
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
epsilon=0.0001)
elif FLAGS.optimizer == 'rmsprop':
optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate,
epsilon=0.01,
momentum=FLAGS.momentum)
else:
raise ValueError("Unknown optimizer %s" % FLAGS.optimizer)
# Make the optimizer distributed
optimizer = hvd.DistributedOptimizer(optimizer)
train_op = tf.contrib.slim.learning.create_train_op(
total_loss,
optimizer,
global_step=tf.train.get_or_create_global_step())
all_ops.append(train_op)
merged = tf.group(*all_ops)
return tf.estimator.EstimatorSpec(mode,
loss=total_loss,
train_op=merged)
image, label = get_raw_img('./re')
image.set_shape([img_size, img_size, 3])
image = tf.to_float(image)
image = _mean_image_subtraction(image)
batchsize = 32
images, labels = tf.train.shuffle_batch([image, label],
batch_size=batchsize,
num_threads=1,
capacity=5 * batchsize,
min_after_dequeue=3 * batchsize)
#check_imgs(images, labels)
with tf.contrib.slim.arg_scope(mnv2.training_scope(is_training=True)):
logits, endpoint = mnv2.mobilenet(images,
num_classes=4,
reuse=tf.AUTO_REUSE)
#labels=tf.Print(labels, [labels], "labels:",summarize=32)
#labels=tf.Print(labels, [tf.sigmoid(logits)], "logits:",summarize=32)
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits))
optimizer = tf.train.AdamOptimizer(1e-4)
train_step = slim.learning.create_train_op(loss, optimizer)
init_op = tf.global_variables_initializer()
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
#saver.restore(sess, './output/chamo_4000.000000_0.000018/chamo.ckpt')
writer = tf.summary.FileWriter("logs/", sess.graph)
def mobilefacenet(features, is_training, num_classes):
if FLAGS.final_activation == 'step':
final_activation = step
elif FLAGS.final_activation == 'relu':
final_activation = tf.nn.relu
elif FLAGS.final_activation == 'relu6':
final_activation = tf.nn.relu6
elif FLAGS.final_activation == 'soft_thresh':
final_activation = soft_thresh(0.5)
elif FLAGS.final_activation == 'none':
final_activation = None
else:
raise ValueError('Unknown activation %s' % str(FLAGS.final_activation))
MFACENET_DEF = dict(
defaults={
# Note: these parameters of batch norm affect the architecture
# that's why they are here and not in training_scope.
(
slim.batch_norm, ): {
'center': True,
'scale': True
},
(slim.conv2d, slim.fully_connected, slim.separable_conv2d): {
'normalizer_fn': slim.batch_norm,
'activation_fn': tf.nn.leaky_relu,
'weights_regularizer': slim.l2_regularizer(4e-5)
},
(ops.expanded_conv, ): {
'split_expansion': 1,
'normalizer_fn': slim.batch_norm,
'residual': True
},
(slim.conv2d, slim.separable_conv2d): {
'padding': 'SAME'
},
},
spec=[
op(slim.conv2d, stride=2, num_outputs=64, kernel_size=[3, 3]),
op(slim.separable_conv2d,
num_outputs=None,
kernel_size=[3, 3],
depth_multiplier=1,
stride=1),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=2,
num_outputs=64),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=64),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=64),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=64),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=64),
op(ops.expanded_conv,
expansion_size=expand_input(4),
stride=2,
num_outputs=128),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=128),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=128),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=128),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=128),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=128),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=128),
op(ops.expanded_conv,
expansion_size=expand_input(4),
stride=2,
num_outputs=128),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=128),
op(ops.expanded_conv,
expansion_size=expand_input(2),
stride=1,
num_outputs=128),
op(slim.conv2d, stride=1, kernel_size=[1, 1], num_outputs=512),
op(slim.separable_conv2d,
num_outputs=None,
kernel_size=[7, 7],
depth_multiplier=1,
stride=1,
padding="VALID",
activation_fn=None),
op(slim.conv2d,
stride=1,
kernel_size=[1, 1],
num_outputs=FLAGS.embedding_size,
weights_regularizer=slim.l2_regularizer(4e-4),
activation_fn=final_activation),
],
)
net, end_points = mobilenet_v2.mobilenet(features,
is_training=is_training,
num_classes=num_classes,
base_only=True,
conv_defs=MFACENET_DEF)
embedding = end_points['embedding']
embedding = tf.squeeze(embedding, [1, 2])
embedding_unnorm = embedding
embedding = tf.nn.l2_normalize(embedding, axis=1)
if FLAGS.final_activation == 'step':
end_points['embedding'] = embedding_unnorm
else:
end_points['embedding'] = embedding
logits = arcface_logits(embedding, num_classes)
end_points['Logits'] = logits
return logits, end_points