def convolution1d(inputs,
kernel_size,
filters,
strides=1,
padding='VALID',
batchnorm=False,
activation='linear',
initializer=G.BACKEND_DEFAULT_INITIALIZER(),
bias_initializer=tf.constant_initializer(0.00),
regularizer=G.BACKEND_DEFAULT_REGULARIZER(0.00),
trainable=True,
use_bias=True,
is_training=True):
"""
Convolutional 1D Layer.
:param inputs: A 3-D Tensor: [batch, feat_num, ifm]
:param kernel_size: an integer indicating the filter kernel size.
:param filters: number of filters in the output.
:param strides: an integer indicating the convolution strides.
:param padding: Whether to use the 'SAME' padding or 'VALID' padding. case insensitive.
:param batchnorm: Whether to use batchnorm.
:param activation: activation function.
:param initializer: Weight initializer. See backend for default settings.
:param bias_initializer: Bias initializer.
:param regularizer: Regularizer for weights. See backend for default settings.
:param trainable: Whether this layer is trainable.
:param use_bias: Use bias or not.
:return: A 3-D tensor: [batch, feat_num_, ofm].
"""
assert G.BACKEND == "tensorflow"
conv = tf.layers.conv1d(inputs=inputs,
filters=filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
kernel_initializer=initializer,
bias_initializer=bias_initializer,
kernel_regularizer=regularizer,
trainable=trainable,
name='conv',
use_bias=use_bias)
if batchnorm:
conv = tf.layers.batch_normalization(inputs=conv,
epsilon=G.BN_EPSILON,
momentum=G.BN_MOMENTUM,
name='bn',
trainable=trainable,
fused=False,
reuse=tf.AUTO_REUSE,
training=is_training)
# will apply activation after
conv = apply_activation(conv, activation)
return conv
def dense(inputs,
units,
activation='relu',
batchnorm=False,
initializer=G.BACKEND_DEFAULT_FC_INITIALIZER(),
bias_initializer=tf.constant_initializer(0.00),
regularizer=None,
trainable=True,
use_bias=True,
is_training=tf.convert_to_tensor(True)):
"""
Dense Connection Layer. (Fully connected Layer)
:param inputs: A 2-D Tensor: [batch, ifm]
:param units: output units number (ofm)
:param activation: activation function to apply.
:param batchnorm: Whether use batch normalization or not.
:param initializer: Initialization for weight parameter. See backend for details.
:param bias_initializer: Initialization for bias parameter. See backend for details.
:param regularizer: regularization function.
:param trainable: Whether this layer is trainable or not.
:param use_bias: Whether to use bias or not.
:param is_training: Whether is training.
:return: A 2-D Tensor: [batch, ofm]
"""
assert G.BACKEND == "tensorflow"
if regularizer is None:
regularizer = G.BACKEND_DEFAULT_REGULARIZER(G.DEFAULT_REG)
else:
regularizer = regularizer
fc = tf.layers.dense(
inputs=inputs,
units=units,
kernel_initializer=initializer,
bias_initializer=bias_initializer,
kernel_regularizer=regularizer,
use_bias=use_bias,
trainable=trainable,
)
if batchnorm:
fc = batch_normalization(
fc,
activation=activation,
trainable=trainable,
is_training=is_training,
)
else:
activation_fn = get_activation_fn(activation)
fc = activation_fn(fc)
return fc
def main(args):
G.set_conv_triplet(args.triplet)
if args.pretrained_path is None:
meta_graph = MetaGraph(nodes=args.nodes,
p=args.prob,
model_root=args.model_root,
protocol=args.protocol,
solvertxt=args.solver)
else:
with open(args.pretrained_path, 'rb') as fp:
meta_graph = pickle.load(fp)
meta_graph.auto_shrink(max_steps_to_action=10, keep_fraction=0.01, drop_k_each_iteration=1,
scale=args.scale,
replicate=args.replicate,
pool=args.pool
)
def main(args):
G.config_bn_params(1e-3, 0.999)
with open(args.graph_path, 'rb') as fp:
meta_graph = pickle.load(fp)
meta_graph.set_depth(5)
model_root = args.graph_path
proto_writer = meta_graph.create_eval_graph(
replicate=args.replicate,
scale=args.scale,
channel_scale=args.channel_scale,
task='imagenet-%d' % args.image_size,
model_root=model_root,
pool=args.pool_size,
use_bias=False,
use_residual=args.use_residual,
bottleneck_factor=args.bottleneck_factor)
proto_writer.set_global_regularization(args.regularizer)
tf.logging.info("Use %s activation" % args.activation)
proto_writer.set_activation(args.activation)
prototxt = "temp/demo-eval.prototxt"
with open(prototxt, 'w') as fp:
proto_writer.dump(fp)
solvertxt = args.solver
estimator = ImageNetEstimator(prototxt,
solvertxt,
image_size=args.image_size,
verbose=2)
estimator.trainval(train_batch_size=args.batch_size,
val_batch_size=100,
epochs=350,
ngpus=args.ngpus)
def main(args):
# Configure batch normalization hyperparameters.
G.config_bn_params(1e-3, 0.999)
# Uncomment this line if you are using sepconvV1 with linear bottleneck, and use "swish" activation.
# G.set_default_sepconvV1_activation("swish")
# Uncomment this line to reconfigure residual regularization, if you are using other than 4e-5 in the prototxt files.
# G.set_default_reg(5e-4)
# Default initialization method.
G.set_initializer("default")
# Configure GPU option. It is good to occupy the whole GPU during ImageNet training.
G.use_all_gpu()
estimator = ImageNetEstimator(image_size=args.image_size,
prototxt=args.proto,
solvertxt=args.solver)
top1_acc, top5_acc, mac = estimator.trainval(
train_batch_size=args.batch_size,
ngpus=args.ngpus,
save_model=args.save_model,
epochs=args.epochs,
max_steps=args.max_steps,
tfrecord_path=args.tfrecord_path)
if args.log_dir is not None:
if not os.path.exists(args.log_dir):
os.makedirs(args.log_dir)
proto_name = args.proto.split("/")[-1]
log_name = os.path.join(args.log_dir, proto_name)
with open(log_name, 'w') as fp:
fp.write("name: %s\n" % args.proto)
fp.write("MillonMACs: %s\n" % mac)
fp.write("Top1Acc: %s\n" % top1_acc)
fp.write("Top5Acc: %s\n" % top5_acc)
pass
def separable_conv2d_v2(inputs,
kernel_size,
filters,
strides=1,
padding='SAME',
batchnorm=False,
activation='linear',
initializer=G.BACKEND_DEFAULT_CONV_INITIALIZER(),
bias_initializer=tf.constant_initializer(0.00),
regularizer=G.BACKEND_DEFAULT_REGULARIZER(0.00),
trainable=True,
regularize_depthwise=False,
use_bias=True,
is_training=tf.convert_to_tensor(True),
mode=G.EXEC_CONV_MODE):
"""
Depthwise Separable Convolution Layer.
:param inputs: A 4-D Tensor: [batch, height, width, ifm]
:param kernel_size: an integer indicating the filter kernel size.
:param filters: number of filters in the output.
:param strides: an integer indicating the convolution strides.
:param padding: Whether to use the 'SAME' padding or 'VALID' padding. case insensitive.
:param batchnorm: Whether to use batchnorm.
:param depthwise_activation: activation function for depthwise convolution.
:param pointwise_activation: activation function for pointwise convolution.
:param initializer: Weight initializer. See backend for default settings.
:param bias_initializer: Bias initializer.
:param regularizer: Regularizer for weights. See backend for default settings.
:param trainable: Whether this layer is trainable.
:param use_bias: Use bias or not.
:return: A 4-D tensor: [batch, height_, width_, ofm].
"""
if filters == -1:
if G.data_format == 'channels_last':
filters = int(inputs.get_shape()[-1])
else:
filters = int(inputs.get_shape()[1])
assert G.BACKEND == "tensorflow"
if regularize_depthwise:
depthwise_regularizer = regularizer
else:
depthwise_regularizer = None
if mode == "conv-bn-relu":
with tf.variable_scope("depthwise") as scope:
conv = tf.layers.separable_conv2d(
inputs=inputs,
filters=filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
activation=None,
depthwise_regularizer=depthwise_regularizer,
pointwise_regularizer=regularizer,
depthwise_initializer=initializer,
pointwise_initializer=initializer,
use_bias=use_bias,
data_format=G.data_format)
if batchnorm:
conv = batch_normalization(conv,
trainable=trainable,
is_training=is_training,
activation=activation)
elif mode == 'relu-conv-bn':
raise DeprecationWarning(
"relu-conv-bn is deprecated. Please use the conv-bn-relu triplet.")
else:
raise NotImplementedError
return conv
def separable_conv2d_v1(inputs,
kernel_size,
filters,
strides=1,
padding='SAME',
batchnorm=False,
activation='linear',
initializer=G.BACKEND_DEFAULT_CONV_INITIALIZER(),
bias_initializer=tf.constant_initializer(0.00),
regularizer=G.BACKEND_DEFAULT_REGULARIZER(0.00),
trainable=True,
regularize_depthwise=False,
use_bias=True,
is_training=tf.convert_to_tensor(True),
mode=G.EXEC_CONV_MODE):
"""
Depthwise Separable Convolution Layer.
:param inputs: A 4-D Tensor: [batch, height, width, ifm]
:param kernel_size: an integer indicating the filter kernel size.
:param filters: number of filters in the output.
:param strides: an integer indicating the convolution strides.
:param padding: Whether to use the 'SAME' padding or 'VALID' padding. case insensitive.
:param batchnorm: Whether to use batchnorm.
:param depthwise_activation: activation function for depthwise convolution.
:param pointwise_activation: activation function for pointwise convolution.
:param initializer: Weight initializer. See backend for default settings.
:param bias_initializer: Bias initializer.
:param regularizer: Regularizer for weights. See backend for default settings.
:param trainable: Whether this layer is trainable.
:param use_bias: Use bias or not.
:return: A 4-D tensor: [batch, height_, width_, ofm].
"""
if filters == -1:
filters = int(inputs.get_shape()[-1])
assert G.BACKEND == "tensorflow"
if activation == "linear":
depthwise_activation = G.SEPCONV_V1_DEFAULT_ACTIVATION
else:
depthwise_activation = activation
if regularize_depthwise:
depthwise_regularizer = regularizer
else:
depthwise_regularizer = None
with tf.variable_scope("depthwise", tf.AUTO_REUSE) as scope:
# Consistent with separable_conv_v2:
with tf.variable_scope("separable_conv2d", tf.AUTO_REUSE) as scope:
conv_depthwise = depthwise_conv2d(
inputs=inputs,
kernel_size=kernel_size,
strides=strides,
padding=padding,
activation=depthwise_activation,
depth_multiplier=1,
initializer=initializer,
bias_initializer=bias_initializer,
regularizer=depthwise_regularizer,
batchnorm=batchnorm,
trainable=trainable,
use_bias=use_bias,
is_training=is_training,
mode=mode)
conv_pointwise = pointwise_conv2d(
conv_depthwise,
filters=filters,
strides=1,
activation=activation,
initializer=initializer,
bias_initializer=bias_initializer,
regularizer=regularizer,
padding='SAME',
trainable=trainable,
use_bias=use_bias,
is_training=is_training,
batchnorm=batchnorm,
mode=mode)
return conv_pointwise
def pointwise_conv2d(inputs,
filters,
strides=1,
padding='SAME',
batchnorm=False,
activation='linear',
initializer=G.BACKEND_DEFAULT_CONV_INITIALIZER(),
bias_initializer=tf.constant_initializer(0.00),
regularizer=G.BACKEND_DEFAULT_REGULARIZER(0.00),
trainable=True,
use_bias=True,
is_training=tf.convert_to_tensor(True),
mode=G.EXEC_CONV_MODE):
assert G.BACKEND == "tensorflow"
input_dim = inputs.get_shape()[-1]
output_dim = filters
weights = tf.get_variable(shape=[1, 1, input_dim, output_dim],
name="pointwise_kernel",
initializer=initializer,
regularizer=regularizer,
trainable=trainable)
activation_fn = get_activation_fn(activation)
if G.data_format == "channels_last":
strides_ = (1, strides, strides, 1)
else:
strides_ = (1, 1, strides, strides)
if mode == 'conv-bn-relu':
if use_bias:
bias = tf.get_variable(shape=[output_dim],
initializer=bias_initializer,
name='bias',
trainable=trainable)
conv_pointwise = tf.nn.bias_add(tf.nn.conv2d(input=inputs,
filter=weights,
strides=strides_,
padding=padding),
bias,
name="pointwise_conv")
else:
conv_pointwise = tf.nn.conv2d(input=inputs,
filter=weights,
strides=strides_,
padding=padding,
name='pointwise_conv')
if batchnorm:
with tf.variable_scope("pointwise_bn", tf.AUTO_REUSE):
conv_pointwise = batch_normalization(
conv_pointwise,
activation=activation,
trainable=trainable,
is_training=is_training,
)
else:
conv_pointwise = activation_fn(conv_pointwise)
elif mode == 'relu-conv-bn':
raise DeprecationWarning(
"relu-conv-bn is deprecated. Please use the conv-bn-relu triplet.")
else:
raise NotImplementedError
return conv_pointwise
def depthwise_conv2d(inputs,
kernel_size,
strides=1,
padding='SAME',
batchnorm=False,
depth_multiplier=1,
activation='linear',
initializer=G.BACKEND_DEFAULT_CONV_INITIALIZER(),
bias_initializer=tf.constant_initializer(0.00),
regularizer=G.BACKEND_DEFAULT_REGULARIZER(0.00),
trainable=True,
use_bias=True,
is_training=tf.convert_to_tensor(True),
mode=G.EXEC_CONV_MODE):
"""
Depthwise Convolution
:param inputs: A 4-D Tensor: [batch, height, width, ifm]
:param kernel_size: Kernel size for depthwise Convolution.
:param strides: Strides for depthwise convolution.
:param padding: Padding for depthwise convolution.
:param batchnorm: Whether use batchnorm or not.
:param depth_multiplier: Multiplier for depthwise convolution.
:param activation: Activation function for depthwise Convolution/
:param initializer: Intializer for depthwise kernel.
:param bias_initializer: Initializer for bias kernel.
:param regularizer: Regularization for depthwise kernel.
:param trainable: Whether this layer is trainable or not.
:param use_bias: Whether to use bias or not.
:return:
"""
assert G.BACKEND == "tensorflow"
if G.data_format == "channels_last":
input_dim = inputs.get_shape()[-1]
strides_ = (1, strides, strides, 1)
else:
input_dim = inputs.get_shape()[1]
strides_ = (1, 1, strides, strides)
output_dim = input_dim * depth_multiplier
if G.data_format == 'channels_first':
weights = tf.get_variable(
shape=[kernel_size, kernel_size, input_dim, depth_multiplier],
name="depthwise_kernel",
initializer=initializer,
regularizer=regularizer,
trainable=trainable)
else:
weights = tf.get_variable(
shape=[kernel_size, kernel_size, input_dim, depth_multiplier],
name="depthwise_kernel",
initializer=initializer,
regularizer=regularizer,
trainable=trainable)
activation_fn = get_activation_fn(activation)
if mode == 'conv-bn-relu':
if use_bias:
bias = tf.get_variable(shape=[output_dim],
initializer=bias_initializer,
name='bias',
trainable=trainable)
conv_depthwise = tf.nn.bias_add(tf.nn.depthwise_conv2d(
input=inputs,
filter=weights,
strides=strides_,
padding=padding,
data_format=convert_to_legacy(G.data_format)),
bias,
name="depthwise_conv")
else:
conv_depthwise = tf.nn.depthwise_conv2d(
input=inputs,
filter=weights,
strides=strides_,
padding=padding,
name="depthwise_conv",
data_format=convert_to_legacy(G.data_format))
if batchnorm:
with tf.variable_scope("depthwise_bn", tf.AUTO_REUSE):
conv_depthwise = batch_normalization(
conv_depthwise,
activation=activation,
trainable=trainable,
is_training=is_training,
)
else:
conv_depthwise = activation_fn(conv_depthwise)
elif mode == 'relu-conv-bn':
raise DeprecationWarning(
"relu-conv-bn is deprecated. Please use the conv-bn-relu triplet.")
else:
raise NotImplementedError
return conv_depthwise
def convolution2d(inputs,
kernel_size,
filters,
strides=1,
padding='VALID',
batchnorm=False,
activation='linear',
initializer=G.BACKEND_DEFAULT_CONV_INITIALIZER(),
bias_initializer=tf.constant_initializer(0.00),
regularizer=None,
trainable=True,
use_bias=True,
is_training=tf.convert_to_tensor(True),
mode=None):
"""
Convolutional 2D Layer.
:param inputs: A 4-D Tensor: [batch, height, width, ifm]
:param kernel_size: an integer indicating the filter kernel size.
:param filters: number of filters in the output.
:param strides: an integer indicating the convolution strides.
:param padding: Whether to use the 'SAME' padding or 'VALID' padding. case insensitive.
:param batchnorm: Whether to use batchnorm.
:param activation: activation function.
:param initializer: Weight initializer. See backend for default settings.
:param bias_initializer: Bias initializer.
:param regularizer: Regularizer for weights. See backend for default settings.
:param trainable: Whether this layer is trainable.
:param use_bias: Use bias or not.
:return: A 4-D tensor: [batch, height_, width_, ofm].
"""
assert G.BACKEND == "tensorflow"
if regularizer is None:
regularizer = G.BACKEND_DEFAULT_REGULARIZER(G.DEFAULT_REG)
else:
regularizer = regularizer
if filters == -1:
filters = int(inputs.get_shape()[-1])
activation_fn = get_activation_fn(activation)
if mode == 'conv-bn-relu':
conv = tf.layers.conv2d(inputs=inputs,
filters=filters,
kernel_size=kernel_size,
strides=strides,
padding=padding,
use_bias=use_bias,
trainable=trainable,
activation=None,
kernel_initializer=initializer,
bias_initializer=bias_initializer,
kernel_regularizer=regularizer,
data_format=G.data_format,
name='conv')
if batchnorm:
conv = batch_normalization(
conv,
activation=activation,
trainable=trainable,
is_training=is_training,
)
else:
conv = activation_fn(conv)
elif mode == "relu-conv-bn":
raise DeprecationWarning(
"relu-conv-bn is deprecated. Please use the conv-bn-relu triplet.")
else:
raise NotImplementedError
return conv