def train_job(
images: tp.Numpy.Placeholder((BATCH_SIZE, 1, 28, 28), dtype=flow.float),
labels: tp.Numpy.Placeholder((BATCH_SIZE, ), dtype=flow.int32),
) -> tp.Numpy:
with flow.scope.placement("cpu", "0:0"):
initializer = flow.truncated_normal(0.1)
reshape = flow.reshape(images, [images.shape[0], -1])
hidden = flow.layers.dense(
reshape,
512,
activation=flow.nn.relu,
kernel_initializer=initializer,
name="dense1",
)
dense2 = flow.layers.dense(hidden,
10,
kernel_initializer=initializer,
name="dense2")
dense3 = flow.layers.dense(dense2,
10,
kernel_initializer=initializer,
name="dense3")
loss = flow.nn.sparse_softmax_cross_entropy_with_logits(labels, dense3)
lr_scheduler = flow.optimizer.PiecewiseConstantScheduler([], [0.1])
flow.optimizer.SGD(lr_scheduler, momentum=0).minimize(loss)
return loss
def lenet(data, train=False):
initializer = flow.truncated_normal(0.1)
conv1 = flow.layers.conv2d(
data,
32,
5,
padding="SAME",
activation=flow.nn.relu,
name="conv1",
kernel_initializer=initializer,
)
pool1 = flow.nn.max_pool2d(conv1, ksize=2, strides=2, padding="SAME", name="pool1")
conv2 = flow.layers.conv2d(
pool1,
64,
5,
padding="SAME",
activation=flow.nn.relu,
name="conv2",
kernel_initializer=initializer,
)
pool2 = flow.nn.max_pool2d(conv2, ksize=2, strides=2, padding="SAME", name="pool2")
reshape = flow.reshape(pool2, [pool2.shape[0], -1])
hidden = flow.layers.dense(
reshape,
512,
activation=flow.nn.relu,
kernel_initializer=initializer,
name="dense1",
)
if train:
hidden = flow.nn.dropout(hidden, rate=0.5, name="dropout")
return flow.layers.dense(hidden, 10, kernel_initializer=initializer, name="dense2")
def LeNet(data, train=False):
initializer = flow.truncated_normal(0.1)
conv_1 = flow.layers.conv2d(data,
6,
5,
padding="SAME",
activation=flow.nn.relu,
name="conv_1",
kernel_initializer=initializer)
pool_1 = flow.nn.max_pool2d(conv_1,
ksize=2,
strides=2,
padding="SAME",
name="pool_1",
data_format="NCHW")
conv_2 = flow.layers.conv2d(
pool_1,
16,
5,
padding="SAME",
activation=flow.nn.relu,
name="conv_2",
kernel_initializer=initializer,
)
pool_2 = flow.nn.max_pool2d(conv_2,
ksize=2,
strides=2,
padding="SAME",
name="pool_2",
data_format="NCHW")
reshape = flow.reshape(pool_2, [pool_2.shape[0], -1])
dense_1 = flow.layers.dense(
reshape,
120,
activation=flow.nn.relu,
name="dense_1",
kernel_initializer=initializer,
)
dense_2 = flow.layers.dense(dense_1,
84,
activation=flow.nn.relu,
kernel_initializer=initializer,
name="dense_2")
if train:
dense_2 = flow.nn.dropout(dense_2, rate=0.5, name="dropout")
dense_3 = flow.layers.dense(dense_2,
10,
kernel_initializer=initializer,
name="dense_3")
return dense_3
def mlp(data):
initializer = flow.truncated_normal(0.1)
reshape = flow.reshape(data, [data.shape[0], -1])
hidden = flow.layers.dense(
reshape,
512,
activation=flow.nn.relu,
kernel_initializer=initializer,
name="hidden",
)
return flow.layers.dense(hidden, 10, kernel_initializer=initializer, name="output")
def positionwiseFeedForward(x, d_model, d_ff, name="FFN_layer"):
"""
Build Positionwise FeedForward Layer
FFN(x) = max(0, W1*X+B1)*W2 + B2
:param x: The input Blob
:param d_model: The channels of input
:param d_ff: The channels of hidden
:param dropout: The dropout probability
:return:
"""
initializer_range = 0.2
init = flow.truncated_normal(initializer_range)
regularizer_range = 0.0005
reg = flow.regularizers.l2(regularizer_range)
x = flow.layers.dense(inputs=x,
units=d_ff,
kernel_initializer=init,
kernel_regularizer=reg,
bias_initializer=init,
bias_regularizer=reg,
name=name+"_W1")
x = flow.layers.dense(inputs=x,
units=d_model,
kernel_initializer=init,
kernel_regularizer=reg,
bias_initializer=init,
bias_regularizer=reg,
name=name+"_W2")
return x
# Test
# if __name__ == "__main__":
# @flow.global_function()
# def test_FFN(x: tp.Numpy.Placeholder(shape=(64, 50, 512), dtype=flow.float32)) -> tp.Numpy:
# out = positionwiseFeedForward(x, 512, 2048)
# return out
# check = flow.train.CheckPoint()
# check.init()
# x = np.random.randn(64, 50, 512).astype(np.float32)
# out = test_FFN(x)
# print(out.shape)
def lenet(data, train=False):
initializer = flow.truncated_normal(0.1)
conv1 = flow.layers.conv2d(
data,
32,
5,
padding="SAME",
activation=flow.nn.relu,
kernel_initializer=initializer,
name="conv1",
)
pool1 = flow.nn.max_pool2d(conv1,
ksize=2,
strides=2,
padding="SAME",
name="pool1")
conv2 = flow.layers.conv2d(
pool1,
64,
5,
padding="SAME",
activation=flow.nn.relu,
kernel_initializer=initializer,
name="conv2",
)
pool2 = flow.nn.max_pool2d(conv2,
ksize=2,
strides=2,
padding="SAME",
name="pool2")
reshape = flow.reshape(pool2, [pool2.shape[0], -1])
with flow.scope.placement("gpu", "0:0"):
hidden = flow.layers.dense(
reshape,
512,
activation=flow.nn.relu,
kernel_initializer=initializer,
name="hidden",
)
if train:
hidden = flow.nn.dropout(hidden, rate=0.5)
with flow.scope.placement("gpu", "0:1"):
output = flow.layers.dense(hidden,
10,
kernel_initializer=initializer,
name="outlayer")
return output
def mlp(data):
initializer = flow.truncated_normal(0.1)
reshape = flow.reshape(data, [data.shape[0], -1])
hidden = flow.layers.dense(
reshape,
512,
activation=flow.nn.relu,
kernel_initializer=initializer,
name="dense1",
)
return flow.layers.dense(
hidden,
10,
kernel_initializer=initializer,
# dense for column storage, use split(0)
model_distribute=flow.distribute.split(axis=0),
name="dense2",
)
def EmbeddingLayer(input_ids_blob,
vocab_size,
embedding_size=128,
initializer_range=0.02,
word_embedding_name="Embedding_Layer"):
"""
Embedding Layer
:param input_ids_blob:The input ID Blob
:param vocab_size: The input Vocab size
:param embedding_size: The embedding Size
:param initializer_range: The range of Initializer, Use flow.truncated_normal
:param word_embedding_name: The name of Embedding variable
:return: The output and the Embedding table.
"""
embedding_table = flow.get_variable(name=word_embedding_name+"_Embed",
shape=[vocab_size, embedding_size],
dtype=flow.float32,
initializer=flow.truncated_normal(initializer_range))
output = flow.gather(params=embedding_table, indices=input_ids_blob, axis=0)
return output
def Lenet(data):
initializer = flow.truncated_normal(0.1)
conv1 = flow.layers.conv2d(
data,
32,
5,
padding="SAME",
activation=flow.nn.relu,
name="conv1",
kernel_initializer=initializer,
use_bias=False,
)
pool1 = flow.nn.max_pool2d(
conv1, ksize=2, strides=2, padding="VALID", name="pool1", data_format="NCHW"
)
conv2 = flow.layers.conv2d(
pool1,
64,
5,
padding="SAME",
activation=flow.nn.relu,
name="conv2",
kernel_initializer=initializer,
use_bias=False
)
pool2 = flow.nn.max_pool2d(
conv2, ksize=2, strides=2, padding="VALID", name="pool2", data_format="NCHW"
)
reshape = flow.reshape(pool2, [pool2.shape[0], -1])
hidden = flow.layers.dense(
reshape,
512,
activation=flow.nn.relu,
kernel_initializer=initializer,
name="dense1",
use_bias=False
)
return flow.layers.dense(hidden, 10, kernel_initializer=initializer, name="dense2",use_bias=False)
def train_job(
images: tp.Numpy.Placeholder((BATCH_SIZE, 1, 28, 28), dtype=flow.float),
labels: tp.Numpy.Placeholder((BATCH_SIZE, ), dtype=flow.int32),
) -> tp.Callback[tp.Numpy]:
# mlp
initializer = flow.truncated_normal(0.1)
reshape = flow.reshape(images, [images.shape[0], -1])
hidden = flow.layers.dense(
reshape,
512,
activation=flow.nn.relu,
kernel_initializer=initializer,
name="hidden",
)
logits = flow.layers.dense(hidden,
10,
kernel_initializer=initializer,
name="output")
loss = flow.nn.sparse_softmax_cross_entropy_with_logits(
labels, logits, name="softmax_loss")
lr_scheduler = flow.optimizer.PiecewiseConstantScheduler([], [0.1])
flow.optimizer.SGD(lr_scheduler, momentum=0).minimize(loss)
return loss
def CreateInitializer(std):
return flow.truncated_normal(std)
def inceptionv3(images,
trainable=True,
need_transpose=False,
channel_last=False):
if need_transpose:
images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2])
if channel_last:
# if channel_last=True, then change mode from 'nchw' to 'nhwc'
images = flow.transpose(images, name="transpose", perm=[0, 2, 3, 1])
with flow.scope.namespace("InceptionV3"):
# conv0: 299 x 299 x 3
conv0 = conv2d_layer("conv0",
images,
filters=32,
kernel_size=3,
strides=2,
padding="VALID")
conv1 = conv2d_layer("conv1",
conv0,
filters=32,
kernel_size=3,
strides=1,
padding="VALID")
conv2 = conv2d_layer("conv2",
conv1,
filters=64,
kernel_size=3,
strides=1,
padding="SAME")
pool1 = flow.nn.max_pool2d(conv2,
ksize=3,
strides=2,
padding="VALID",
data_format="NCHW",
name="pool1")
conv3 = conv2d_layer("conv3",
pool1,
filters=80,
kernel_size=1,
strides=1,
padding="VALID")
conv4 = conv2d_layer("conv4",
conv3,
filters=192,
kernel_size=3,
strides=1,
padding="VALID")
pool2 = flow.nn.max_pool2d(conv4,
ksize=3,
strides=2,
padding="VALID",
data_format="NCHW",
name="pool2")
# mixed_0 ~ mixed_2
mixed_0 = InceptionA(pool2, 0)
mixed_1 = InceptionA(mixed_0, 1)
mixed_2 = InceptionA(mixed_1, 2)
# mixed_3
mixed_3 = InceptionB(mixed_2, 3)
# mixed_4 ~ mixed_7
mixed_4 = InceptionC(mixed_3, 4, 128)
mixed_5 = InceptionC(mixed_4, 5, 160)
mixed_6 = InceptionC(mixed_5, 6, 160)
mixed_7 = InceptionC(mixed_6, 7, 192)
# mixed_8
mixed_8 = InceptionD(mixed_7, 8)
# mixed_9 ~ mixed_10
mixed_9 = InceptionE(mixed_8, 9, 'avg')
mixed_10 = InceptionE(mixed_9, 10, 'max')
pool3 = flow.nn.avg_pool2d(mixed_10,
ksize=8,
strides=1,
padding="VALID",
data_format="NCHW",
name="pool3")
# TODO: Need to transpose weight when converting model from TF to OF if
# you want to use layers.dense interface.
fc1 = flow.layers.dense(
inputs=flow.reshape(pool3, [pool3.shape[0], -1]),
units=1000,
activation=None,
use_bias=True,
kernel_initializer=flow.truncated_normal(0.816496580927726),
bias_initializer=flow.constant_initializer(),
trainable=trainable,
name="fc1",
)
return fc1
def inceptionv3(images, labels, trainable=True):
conv0 = _conv2d_layer("conv0",
images,
filters=32,
kernel_size=3,
strides=2,
padding="VALID")
conv1 = _conv2d_layer("conv1",
conv0,
filters=32,
kernel_size=3,
strides=1,
padding="VALID")
conv2 = _conv2d_layer("conv2",
conv1,
filters=64,
kernel_size=3,
strides=1,
padding="SAME")
pool1 = flow.nn.max_pool2d(conv2,
ksize=3,
strides=2,
padding="VALID",
data_format="NCHW",
name="pool1")
conv3 = _conv2d_layer("conv3",
pool1,
filters=80,
kernel_size=1,
strides=1,
padding="VALID")
conv4 = _conv2d_layer("conv4",
conv3,
filters=192,
kernel_size=3,
strides=1,
padding="VALID")
pool2 = flow.nn.max_pool2d(conv4,
ksize=3,
strides=2,
padding="VALID",
data_format="NCHW",
name="pool2")
# mixed_0 ~ mixed_2
mixed_0 = InceptionA(pool2, 0)
mixed_1 = InceptionA(mixed_0, 1)
mixed_2 = InceptionA(mixed_1, 2)
# mixed_3
mixed_3 = InceptionB(mixed_2, 3)
# mixed_4 ~ mixed_7
mixed_4 = InceptionC(mixed_3, 4, 128)
mixed_5 = InceptionC(mixed_4, 5, 160)
mixed_6 = InceptionC(mixed_5, 6, 160)
mixed_7 = InceptionC(mixed_6, 7, 192)
# mixed_8
mixed_8 = InceptionD(mixed_7, 8)
# mixed_9 ~ mixed_10
mixed_9 = InceptionE(mixed_8, 9)
mixed_10 = InceptionE(mixed_9, 10)
# pool3
pool3 = flow.nn.avg_pool2d(mixed_10,
ksize=8,
strides=1,
padding="VALID",
data_format="NCHW",
name="pool3")
with flow.scope.namespace("logits"):
pool3 = flow.reshape(pool3, [pool3.shape[0], -1])
# TODO: Need to transpose weight when converting model from TF to OF if
# you want to use layers.dense interface.
# fc1 = flow.layers.dense(
# pool3,
# 1001,
# activation=None,
# use_bias=False,
# kernel_initializer=flow.truncated_normal(0.816496580927726),
# bias_initializer=flow.constant_initializer(),
# name="fc1",
# )
weight = flow.get_variable(
"fc1-weight",
shape=(pool3.shape[1], 1001),
dtype=flow.float,
initializer=flow.truncated_normal(0.816496580927726),
model_name="weight",
)
bias = flow.get_variable(
"fc1-bias",
shape=(1001, ),
dtype=flow.float,
initializer=flow.constant_initializer(),
model_name="bias",
)
fc1 = flow.matmul(pool3, weight)
fc1 = flow.nn.bias_add(fc1, bias)
return fc1