def _model(dense_fields, wide_sparse_fields, deep_sparse_fields):
# wide_embedding = _embedding('wide_embedding', wide_sparse_fields, 1, FLAGS.wide_vocab_size)
wide_embedding = _hybrid_embedding('wide_embedding', wide_sparse_fields, 1, FLAGS.wide_vocab_size,
FLAGS.hf_wide_vocab_size)
wide_scores = flow.math.reduce_sum(wide_embedding, axis=[1], keepdims=True)
wide_scores = flow.parallel_cast(wide_scores, distribute=flow.distribute.split(0),
gradient_distribute=flow.distribute.broadcast())
# deep_embedding = _embedding('deep_embedding', deep_sparse_fields, FLAGS.deep_embedding_vec_size,
# FLAGS.deep_vocab_size, split_axis=1)
deep_embedding = _hybrid_embedding('deep_embedding', deep_sparse_fields, FLAGS.deep_embedding_vec_size,
FLAGS.deep_vocab_size, FLAGS.hf_deep_vocab_size)
deep_features = flow.concat([deep_embedding, dense_fields], axis=1)
for idx, units in enumerate(DEEP_HIDDEN_UNITS):
deep_features = flow.layers.dense(
deep_features,
units=units,
kernel_initializer=flow.glorot_uniform_initializer(),
bias_initializer=flow.constant_initializer(0.0),
activation=flow.math.relu,
name='fc' + str(idx + 1)
)
deep_features = flow.nn.dropout(deep_features, rate=FLAGS.deep_dropout_rate)
deep_scores = flow.layers.dense(
deep_features,
units=1,
kernel_initializer=flow.glorot_uniform_initializer(),
bias_initializer=flow.constant_initializer(0.0),
name='fc' + str(len(DEEP_HIDDEN_UNITS) + 1)
)
scores = wide_scores + deep_scores
return scores
def ConcatJob():
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"x",
shape=x_shape,
dtype=type_name_to_flow_type[dtype],
initializer=flow.random_uniform_initializer(minval=-10, maxval=10),
trainable=True,
)
y = flow.get_variable(
"y",
shape=y_shape,
dtype=type_name_to_flow_type[dtype],
initializer=flow.random_uniform_initializer(minval=-10, maxval=10),
trainable=True,
)
x = flow.cast_to_current_logical_view(x)
y = flow.cast_to_current_logical_view(y)
loss = flow.concat([x, y], axis)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0
).minimize(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(y, test_global_storage.Setter("y"))
flow.watch_diff(y, test_global_storage.Setter("y_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return loss
def hybrid_concat_job(
input_0_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float),
input_1_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float),
):
var = flow.get_variable(
"var",
shape=static_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(),
trainable=True,
)
constant = flow.constant(1.0, dtype=flow.float, shape=rand_sub_shape)
inputs = [
flow.cast_to_current_logical_view(input)
for input in [var, input_0_def, input_1_def, constant]
]
concated = flow.concat(inputs, axis=axis, max_dim_size=max_dim_size)
if verbose:
print("concated static shape:", concated.shape)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0
).minimize(concated)
flow.watch_diff(var, compare_var_diff)
if max_dim_size is None:
test_case.assertTrue(
concated.shape[axis] == static_shape[axis] * 3 + rand_sub_shape[axis]
)
else:
test_case.assertTrue(concated.shape[axis] == max_dim_size)
return (var, concated)
def _model(dense_fields, wide_sparse_fields, deep_sparse_fields):
wide_sparse_fields = flow.parallel_cast(
wide_sparse_fields, distribute=flow.distribute.broadcast())
wide_embedding_table = flow.get_variable(
name='wide_embedding',
shape=(FLAGS.wide_vocab_size, 1),
initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05),
distribute=flow.distribute.split(0),
)
wide_embedding = flow.gather(params=wide_embedding_table,
indices=wide_sparse_fields)
wide_embedding = flow.reshape(wide_embedding,
shape=(-1, wide_embedding.shape[-1] *
wide_embedding.shape[-2]))
wide_scores = flow.math.reduce_sum(wide_embedding, axis=[1], keepdims=True)
wide_scores = flow.parallel_cast(
wide_scores,
distribute=flow.distribute.split(0),
gradient_distribute=flow.distribute.broadcast())
deep_sparse_fields = flow.parallel_cast(
deep_sparse_fields, distribute=flow.distribute.broadcast())
deep_embedding_table = flow.get_variable(
name='deep_embedding',
shape=(FLAGS.deep_vocab_size, FLAGS.deep_embedding_vec_size),
initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05),
distribute=flow.distribute.split(1),
)
deep_embedding = flow.gather(params=deep_embedding_table,
indices=deep_sparse_fields)
deep_embedding = flow.parallel_cast(
deep_embedding,
distribute=flow.distribute.split(0),
gradient_distribute=flow.distribute.split(2))
deep_embedding = flow.reshape(deep_embedding,
shape=(-1, deep_embedding.shape[-1] *
deep_embedding.shape[-2]))
deep_features = flow.concat([deep_embedding, dense_fields], axis=1)
for idx, units in enumerate(DEEP_HIDDEN_UNITS):
deep_features = flow.layers.dense(
deep_features,
units=units,
kernel_initializer=flow.glorot_uniform_initializer(),
bias_initializer=flow.constant_initializer(0.0),
activation=flow.math.relu,
name='fc' + str(idx + 1))
deep_features = flow.nn.dropout(deep_features,
rate=FLAGS.deep_dropout_rate)
deep_scores = flow.layers.dense(
deep_features,
units=1,
kernel_initializer=flow.glorot_uniform_initializer(),
bias_initializer=flow.constant_initializer(0.0),
name='fc' + str(len(DEEP_HIDDEN_UNITS) + 1))
scores = wide_scores + deep_scores
return scores
def InceptionD(in_blob, index):
with flow.scope.namespace("mixed_{}".format(index)):
with flow.scope.namespace("branch3x3"):
branch3x3_1 = _conv2d_layer(
"conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME"
)
branch3x3_2 = _conv2d_layer(
"conv1",
branch3x3_1,
filters=320,
kernel_size=3,
strides=2,
padding="VALID",
)
with flow.scope.namespace("branch7x7x3"):
branch7x7x3_1 = _conv2d_layer(
"conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME"
)
branch7x7x3_2 = _conv2d_layer(
"conv1",
branch7x7x3_1,
filters=192,
kernel_size=[1, 7],
strides=1,
padding="SAME",
)
branch7x7x3_3 = _conv2d_layer(
"conv2",
branch7x7x3_2,
filters=192,
kernel_size=[7, 1],
strides=1,
padding="SAME",
)
branch7x7x3_4 = _conv2d_layer(
"conv3",
branch7x7x3_3,
filters=192,
kernel_size=3,
strides=2,
padding="VALID",
)
with flow.scope.namespace("branch_pool"):
branch_pool = flow.nn.max_pool2d(
in_blob,
ksize=3,
strides=2,
padding="VALID",
data_format="NCHW",
name="pool",
)
inceptionD_bn = []
inceptionD_bn.append(branch3x3_2)
inceptionD_bn.append(branch7x7x3_4)
inceptionD_bn.append(branch_pool)
mixed_concat = flow.concat(values=inceptionD_bn, axis=1, name="concat")
return mixed_concat
def InceptionB(in_blob, index):
with flow.scope.namespace("mixed_{}".format(index)):
with flow.scope.namespace("branch3x3"):
branch3x3 = conv2d_layer("conv0",
in_blob,
filters=384,
kernel_size=3,
strides=2,
padding="VALID")
with flow.scope.namespace("branch3x3dbl"):
branch3x3dbl_1 = conv2d_layer("conv0",
in_blob,
filters=64,
kernel_size=1,
strides=1,
padding="SAME")
branch3x3dbl_2 = conv2d_layer(
"conv1",
branch3x3dbl_1,
filters=96,
kernel_size=3,
strides=1,
padding="SAME",
)
branch3x3dbl_3 = conv2d_layer(
"conv2",
branch3x3dbl_2,
filters=96,
kernel_size=3,
strides=2,
padding="VALID",
)
with flow.scope.namespace("branch_pool"):
branch_pool = flow.nn.max_pool2d(
in_blob,
ksize=3,
strides=2,
padding="VALID",
data_format="NCHW",
name="pool0",
)
inceptionB_bn = []
inceptionB_bn.append(branch3x3)
inceptionB_bn.append(branch3x3dbl_3)
inceptionB_bn.append(branch_pool)
mixed_concat = flow.concat(values=inceptionB_bn, axis=1, name="concat")
return mixed_concat
def dynamic_concat_job(
input_0_def: oft.ListNumpy.Placeholder(
shape=input_static_shape, dtype=flow.float
),
input_1_def: oft.ListNumpy.Placeholder(
shape=input_static_shape, dtype=flow.float
),
):
var_0 = flow.get_variable(
"Var0",
shape=(1,),
dtype=flow.float,
initializer=flow.constant_initializer(value=1, dtype=flow.float),
trainable=True,
)
var_1 = flow.get_variable(
"Var1",
shape=(1,),
dtype=flow.float,
initializer=flow.constant_initializer(value=1, dtype=flow.float),
trainable=True,
)
var_0 = flow.cast_to_current_logical_view(var_0)
var_1 = flow.cast_to_current_logical_view(var_1)
input_0_def = flow.cast_to_current_logical_view(input_0_def)
input_1_def = flow.cast_to_current_logical_view(input_1_def)
if callable(watch_cb):
flow.watch(var_0, watch_cb)
flow.watch(var_1, watch_cb)
flow.watch(flow.identity(input_0_def), watch_cb)
flow.watch(flow.identity(input_1_def), watch_cb)
var_0 = var_0 * input_0_def
var_1 = var_1 * input_1_def
if callable(watch_cb):
flow.watch(var_0, watch_cb)
flow.watch(var_1, watch_cb)
result = flow.concat(
[var_0, var_1], axis=axis, max_dim_size=input_static_shape[axis]
)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0
).minimize(result)
flow.watch_diff(var_0, make_watch_diff_cb(0))
flow.watch_diff(var_1, make_watch_diff_cb(1))
return result
def static_concat_job(
input_0_def: oft.Numpy.Placeholder(shape=shape, dtype=flow.float),
input_1_def: oft.Numpy.Placeholder(shape=shape, dtype=flow.float),
):
var = flow.get_variable(
"var",
shape=shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(),
trainable=True,
)
concated = flow.concat([input_0_def, input_1_def, var], axis=axis)
test_case.assertTrue(not concated.is_dynamic)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0
).minimize(concated)
flow.watch_diff(var, compare_var_diff)
return (var, concated)
def InceptionE(in_blob, index, pooltype):
with flow.scope.namespace("mixed_{}".format(index)):
with flow.scope.namespace("branch1x1"):
branch1x1 = conv2d_layer("conv0",
in_blob,
filters=320,
kernel_size=1,
strides=1,
padding="SAME")
with flow.scope.namespace("branch3x3"):
branch3x3_1 = conv2d_layer("conv0",
in_blob,
filters=384,
kernel_size=1,
strides=1,
padding="SAME")
branch3x3_2 = conv2d_layer(
"conv1",
branch3x3_1,
filters=384,
kernel_size=[1, 3],
strides=1,
padding="SAME",
)
branch3x3_3 = conv2d_layer(
"conv2",
branch3x3_1,
filters=384,
kernel_size=[3, 1],
strides=[1, 1],
padding="SAME",
)
inceptionE_1_bn = []
inceptionE_1_bn.append(branch3x3_2)
inceptionE_1_bn.append(branch3x3_3)
concat_branch3x3 = flow.concat(values=inceptionE_1_bn,
axis=1,
name="concat")
with flow.scope.namespace("branch3x3dbl"):
branch3x3dbl_1 = conv2d_layer("conv0",
in_blob,
filters=448,
kernel_size=1,
strides=1,
padding="SAME")
branch3x3dbl_2 = conv2d_layer(
"conv1",
branch3x3dbl_1,
filters=384,
kernel_size=3,
strides=1,
padding="SAME",
)
branch3x3dbl_3 = conv2d_layer(
"conv2",
branch3x3dbl_2,
filters=384,
kernel_size=[1, 3],
strides=1,
padding="SAME",
)
branch3x3dbl_4 = conv2d_layer(
"conv3",
branch3x3dbl_2,
filters=384,
kernel_size=[3, 1],
strides=1,
padding="SAME",
)
inceptionE_2_bn = []
inceptionE_2_bn.append(branch3x3dbl_3)
inceptionE_2_bn.append(branch3x3dbl_4)
concat_branch3x3dbl = flow.concat(values=inceptionE_2_bn,
axis=1,
name="concat")
with flow.scope.namespace("branch_pool"):
if pooltype == "avg":
branch_pool_1 = flow.nn.avg_pool2d(
in_blob,
ksize=3,
strides=1,
padding="SAME",
data_format="NCHW",
name="pool",
)
elif pooltype == "max":
branch_pool_1 = flow.nn.max_pool2d(
in_blob,
ksize=3,
strides=1,
padding="SAME",
data_format="NCHW",
name="pool",
)
branch_pool_2 = conv2d_layer(
"conv",
branch_pool_1,
filters=192,
kernel_size=[1, 1],
strides=1,
padding="SAME",
)
inceptionE_total_bn = []
inceptionE_total_bn.append(branch1x1)
inceptionE_total_bn.append(concat_branch3x3)
inceptionE_total_bn.append(concat_branch3x3dbl)
inceptionE_total_bn.append(branch_pool_2)
concat_total = flow.concat(values=inceptionE_total_bn,
axis=1,
name="concat")
return concat_total
def InceptionC(in_blob, index, filters):
with flow.scope.namespace("mixed_{}".format(index)):
with flow.scope.namespace("branch1x1"):
branch1x1 = conv2d_layer("conv0",
in_blob,
filters=192,
kernel_size=1,
strides=1,
padding="SAME")
with flow.scope.namespace("branch7x7"):
branch7x7_1 = conv2d_layer(
"conv0",
in_blob,
filters=filters,
kernel_size=1,
strides=1,
padding="SAME",
)
branch7x7_2 = conv2d_layer(
"conv1",
branch7x7_1,
filters=filters,
kernel_size=[1, 7],
strides=1,
padding="SAME",
)
branch7x7_3 = conv2d_layer(
"conv2",
branch7x7_2,
filters=192,
kernel_size=[7, 1],
strides=[1, 1],
padding="SAME",
)
with flow.scope.namespace("branch7x7dbl"):
branch7x7dbl_1 = conv2d_layer(
"conv0",
in_blob,
filters=filters,
kernel_size=1,
strides=1,
padding="SAME",
)
branch7x7dbl_2 = conv2d_layer(
"conv1",
branch7x7dbl_1,
filters=filters,
kernel_size=[7, 1],
strides=1,
padding="SAME",
)
branch7x7dbl_3 = conv2d_layer(
"conv2",
branch7x7dbl_2,
filters=filters,
kernel_size=[1, 7],
strides=1,
padding="SAME",
)
branch7x7dbl_4 = conv2d_layer(
"conv3",
branch7x7dbl_3,
filters=filters,
kernel_size=[7, 1],
strides=1,
padding="SAME",
)
branch7x7dbl_5 = conv2d_layer(
"conv4",
branch7x7dbl_4,
filters=192,
kernel_size=[1, 7],
strides=1,
padding="SAME",
)
with flow.scope.namespace("branch_pool"):
branch_pool_1 = flow.nn.avg_pool2d(
in_blob,
ksize=3,
strides=1,
padding="SAME",
data_format="NCHW",
name="pool",
)
branch_pool_2 = conv2d_layer(
"conv",
branch_pool_1,
filters=192,
kernel_size=[1, 1],
strides=1,
padding="SAME",
)
inceptionC_bn = []
inceptionC_bn.append(branch1x1)
inceptionC_bn.append(branch7x7_3)
inceptionC_bn.append(branch7x7dbl_5)
inceptionC_bn.append(branch_pool_2)
mixed_concat = flow.concat(values=inceptionC_bn, axis=1, name="concat")
return mixed_concat
def InceptionA(in_blob, index):
with flow.scope.namespace("mixed_{}".format(index)):
with flow.scope.namespace("branch1x1"):
branch1x1 = conv2d_layer("conv0",
in_blob,
filters=64,
kernel_size=1,
strides=1,
padding="SAME")
with flow.scope.namespace("branch5x5"):
branch5x5_1 = conv2d_layer("conv0",
in_blob,
filters=48,
kernel_size=1,
strides=1,
padding="SAME")
branch5x5_2 = conv2d_layer(
"conv1",
branch5x5_1,
filters=64,
kernel_size=5,
strides=1,
padding="SAME",
)
with flow.scope.namespace("branch3x3dbl"):
branch3x3dbl_1 = conv2d_layer("conv0",
in_blob,
filters=64,
kernel_size=1,
strides=1,
padding="SAME")
branch3x3dbl_2 = conv2d_layer(
"conv1",
branch3x3dbl_1,
filters=96,
kernel_size=3,
strides=1,
padding="SAME",
)
branch3x3dbl_3 = conv2d_layer(
"conv2",
branch3x3dbl_2,
filters=96,
kernel_size=3,
strides=1,
padding="SAME",
)
with flow.scope.namespace("branch_pool"):
branch_pool_1 = flow.nn.avg_pool2d(
in_blob,
ksize=3,
strides=1,
padding="SAME",
data_format="NCHW",
name="pool",
)
branch_pool_2 = conv2d_layer(
"conv",
branch_pool_1,
filters=32 if index == 0 else 64,
kernel_size=1,
strides=1,
padding="SAME",
)
inceptionA_bn = []
inceptionA_bn.append(branch1x1)
inceptionA_bn.append(branch5x5_2)
inceptionA_bn.append(branch3x3dbl_3)
inceptionA_bn.append(branch_pool_2)
mixed_concat = flow.concat(values=inceptionA_bn, axis=1, name="concat")
return mixed_concat
def trt_concat_job(
x=flow.FixedTensorDef(a_shape, dtype=dtype),
y=flow.FixedTensorDef(b_shape, dtype=dtype),
):
return flow.concat([x, y], axis=axis)
