def _model(dense_fields, wide_sparse_fields, deep_sparse_fields): # wide_embedding = _embedding('wide_embedding', wide_sparse_fields, 1, FLAGS.wide_vocab_size) wide_embedding = _hybrid_embedding('wide_embedding', wide_sparse_fields, 1, FLAGS.wide_vocab_size, FLAGS.hf_wide_vocab_size) wide_scores = flow.math.reduce_sum(wide_embedding, axis=[1], keepdims=True) wide_scores = flow.parallel_cast(wide_scores, distribute=flow.distribute.split(0), gradient_distribute=flow.distribute.broadcast()) # deep_embedding = _embedding('deep_embedding', deep_sparse_fields, FLAGS.deep_embedding_vec_size, # FLAGS.deep_vocab_size, split_axis=1) deep_embedding = _hybrid_embedding('deep_embedding', deep_sparse_fields, FLAGS.deep_embedding_vec_size, FLAGS.deep_vocab_size, FLAGS.hf_deep_vocab_size) deep_features = flow.concat([deep_embedding, dense_fields], axis=1) for idx, units in enumerate(DEEP_HIDDEN_UNITS): deep_features = flow.layers.dense( deep_features, units=units, kernel_initializer=flow.glorot_uniform_initializer(), bias_initializer=flow.constant_initializer(0.0), activation=flow.math.relu, name='fc' + str(idx + 1) ) deep_features = flow.nn.dropout(deep_features, rate=FLAGS.deep_dropout_rate) deep_scores = flow.layers.dense( deep_features, units=1, kernel_initializer=flow.glorot_uniform_initializer(), bias_initializer=flow.constant_initializer(0.0), name='fc' + str(len(DEEP_HIDDEN_UNITS) + 1) ) scores = wide_scores + deep_scores return scores
def ConcatJob(): with flow.scope.placement(device_type, "0:0"): x = flow.get_variable( "x", shape=x_shape, dtype=type_name_to_flow_type[dtype], initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) y = flow.get_variable( "y", shape=y_shape, dtype=type_name_to_flow_type[dtype], initializer=flow.random_uniform_initializer(minval=-10, maxval=10), trainable=True, ) x = flow.cast_to_current_logical_view(x) y = flow.cast_to_current_logical_view(y) loss = flow.concat([x, y], axis) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(y, test_global_storage.Setter("y")) flow.watch_diff(y, test_global_storage.Setter("y_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return loss
def hybrid_concat_job( input_0_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float), input_1_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float), ): var = flow.get_variable( "var", shape=static_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(), trainable=True, ) constant = flow.constant(1.0, dtype=flow.float, shape=rand_sub_shape) inputs = [ flow.cast_to_current_logical_view(input) for input in [var, input_0_def, input_1_def, constant] ] concated = flow.concat(inputs, axis=axis, max_dim_size=max_dim_size) if verbose: print("concated static shape:", concated.shape) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0 ).minimize(concated) flow.watch_diff(var, compare_var_diff) if max_dim_size is None: test_case.assertTrue( concated.shape[axis] == static_shape[axis] * 3 + rand_sub_shape[axis] ) else: test_case.assertTrue(concated.shape[axis] == max_dim_size) return (var, concated)
def _model(dense_fields, wide_sparse_fields, deep_sparse_fields): wide_sparse_fields = flow.parallel_cast( wide_sparse_fields, distribute=flow.distribute.broadcast()) wide_embedding_table = flow.get_variable( name='wide_embedding', shape=(FLAGS.wide_vocab_size, 1), initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05), distribute=flow.distribute.split(0), ) wide_embedding = flow.gather(params=wide_embedding_table, indices=wide_sparse_fields) wide_embedding = flow.reshape(wide_embedding, shape=(-1, wide_embedding.shape[-1] * wide_embedding.shape[-2])) wide_scores = flow.math.reduce_sum(wide_embedding, axis=[1], keepdims=True) wide_scores = flow.parallel_cast( wide_scores, distribute=flow.distribute.split(0), gradient_distribute=flow.distribute.broadcast()) deep_sparse_fields = flow.parallel_cast( deep_sparse_fields, distribute=flow.distribute.broadcast()) deep_embedding_table = flow.get_variable( name='deep_embedding', shape=(FLAGS.deep_vocab_size, FLAGS.deep_embedding_vec_size), initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05), distribute=flow.distribute.split(1), ) deep_embedding = flow.gather(params=deep_embedding_table, indices=deep_sparse_fields) deep_embedding = flow.parallel_cast( deep_embedding, distribute=flow.distribute.split(0), gradient_distribute=flow.distribute.split(2)) deep_embedding = flow.reshape(deep_embedding, shape=(-1, deep_embedding.shape[-1] * deep_embedding.shape[-2])) deep_features = flow.concat([deep_embedding, dense_fields], axis=1) for idx, units in enumerate(DEEP_HIDDEN_UNITS): deep_features = flow.layers.dense( deep_features, units=units, kernel_initializer=flow.glorot_uniform_initializer(), bias_initializer=flow.constant_initializer(0.0), activation=flow.math.relu, name='fc' + str(idx + 1)) deep_features = flow.nn.dropout(deep_features, rate=FLAGS.deep_dropout_rate) deep_scores = flow.layers.dense( deep_features, units=1, kernel_initializer=flow.glorot_uniform_initializer(), bias_initializer=flow.constant_initializer(0.0), name='fc' + str(len(DEEP_HIDDEN_UNITS) + 1)) scores = wide_scores + deep_scores return scores
def InceptionD(in_blob, index): with flow.scope.namespace("mixed_{}".format(index)): with flow.scope.namespace("branch3x3"): branch3x3_1 = _conv2d_layer( "conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME" ) branch3x3_2 = _conv2d_layer( "conv1", branch3x3_1, filters=320, kernel_size=3, strides=2, padding="VALID", ) with flow.scope.namespace("branch7x7x3"): branch7x7x3_1 = _conv2d_layer( "conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME" ) branch7x7x3_2 = _conv2d_layer( "conv1", branch7x7x3_1, filters=192, kernel_size=[1, 7], strides=1, padding="SAME", ) branch7x7x3_3 = _conv2d_layer( "conv2", branch7x7x3_2, filters=192, kernel_size=[7, 1], strides=1, padding="SAME", ) branch7x7x3_4 = _conv2d_layer( "conv3", branch7x7x3_3, filters=192, kernel_size=3, strides=2, padding="VALID", ) with flow.scope.namespace("branch_pool"): branch_pool = flow.nn.max_pool2d( in_blob, ksize=3, strides=2, padding="VALID", data_format="NCHW", name="pool", ) inceptionD_bn = [] inceptionD_bn.append(branch3x3_2) inceptionD_bn.append(branch7x7x3_4) inceptionD_bn.append(branch_pool) mixed_concat = flow.concat(values=inceptionD_bn, axis=1, name="concat") return mixed_concat
def InceptionB(in_blob, index): with flow.scope.namespace("mixed_{}".format(index)): with flow.scope.namespace("branch3x3"): branch3x3 = conv2d_layer("conv0", in_blob, filters=384, kernel_size=3, strides=2, padding="VALID") with flow.scope.namespace("branch3x3dbl"): branch3x3dbl_1 = conv2d_layer("conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME") branch3x3dbl_2 = conv2d_layer( "conv1", branch3x3dbl_1, filters=96, kernel_size=3, strides=1, padding="SAME", ) branch3x3dbl_3 = conv2d_layer( "conv2", branch3x3dbl_2, filters=96, kernel_size=3, strides=2, padding="VALID", ) with flow.scope.namespace("branch_pool"): branch_pool = flow.nn.max_pool2d( in_blob, ksize=3, strides=2, padding="VALID", data_format="NCHW", name="pool0", ) inceptionB_bn = [] inceptionB_bn.append(branch3x3) inceptionB_bn.append(branch3x3dbl_3) inceptionB_bn.append(branch_pool) mixed_concat = flow.concat(values=inceptionB_bn, axis=1, name="concat") return mixed_concat
def dynamic_concat_job( input_0_def: oft.ListNumpy.Placeholder( shape=input_static_shape, dtype=flow.float ), input_1_def: oft.ListNumpy.Placeholder( shape=input_static_shape, dtype=flow.float ), ): var_0 = flow.get_variable( "Var0", shape=(1,), dtype=flow.float, initializer=flow.constant_initializer(value=1, dtype=flow.float), trainable=True, ) var_1 = flow.get_variable( "Var1", shape=(1,), dtype=flow.float, initializer=flow.constant_initializer(value=1, dtype=flow.float), trainable=True, ) var_0 = flow.cast_to_current_logical_view(var_0) var_1 = flow.cast_to_current_logical_view(var_1) input_0_def = flow.cast_to_current_logical_view(input_0_def) input_1_def = flow.cast_to_current_logical_view(input_1_def) if callable(watch_cb): flow.watch(var_0, watch_cb) flow.watch(var_1, watch_cb) flow.watch(flow.identity(input_0_def), watch_cb) flow.watch(flow.identity(input_1_def), watch_cb) var_0 = var_0 * input_0_def var_1 = var_1 * input_1_def if callable(watch_cb): flow.watch(var_0, watch_cb) flow.watch(var_1, watch_cb) result = flow.concat( [var_0, var_1], axis=axis, max_dim_size=input_static_shape[axis] ) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(result) flow.watch_diff(var_0, make_watch_diff_cb(0)) flow.watch_diff(var_1, make_watch_diff_cb(1)) return result
def static_concat_job( input_0_def: oft.Numpy.Placeholder(shape=shape, dtype=flow.float), input_1_def: oft.Numpy.Placeholder(shape=shape, dtype=flow.float), ): var = flow.get_variable( "var", shape=shape, dtype=flow.float, initializer=flow.random_uniform_initializer(), trainable=True, ) concated = flow.concat([input_0_def, input_1_def, var], axis=axis) test_case.assertTrue(not concated.is_dynamic) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0 ).minimize(concated) flow.watch_diff(var, compare_var_diff) return (var, concated)
def InceptionE(in_blob, index, pooltype): with flow.scope.namespace("mixed_{}".format(index)): with flow.scope.namespace("branch1x1"): branch1x1 = conv2d_layer("conv0", in_blob, filters=320, kernel_size=1, strides=1, padding="SAME") with flow.scope.namespace("branch3x3"): branch3x3_1 = conv2d_layer("conv0", in_blob, filters=384, kernel_size=1, strides=1, padding="SAME") branch3x3_2 = conv2d_layer( "conv1", branch3x3_1, filters=384, kernel_size=[1, 3], strides=1, padding="SAME", ) branch3x3_3 = conv2d_layer( "conv2", branch3x3_1, filters=384, kernel_size=[3, 1], strides=[1, 1], padding="SAME", ) inceptionE_1_bn = [] inceptionE_1_bn.append(branch3x3_2) inceptionE_1_bn.append(branch3x3_3) concat_branch3x3 = flow.concat(values=inceptionE_1_bn, axis=1, name="concat") with flow.scope.namespace("branch3x3dbl"): branch3x3dbl_1 = conv2d_layer("conv0", in_blob, filters=448, kernel_size=1, strides=1, padding="SAME") branch3x3dbl_2 = conv2d_layer( "conv1", branch3x3dbl_1, filters=384, kernel_size=3, strides=1, padding="SAME", ) branch3x3dbl_3 = conv2d_layer( "conv2", branch3x3dbl_2, filters=384, kernel_size=[1, 3], strides=1, padding="SAME", ) branch3x3dbl_4 = conv2d_layer( "conv3", branch3x3dbl_2, filters=384, kernel_size=[3, 1], strides=1, padding="SAME", ) inceptionE_2_bn = [] inceptionE_2_bn.append(branch3x3dbl_3) inceptionE_2_bn.append(branch3x3dbl_4) concat_branch3x3dbl = flow.concat(values=inceptionE_2_bn, axis=1, name="concat") with flow.scope.namespace("branch_pool"): if pooltype == "avg": branch_pool_1 = flow.nn.avg_pool2d( in_blob, ksize=3, strides=1, padding="SAME", data_format="NCHW", name="pool", ) elif pooltype == "max": branch_pool_1 = flow.nn.max_pool2d( in_blob, ksize=3, strides=1, padding="SAME", data_format="NCHW", name="pool", ) branch_pool_2 = conv2d_layer( "conv", branch_pool_1, filters=192, kernel_size=[1, 1], strides=1, padding="SAME", ) inceptionE_total_bn = [] inceptionE_total_bn.append(branch1x1) inceptionE_total_bn.append(concat_branch3x3) inceptionE_total_bn.append(concat_branch3x3dbl) inceptionE_total_bn.append(branch_pool_2) concat_total = flow.concat(values=inceptionE_total_bn, axis=1, name="concat") return concat_total
def InceptionC(in_blob, index, filters): with flow.scope.namespace("mixed_{}".format(index)): with flow.scope.namespace("branch1x1"): branch1x1 = conv2d_layer("conv0", in_blob, filters=192, kernel_size=1, strides=1, padding="SAME") with flow.scope.namespace("branch7x7"): branch7x7_1 = conv2d_layer( "conv0", in_blob, filters=filters, kernel_size=1, strides=1, padding="SAME", ) branch7x7_2 = conv2d_layer( "conv1", branch7x7_1, filters=filters, kernel_size=[1, 7], strides=1, padding="SAME", ) branch7x7_3 = conv2d_layer( "conv2", branch7x7_2, filters=192, kernel_size=[7, 1], strides=[1, 1], padding="SAME", ) with flow.scope.namespace("branch7x7dbl"): branch7x7dbl_1 = conv2d_layer( "conv0", in_blob, filters=filters, kernel_size=1, strides=1, padding="SAME", ) branch7x7dbl_2 = conv2d_layer( "conv1", branch7x7dbl_1, filters=filters, kernel_size=[7, 1], strides=1, padding="SAME", ) branch7x7dbl_3 = conv2d_layer( "conv2", branch7x7dbl_2, filters=filters, kernel_size=[1, 7], strides=1, padding="SAME", ) branch7x7dbl_4 = conv2d_layer( "conv3", branch7x7dbl_3, filters=filters, kernel_size=[7, 1], strides=1, padding="SAME", ) branch7x7dbl_5 = conv2d_layer( "conv4", branch7x7dbl_4, filters=192, kernel_size=[1, 7], strides=1, padding="SAME", ) with flow.scope.namespace("branch_pool"): branch_pool_1 = flow.nn.avg_pool2d( in_blob, ksize=3, strides=1, padding="SAME", data_format="NCHW", name="pool", ) branch_pool_2 = conv2d_layer( "conv", branch_pool_1, filters=192, kernel_size=[1, 1], strides=1, padding="SAME", ) inceptionC_bn = [] inceptionC_bn.append(branch1x1) inceptionC_bn.append(branch7x7_3) inceptionC_bn.append(branch7x7dbl_5) inceptionC_bn.append(branch_pool_2) mixed_concat = flow.concat(values=inceptionC_bn, axis=1, name="concat") return mixed_concat
def InceptionA(in_blob, index): with flow.scope.namespace("mixed_{}".format(index)): with flow.scope.namespace("branch1x1"): branch1x1 = conv2d_layer("conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME") with flow.scope.namespace("branch5x5"): branch5x5_1 = conv2d_layer("conv0", in_blob, filters=48, kernel_size=1, strides=1, padding="SAME") branch5x5_2 = conv2d_layer( "conv1", branch5x5_1, filters=64, kernel_size=5, strides=1, padding="SAME", ) with flow.scope.namespace("branch3x3dbl"): branch3x3dbl_1 = conv2d_layer("conv0", in_blob, filters=64, kernel_size=1, strides=1, padding="SAME") branch3x3dbl_2 = conv2d_layer( "conv1", branch3x3dbl_1, filters=96, kernel_size=3, strides=1, padding="SAME", ) branch3x3dbl_3 = conv2d_layer( "conv2", branch3x3dbl_2, filters=96, kernel_size=3, strides=1, padding="SAME", ) with flow.scope.namespace("branch_pool"): branch_pool_1 = flow.nn.avg_pool2d( in_blob, ksize=3, strides=1, padding="SAME", data_format="NCHW", name="pool", ) branch_pool_2 = conv2d_layer( "conv", branch_pool_1, filters=32 if index == 0 else 64, kernel_size=1, strides=1, padding="SAME", ) inceptionA_bn = [] inceptionA_bn.append(branch1x1) inceptionA_bn.append(branch5x5_2) inceptionA_bn.append(branch3x3dbl_3) inceptionA_bn.append(branch_pool_2) mixed_concat = flow.concat(values=inceptionA_bn, axis=1, name="concat") return mixed_concat
def trt_concat_job( x=flow.FixedTensorDef(a_shape, dtype=dtype), y=flow.FixedTensorDef(b_shape, dtype=dtype), ): return flow.concat([x, y], axis=axis)