def TestMultiInputJob():
with flow.scope.placement("gpu", "0:0"):
x1 = flow.get_variable(
"x1",
shape=shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=-10, maxval=10),
trainable=True,
)
x2 = flow.get_variable(
"x2",
shape=shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=-10, maxval=10),
trainable=True,
)
loss = TestMultiInput(x1, x2)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0
).minimize(loss)
flow.watch(x1, test_global_storage.Setter("x1"))
flow.watch_diff(x1, test_global_storage.Setter("x1_diff"))
flow.watch(x2, test_global_storage.Setter("x2"))
flow.watch_diff(x2, test_global_storage.Setter("x2_diff"))
return loss
def _hybrid_embedding(name, ids, embedding_size, vocab_size, hf_vocab_size):
b, s = ids.shape
ids = flow.flatten(ids)
unique_ids, unique_ids_idx, _, _ = flow.experimental.unique_with_counts(ids)
hf_vocab_size_constant = flow.constant(hf_vocab_size, dtype=flow.int32)
hf_indices = flow.argwhere(flow.math.less(unique_ids, hf_vocab_size_constant))
lf_indices = flow.argwhere(flow.math.greater_equal(unique_ids, hf_vocab_size_constant))
hf_ids = flow.gather_nd(params=unique_ids, indices=hf_indices)
lf_ids = flow.gather_nd(params=unique_ids, indices=lf_indices)
hf_embedding_table = flow.get_variable(
name=f'hf_{name}',
shape=(hf_vocab_size, embedding_size),
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05),
)
hf_embedding = flow.gather(params=hf_embedding_table, indices=hf_ids)
lf_ids = lf_ids - hf_vocab_size_constant
with flow.scope.placement('cpu', '0:0'):
lf_embedding_table = flow.get_variable(
name=f'lf_{name}',
shape=(vocab_size - hf_vocab_size, embedding_size),
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05),
)
lf_embedding = flow.gather(params=lf_embedding_table, indices=lf_ids)
unique_embedding = flow.reshape(flow.zeros_like(unique_ids, dtype=flow.float), (-1, 1)) * flow.constant(0.0, dtype=flow.float, shape=(1,embedding_size))
unique_embedding = flow.tensor_scatter_nd_update(params=unique_embedding, updates=hf_embedding, indices=hf_indices)
unique_embedding = flow.tensor_scatter_nd_update(params=unique_embedding, updates=lf_embedding, indices=lf_indices)
unique_embedding = flow.gather(params=unique_embedding, indices=unique_ids_idx)
unique_embedding = flow.cast_to_static_shape(unique_embedding)
unique_embedding = flow.reshape(unique_embedding, shape=(b, s*embedding_size))
return unique_embedding
def _conv2d_layer(
name,
input,
filters,
kernel_size=3,
strides=1,
padding="SAME",
data_format="NCHW",
dilation_rate=1,
activation=flow.math.relu,
use_bias=False,
weight_initializer=flow.random_uniform_initializer(),
bias_initializer=flow.random_uniform_initializer(),
):
weight_shape = (filters, input.shape[1], kernel_size, kernel_size)
weight = flow.get_variable(
name + "-weight",
shape=weight_shape,
dtype=input.dtype,
initializer=weight_initializer,
)
output = flow.nn.conv2d(
input, weight, strides, padding, None, data_format, dilation_rate, name=name
)
if use_bias:
bias = flow.get_variable(
name + "-bias",
shape=(filters,),
dtype=input.dtype,
initializer=bias_initializer,
)
output = flow.nn.bias_add(output, bias, data_format)
if callable(activation):
output = activation(output)
return output
def ConcatJob():
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"x",
shape=x_shape,
dtype=type_name_to_flow_type[dtype],
initializer=flow.random_uniform_initializer(minval=-10, maxval=10),
trainable=True,
)
y = flow.get_variable(
"y",
shape=y_shape,
dtype=type_name_to_flow_type[dtype],
initializer=flow.random_uniform_initializer(minval=-10, maxval=10),
trainable=True,
)
x = flow.cast_to_current_logical_view(x)
y = flow.cast_to_current_logical_view(y)
loss = flow.concat([x, y], axis)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0
).minimize(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(y, test_global_storage.Setter("y"))
flow.watch_diff(y, test_global_storage.Setter("y_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return loss
def MatmulJob():
with flow.scope.placement(device_type, "0:0"):
a = flow.get_variable(
"a",
shape=a_shape,
dtype=dtype,
initializer=flow.random_uniform_initializer(minval=0,
maxval=1),
trainable=True,
)
b = flow.get_variable(
"b",
shape=b_shape,
dtype=dtype,
initializer=flow.random_uniform_initializer(minval=0,
maxval=1),
trainable=True,
)
if data_type == "float16":
out = flow.matmul(
flow.cast(a, dtype=flow.float16),
flow.cast(b, dtype=flow.float16),
transpose_a,
transpose_b,
alpha,
)
c = flow.get_variable(
"c",
shape=out.shape,
dtype=dtype,
initializer=flow.random_uniform_initializer(minval=-1,
maxval=1),
trainable=True,
)
loss = flow.cast(out + flow.cast(c, dtype=flow.float16),
dtype=flow.float)
else:
out = flow.matmul(a, b, transpose_a, transpose_b, alpha)
c = flow.get_variable(
"c",
shape=out.shape,
dtype=dtype,
initializer=flow.random_uniform_initializer(minval=-1,
maxval=1),
trainable=True,
)
loss = out + c
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [0.0001]),
momentum=0).minimize(loss)
flow.watch(a, test_global_storage.Setter("a"))
flow.watch_diff(a, test_global_storage.Setter("a_diff"))
flow.watch(b, test_global_storage.Setter("b"))
flow.watch_diff(b, test_global_storage.Setter("b_diff"))
flow.watch(c, test_global_storage.Setter("c"))
flow.watch_diff(c, test_global_storage.Setter("c_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return loss
def _model(dense_fields, wide_sparse_fields, deep_sparse_fields):
wide_sparse_fields = flow.parallel_cast(
wide_sparse_fields, distribute=flow.distribute.broadcast())
wide_embedding_table = flow.get_variable(
name='wide_embedding',
shape=(FLAGS.wide_vocab_size, 1),
initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05),
distribute=flow.distribute.split(0),
)
wide_embedding = flow.gather(params=wide_embedding_table,
indices=wide_sparse_fields)
wide_embedding = flow.reshape(wide_embedding,
shape=(-1, wide_embedding.shape[-1] *
wide_embedding.shape[-2]))
wide_scores = flow.math.reduce_sum(wide_embedding, axis=[1], keepdims=True)
wide_scores = flow.parallel_cast(
wide_scores,
distribute=flow.distribute.split(0),
gradient_distribute=flow.distribute.broadcast())
deep_sparse_fields = flow.parallel_cast(
deep_sparse_fields, distribute=flow.distribute.broadcast())
deep_embedding_table = flow.get_variable(
name='deep_embedding',
shape=(FLAGS.deep_vocab_size, FLAGS.deep_embedding_vec_size),
initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05),
distribute=flow.distribute.split(1),
)
deep_embedding = flow.gather(params=deep_embedding_table,
indices=deep_sparse_fields)
deep_embedding = flow.parallel_cast(
deep_embedding,
distribute=flow.distribute.split(0),
gradient_distribute=flow.distribute.split(2))
deep_embedding = flow.reshape(deep_embedding,
shape=(-1, deep_embedding.shape[-1] *
deep_embedding.shape[-2]))
deep_features = flow.concat([deep_embedding, dense_fields], axis=1)
for idx, units in enumerate(DEEP_HIDDEN_UNITS):
deep_features = flow.layers.dense(
deep_features,
units=units,
kernel_initializer=flow.glorot_uniform_initializer(),
bias_initializer=flow.constant_initializer(0.0),
activation=flow.math.relu,
name='fc' + str(idx + 1))
deep_features = flow.nn.dropout(deep_features,
rate=FLAGS.deep_dropout_rate)
deep_scores = flow.layers.dense(
deep_features,
units=1,
kernel_initializer=flow.glorot_uniform_initializer(),
bias_initializer=flow.constant_initializer(0.0),
name='fc' + str(len(DEEP_HIDDEN_UNITS) + 1))
scores = wide_scores + deep_scores
return scores
def _data_loader_synthetic(batch_size):
def _blob_random(shape, dtype=flow.int32, initializer=flow.zeros_initializer(flow.int32)):
return flow.data.decode_random(shape=shape, dtype=dtype, batch_size=batch_size,
initializer=initializer)
labels = _blob_random((1,), initializer=flow.random_uniform_initializer(dtype=flow.int32))
dense_fields = _blob_random((FLAGS.num_dense_fields,), dtype=flow.float,
initializer=flow.random_uniform_initializer())
wide_sparse_fields = _blob_random((FLAGS.num_wide_sparse_fields,))
deep_sparse_fields = _blob_random((FLAGS.num_deep_sparse_fields,))
print('use synthetic data')
return [labels, dense_fields, wide_sparse_fields, deep_sparse_fields]
def TestMultiOptimizerJob():
with flow.scope.placement(device_type, "0:0-0"):
var1 = flow.get_variable(
name="var1",
shape=var1_shape,
dtype=flow.float32,
initializer=flow.random_uniform_initializer(minval=0,
maxval=100),
trainable=True,
)
var2 = flow.get_variable(
name="var2",
shape=var2_shape,
dtype=flow.float32,
initializer=flow.random_uniform_initializer(minval=0,
maxval=100),
trainable=True,
)
var3 = flow.get_variable(
name="var3",
shape=var3_shape,
dtype=flow.float32,
initializer=flow.random_uniform_initializer(minval=0,
maxval=100),
trainable=True,
)
loss = flow.math.reduce_sum(var1 + var2 + var3)
sgd_opt = flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler(
[], [sgd_opt_args["lr"]]),
momentum=sgd_opt_args["momentum"],
variables=["var1"],
)
rmsprop_opt = flow.optimizer.RMSProp(
flow.optimizer.PiecewiseConstantScheduler(
[], [rmsprop_opt_args["lr"]]),
decay_rate=rmsprop_opt_args["decay_rate"],
epsilon=0,
centered=rmsprop_opt_args["centered"],
variables=["var2"],
)
adam_opt = flow.optimizer.Adam(
flow.optimizer.PiecewiseConstantScheduler(
[], [adam_opt_args["lr"]]),
beta1=adam_opt_args["beta1"],
beta2=adam_opt_args["beta2"],
epsilon=adam_opt_args["epsilon"],
do_bias_correction=True,
variables=["var3"],
)
flow.optimizer.CombinedOptimizer([sgd_opt, rmsprop_opt,
adam_opt]).minimize(loss)
return (var1, var2, var3)
def RunConvBias():
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"x",
shape=x_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=0,
maxval=100),
trainable=True,
)
if data_format == "NCHW":
weight_shape = (filters, x.shape[1] // groups, kernel_size,
kernel_size)
else:
weight_shape = (filters, kernel_size, kernel_size,
x.shape[3] // groups)
weight = flow.get_variable(
"conv-weight",
shape=weight_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=0,
maxval=100),
)
bias = flow.get_variable(
"conv-bias",
shape=(filters, ),
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=0,
maxval=100),
)
loss = flow.nn.conv2d(
x,
weight,
bias=bias,
strides=[stride, stride],
padding=padding,
dilations=[1, 1],
groups=groups,
name="conv",
)
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [0.0001]),
momentum=0).minimize(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(weight, test_global_storage.Setter("weight"))
flow.watch_diff(weight, test_global_storage.Setter("weight_diff"))
flow.watch(bias, test_global_storage.Setter("bias"))
flow.watch_diff(bias, test_global_storage.Setter("bias_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return loss
def add() -> tp.Numpy:
with flow.scope.placement("gpu", "0:0-1"):
x = flow.get_variable(
name="x",
shape=(2, 3),
initializer=flow.random_uniform_initializer(),
)
y = flow.get_variable(
name="y",
shape=(2, 3),
initializer=flow.random_uniform_initializer(),
)
return flow.math.add_n([x, y])
def alexnet(images, labels):
conv1 = _conv2d_layer("conv1",
images,
filters=64,
kernel_size=11,
strides=4,
padding="VALID")
pool1 = flow.nn.avg_pool2d(conv1, 3, 2, "VALID", "NCHW", name="pool1")
conv2 = _conv2d_layer("conv2", pool1, filters=192, kernel_size=5)
pool2 = flow.nn.avg_pool2d(conv2, 3, 2, "VALID", "NCHW", name="pool2")
conv3 = _conv2d_layer("conv3", pool2, filters=384)
conv4 = _conv2d_layer("conv4", conv3, filters=384)
conv5 = _conv2d_layer("conv5", conv4, filters=256)
pool5 = flow.nn.avg_pool2d(conv5, 3, 2, "VALID", "NCHW", name="pool5")
if len(pool5.shape) > 2:
pool5 = flow.reshape(pool5, shape=(pool5.shape[0], -1))
fc1 = flow.layers.dense(
inputs=pool5,
units=4096,
activation=flow.math.relu,
use_bias=False,
kernel_initializer=flow.random_uniform_initializer(),
bias_initializer=False,
trainable=True,
name="fc1",
)
dropout1 = flow.nn.dropout(fc1, rate=0.5)
fc2 = flow.layers.dense(
inputs=dropout1,
units=4096,
activation=flow.math.relu,
use_bias=False,
kernel_initializer=flow.random_uniform_initializer(),
bias_initializer=False,
trainable=True,
name="fc2",
)
dropout2 = flow.nn.dropout(fc2, rate=0.5)
fc3 = flow.layers.dense(
inputs=dropout2,
units=1001,
activation=None,
use_bias=False,
kernel_initializer=flow.random_uniform_initializer(),
bias_initializer=False,
trainable=True,
name="fc3",
)
loss = flow.nn.sparse_softmax_cross_entropy_with_logits(
labels, fc3, name="softmax_loss")
return loss
def ConvJob():
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"x",
shape=x_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=0, maxval=100),
trainable=True,
)
if data_format == "NCDHW":
weight_shape = (
filters,
x.shape[1] // groups,
kernel_size,
kernel_size,
kernel_size,
)
else:
weight_shape = (
filters,
kernel_size,
kernel_size,
kernel_size,
x.shape[4] // groups,
)
weight = flow.get_variable(
"conv-weight",
shape=weight_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=0, maxval=100),
)
loss = flow.nn.conv3d(
x,
weight,
strides=[stride_d, stride_h, stride_w],
padding=of_padding,
data_format=data_format,
dilations=[dilation_d, dilation_h, dilation_w],
groups=groups,
)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0
).minimize(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(weight, test_global_storage.Setter("weight"))
flow.watch_diff(weight, test_global_storage.Setter("weight_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return loss
def ConvJob():
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"x",
shape=x_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=0,
maxval=100),
trainable=True,
)
loss = flow.layers.conv3d(
x,
filters,
kernel_size=kernel_size,
strides=1,
padding="valid",
data_format="NCDHW",
dilation_rate=1,
groups=groups,
use_bias=False,
kernel_initializer=flow.random_uniform_initializer(minval=0,
maxval=100),
weight_name="conv3d_weight",
)
weight_shape = (
filters,
x.shape[1] // groups,
kernel_size,
kernel_size,
kernel_size,
)
weight = flow.get_variable(
name="conv3d_weight",
shape=weight_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=0,
maxval=100),
)
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [0.0001]),
momentum=0).minimize(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(weight, test_global_storage.Setter("weight"))
flow.watch_diff(weight, test_global_storage.Setter("weight_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return loss
def _conv2d_layer(
args,
name,
input,
filters,
kernel_size=3,
strides=1,
padding="SAME",
data_format="NCHW",
dilation_rate=1,
activation=op_conf_util.kRelu,
use_bias=False,
weight_initializer=flow.random_uniform_initializer(),
bias_initializer=flow.random_uniform_initializer(),
):
weight_shape = (filters, input.shape[1], kernel_size, kernel_size)
weight = flow.get_variable(
name + "-weight",
shape=weight_shape,
dtype=input.dtype,
initializer=weight_initializer,
)
weight = flow.identity(weight)
weight = flow.repeat(weight, args.num_piece_in_batch)
output = flow.nn.conv2d(input,
weight,
strides,
padding,
None,
data_format,
dilation_rate,
name=name)
if use_bias:
bias = flow.get_variable(
name + "-bias",
shape=(filters, ),
dtype=input.dtype,
initializer=bias_initializer,
)
bias = flow.identity(bias)
bias = flow.repeat(bias, args.num_piece_in_batch)
output = flow.nn.bias_add(output, bias, data_format)
if activation is not None:
if activation == op_conf_util.kRelu:
output = flow.math.relu(output)
else:
raise NotImplementedError
return output
def PartialFcJob(labels: oft.Numpy.Placeholder(
(batch_size, ), dtype=type_name_to_flow_type[label_type])):
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"x-weight",
shape=(num_classes, 128),
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=-10,
maxval=10),
trainable=True,
)
with flow.scope.placement(device_type, "0:0-3"):
lebels_distribute = flow.distribute.broadcast()
weight_distribute = flow.distribute.split(0)
(
maped_label,
sampled_label,
sampled_weight,
) = flow.distributed_partial_fc_sample(
weight=x.with_distribute(weight_distribute),
label=labels.with_distribute(lebels_distribute),
num_sample=num_sample,
)
with flow.scope.placement(device_type, "0:0"):
sampled_weight = flow.identity(sampled_weight)
loss = flow.math.square(sampled_weight)
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [0.0001]),
momentum=0).minimize(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch_diff(sampled_weight,
test_global_storage.Setter("sampled_weight_diff"))
return (x, maped_label, sampled_label, sampled_weight)
def SoftmaxJob():
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"x",
shape=x_shape,
dtype=dtype,
initializer=flow.random_uniform_initializer(minval=-1.0,
maxval=1.0),
trainable=True,
)
x1 = x
x = flow.identity(x)
if data_type == "float16":
loss = flow.cast(
flow.nn.softmax(flow.cast(x, dtype=flow.float16),
axis=axis),
dtype=flow.float,
)
else:
loss = flow.nn.softmax(x, axis=axis)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
total_loss = loss * x1
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [0.0001]),
momentum=0).minimize(total_loss)
return loss
def DropoutJob():
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"x",
shape=x_shape,
dtype=dtype,
initializer=flow.random_uniform_initializer(minval=-1,
maxval=1),
trainable=True,
)
if data_type == "float16":
x = flow.cast(flow.cast(x, flow.float16), dtype)
of_out = flow.cast(
flow.nn.dropout(flow.cast(x, flow.float16),
rate=rate,
name="dropout"),
dtype,
)
else:
of_out = flow.nn.dropout(x, rate=rate, name="dropout")
loss = flow.math.square(of_out)
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [0.0001]),
momentum=0).minimize(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(of_out, test_global_storage.Setter("out"))
flow.watch_diff(of_out, test_global_storage.Setter("out_diff"))
return loss
def get_var(var_name):
return flow.get_variable(
name=var_name,
shape=(2, 256, 14, 14),
dtype=flow.float32,
initializer=flow.random_uniform_initializer(),
)
def SparseSoftmaxCrossEntropyWithLogitsJob(labels: oft.Numpy.Placeholder(
(batch_size, ), dtype=type_name_to_flow_type[label_type])):
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"x",
shape=(batch_size, num_classes),
dtype=type_name_to_flow_type[data_type],
initializer=flow.random_uniform_initializer(minval=-10,
maxval=10),
trainable=True,
)
with flow.scope.placement(device_type, "0:0-3"):
labels = flow.parallel_cast(labels,
distribute=flow.distribute.broadcast())
logits = flow.parallel_cast(
x, distribute=flow.distribute.split(len(x.shape) - 1))
loss = flow.nn.distributed_sparse_softmax_cross_entropy_with_logits(
labels, logits)
loss = flow.math.square(loss)
with flow.scope.placement(device_type, "0:0"):
loss = flow.identity(loss)
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [0.0001]),
momentum=0).minimize(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return loss
def decode_random(
shape: Sequence[int],
dtype: flow.dtype,
batch_size: int = 1,
initializer: Optional[initializer_conf_util.InitializerConf] = None,
tick: Optional[oneflow._oneflow_internal.BlobDesc] = None,
name: Optional[str] = None,
) -> oneflow._oneflow_internal.BlobDesc:
op_conf = op_conf_util.OperatorConf()
if name is None:
name = id_util.UniqueStr("DecodeRandom_")
assert isinstance(name, str)
op_conf.name = name
assert isinstance(shape, (list, tuple))
op_conf.decode_random_conf.shape.dim.extend(shape)
assert dtype is not None
setattr(
op_conf.decode_random_conf,
"data_type",
oneflow._oneflow_internal.deprecated.GetProtoDtype4OfDtype(dtype),
)
op_conf.decode_random_conf.batch_size = batch_size
if initializer is not None:
op_conf.decode_random_conf.data_initializer.CopyFrom(initializer)
else:
op_conf.decode_random_conf.data_initializer.CopyFrom(
flow.random_uniform_initializer())
if tick:
op_conf.decode_random_conf.tick = tick.unique_name
op_conf.decode_random_conf.out = "out"
lbi = logical_blob_id_util.LogicalBlobId()
lbi.op_name = op_conf.name
lbi.blob_name = "out"
interpret_util.ConsistentForward(op_conf)
return remote_blob_util.RemoteBlob(lbi)
def hybrid_concat_job(
input_0_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float),
input_1_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float),
):
var = flow.get_variable(
"var",
shape=static_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(),
trainable=True,
)
constant = flow.constant(1.0, dtype=flow.float, shape=rand_sub_shape)
inputs = [
flow.cast_to_current_logical_view(input)
for input in [var, input_0_def, input_1_def, constant]
]
concated = flow.concat(inputs, axis=axis, max_dim_size=max_dim_size)
if verbose:
print("concated static shape:", concated.shape)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0
).minimize(concated)
flow.watch_diff(var, compare_var_diff)
if max_dim_size is None:
test_case.assertTrue(
concated.shape[axis] == static_shape[axis] * 3 + rand_sub_shape[axis]
)
else:
test_case.assertTrue(concated.shape[axis] == max_dim_size)
return (var, concated)
def UpsampleJob():
with flow.scope.placement(device_type, "0:0"):
x = flow.get_variable(
"input",
shape=input_shape,
dtype=type_name_to_flow_type[dtype],
initializer=flow.random_uniform_initializer(minval=2,
maxval=5),
trainable=True,
)
loss = flow.layers.upsample_2d(
x,
size=size,
data_format=data_format,
interpolation=interpolation,
align_corners=align_corners,
)
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [0.0001]),
momentum=0).minimize(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return loss
def get_v():
return flow.get_variable(
name="var",
shape=(5, 2),
dtype=flow.float32,
initializer=flow.random_uniform_initializer(),
)
def deconv(input,
out_channel,
name_prefix,
kernel_size=4,
strides=[2, 2],
trainable=True):
weight = flow.get_variable(
name_prefix + "_weight",
shape=(input.shape[1], out_channel, kernel_size, kernel_size),
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=-10, maxval=10),
trainable=True,
)
return flow.nn.conv2d_transpose(
input,
weight,
strides=strides,
padding="SAME",
output_shape=(
input.shape[0],
out_channel,
input.shape[2] * strides[0],
input.shape[3] * strides[1],
),
)
def get_var() -> tp.Numpy:
return flow.get_variable(
name="var",
shape=shape,
dtype=dtype,
initializer=flow.random_uniform_initializer(),
reuse=True,
)
def get_var(name, shape=(2, 5), dtype=flow.float, trainable=False):
return flow.get_variable(
name=name,
shape=shape,
dtype=dtype,
trainable=trainable,
initializer=flow.random_uniform_initializer(),
)
def _embedding(name, ids, embedding_size, vocab_size, split_axis=0):
ids = flow.parallel_cast(ids, distribute=flow.distribute.broadcast())
params = flow.get_variable(
name=name,
shape=(vocab_size, embedding_size),
initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05),
distribute=flow.distribute.split(split_axis),
)
embedding = flow.gather(params=params, indices=ids)
embedding = flow.reshape(embedding, shape=(-1, embedding.shape[-1] * embedding.shape[-2]))
return embedding
def gen_var(x: tp.Numpy.Placeholder(shape=shape, dtype=dtype)) -> tp.Numpy:
var = flow.get_variable(
name="var",
shape=shape,
dtype=dtype,
initializer=flow.random_uniform_initializer(),
)
y = var + x
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler([], [lr]),
momentum=0).minimize(y)
return var
def FuseCastScaleJob(x: oft.Numpy.Placeholder(
shape, dtype=in_type)) -> Tuple[oft.Numpy, oft.Numpy]:
with flow.scope.placement(device, "0:0-0"):
scale = flow.get_variable(
"scale",
shape=(1, ),
dtype=out_type,
initializer=flow.random_uniform_initializer(),
trainable=False,
)
loss = flow.cast(x, dtype=out_type) * scale
return (loss, scale)
def matmul_job() -> typing.Tuple[flow.typing.Numpy, flow.typing.Numpy,
flow.typing.Numpy, flow.typing.Numpy]:
a_var = flow.get_variable(
"a",
shape=a_shape,
dtype=flow.float32,
initializer=flow.random_uniform_initializer(minval=0, maxval=1),
trainable=True,
)
b_var = flow.get_variable(
"b",
shape=b_shape,
dtype=flow.float32,
initializer=flow.random_uniform_initializer(minval=0, maxval=1),
trainable=True,
)
flow.watch_diff(a_var, test_global_storage.Setter("a_diff"))
flow.watch_diff(b_var, test_global_storage.Setter("b_diff"))
if dtype is flow.float16:
a = flow.amp_white_identity(a_var)
b = flow.amp_white_identity(b_var)
else:
a = a_var
b = b_var
c = flow.matmul(a, b, trans_a, trans_b, alpha)
add_to = flow.get_variable(
"c",
shape=c.shape,
dtype=flow.float32,
initializer=flow.random_uniform_initializer(minval=-1, maxval=1),
trainable=True,
)
if test_add_to_output:
flow.watch_diff(add_to, test_global_storage.Setter("add_to_diff"))
if dtype is flow.float16:
add_to = flow.amp_white_identity(add_to)
c = c + add_to
flow.watch_diff(c, test_global_storage.Setter("c_diff"))
get_optimizer().minimize(c)
return (a_var, b_var, add_to, c)