def avg_pooling_2d_k2s2_same_nhwc(x=flow.FixedTensorDef((2, 3, 5, 4))):
x += flow.get_variable(
name="v1",
shape=(1, 1),
dtype=flow.float,
initializer=flow.zeros_initializer(),
)
return flow.nn.avg_pool2d(
x, ksize=2, strides=2, padding="SAME", data_format="NHWC"
)
def max_pooling_2d_k3s1_valid_nchw(x: tp.Numpy.Placeholder((2, 3, 5, 4))):
x += flow.get_variable(
name="v1",
shape=(1, 1),
dtype=flow.float,
initializer=flow.zeros_initializer(),
)
return flow.nn.max_pool2d(
x, ksize=3, strides=1, padding="VALID", data_format="NCHW"
)
def avg_pooling_2d_k3s1_valid_nchw(x=flow.FixedTensorDef((2, 3, 5, 4))):
x += flow.get_variable(
name="v1",
shape=(1, 1),
dtype=flow.float,
initializer=flow.zeros_initializer(),
)
return flow.nn.avg_pool2d(
x, ksize=3, strides=1, padding="VALID", data_format="NCHW"
)
def max_pooling_2d_k2s2_same_nhwc(x: tp.Numpy.Placeholder((2, 3, 5, 4))):
x += flow.get_variable(
name="v1",
shape=(1, 1),
dtype=flow.float,
initializer=flow.zeros_initializer(),
)
return flow.nn.max_pool2d(
x, ksize=2, strides=2, padding="SAME", data_format="NHWC"
)
def test_job(
x: oft.Numpy.Placeholder(input_shape, dtype=flow.float32),
addend: oft.Numpy.Placeholder(input_shape, dtype=flow.float32),
):
v = flow.get_variable(
name="v",
shape=(1, ),
dtype=flow.float32,
initializer=flow.zeros_initializer(),
)
x = x + v
addend = addend + v
x1 = flow.identity(x)
x2 = flow.identity(x)
addend1 = flow.identity(addend)
addend2 = flow.identity(addend)
flow.watch_diff(x1, test_global_storage.Setter("x1_diff"))
flow.watch_diff(x2, test_global_storage.Setter("x2_diff"))
flow.watch_diff(addend1, test_global_storage.Setter("addend1_diff"))
flow.watch_diff(addend2, test_global_storage.Setter("addend2_diff"))
x1 = flow.cast(x1, data_type)
x2 = flow.cast(x2, data_type)
addend1 = flow.cast(addend1, data_type)
addend2 = flow.cast(addend2, data_type)
y1 = flow.layers.batch_normalization_add_relu(x1,
addend=addend1,
axis=axis,
name="BN1")
y2 = flow.math.relu(
flow.layers.batch_normalization(x2, axis=axis, name="BN2") +
addend2)
y1 = flow.cast(y1, flow.float32)
y2 = flow.cast(y2, flow.float32)
flow.watch(y1, test_global_storage.Setter("y1"))
flow.watch(y2, test_global_storage.Setter("y2"))
y1 = flow.where(flow.math.greater(y2, v), y1, v)
y2 = flow.where(flow.math.greater(y1, v), y2, v)
loss = y1 + y2
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler([],
[0.001]),
momentum=0).minimize(flow.math.reduce_sum(loss))
return loss
def conv2d_layer(
name,
input,
filters,
weight_initializer,
kernel_size=3,
strides=1,
padding="SAME",
data_format="NCHW",
dilation_rate=1,
activation="Relu",
use_bias=True,
bias_initializer=flow.zeros_initializer(),
weight_regularizer=_get_regularizer(), # weight_decay
bias_regularizer=_get_regularizer(),
bn=True,
):
weight_shape = (filters, input.shape[1], kernel_size,
kernel_size) if data_format == "NCHW" else (filters,
kernel_size,
kernel_size,
input.shape[3])
weight = flow.get_variable(
name + "_weight",
shape=weight_shape,
dtype=input.dtype,
initializer=weight_initializer,
)
output = flow.nn.conv2d(input,
weight,
strides,
padding,
data_format,
dilation_rate,
name=name)
if use_bias:
bias = flow.get_variable(
name + "_bias",
shape=(filters, ),
dtype=input.dtype,
initializer=bias_initializer,
)
output = flow.nn.bias_add(output, bias, data_format)
if activation is not None:
if activation == "Relu":
if bn:
output = _batch_norm(output, name + "_bn", True, data_format)
output = flow.nn.relu(output)
else:
output = flow.nn.relu(output)
else:
raise NotImplementedError
return output
def conv2d_layer(name,
input,
filters,
kernel_size=3,
strides=1,
padding="SAME",
data_format="NCHW",
dilation_rate=1,
activation="Relu",
use_bias=True,
weight_initializer=flow.variance_scaling_initializer(
2, 'fan_out', 'random_normal',
data_format="NCHW"),
bias_initializer=flow.zeros_initializer(),
bn=True,
reuse=False,
trainable=True):
name_ = name if reuse == False else name + "_reuse"
weight_shape = (filters, input.shape[1], kernel_size, kernel_size)
weight = flow.get_variable(name + "_weight",
shape=weight_shape,
dtype=input.dtype,
initializer=weight_initializer,
trainable=trainable)
output = flow.nn.conv2d(input,
weight,
strides,
padding,
data_format,
dilation_rate,
name=name_)
if use_bias:
bias = flow.get_variable(name + "_bias",
shape=(filters, ),
dtype=input.dtype,
initializer=bias_initializer,
trainable=trainable)
output = flow.nn.bias_add(output, bias, data_format)
if activation is not None:
if activation == "Relu":
if bn:
# use of_layers(layers) batch_norm
output = layers.batch_norm(output,
name + "_bn",
reuse=reuse)
output = flow.nn.relu(output)
else:
output = flow.nn.relu(output)
else:
raise NotImplementedError
return output
def oneflow_Xmum(
of_input_1: tp.Numpy.Placeholder(shape=input_1.shape,
dtype=value_type["of_type"]),
of_input_2: tp.Numpy.Placeholder(shape=input_2.shape,
dtype=value_type["of_type"]),
) -> tp.Numpy:
with flow.scope.placement(device_type, "0:0"):
v1 = flow.get_variable(
shape=input_1.shape,
dtype=value_type["of_type"],
initializer=flow.zeros_initializer(),
name="x1_var",
)
x1_var = of_input_1 + v1
if not dx_only:
v2 = flow.get_variable(
shape=input_2.shape,
dtype=value_type["of_type"],
initializer=flow.zeros_initializer(),
name="x2_var",
)
x2_var = of_input_2 + v2
else:
x2_var = flow.constant(value=1.5,
shape=of_input_2.shape,
dtype=value_type["of_type"])
flow.watch_diff(x1_var,
assert_prediction_grad) # Only Compare input1 Grad
if compare_type == "maximum":
of_Xmum_out = flow.math.maximum(x1_var, x2_var)
elif compare_type == "minimum":
of_Xmum_out = flow.math.minimum(x1_var, x2_var)
with flow.scope.placement(device_type, "0:0"):
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [1e-3]),
momentum=0).minimize(of_Xmum_out)
return of_Xmum_out
def oneflow_bce_with_logits_loss(
of_input: tp.Numpy.Placeholder(shape=input.shape),
of_target: tp.Numpy.Placeholder(shape=target.shape),
of_weight: tp.Numpy.Placeholder(shape=weight.shape),
of_pos_weight: tp.Numpy.Placeholder(shape=pos_weight.shape),
) -> Dict[str, tp.Numpy]:
with flow.scope.placement(device_type, "0:0"):
v = flow.get_variable(
shape=input.shape,
dtype=flow.float32,
initializer=flow.zeros_initializer(),
name="v",
)
x_var = of_input + v
flow.watch_diff(x_var, assert_prediction_grad)
bceloss = flow.nn.BCEWithLogitsLoss(
x_var,
of_target,
of_weight,
of_pos_weight,
reduction="none",
name="of_mseloss",
)
bceloss_mean = flow.nn.BCEWithLogitsLoss(
x_var,
of_target,
of_weight,
of_pos_weight,
reduction="mean",
name="of_mseloss_reduce_mean",
)
bceloss_sum = flow.nn.BCEWithLogitsLoss(
x_var,
of_target,
of_weight,
of_pos_weight,
reduction="sum",
name="of_mseloss_reduce_sum",
)
# Because our gradient is use "mean" mode to compute
with flow.scope.placement(device_type, "0:0"):
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [1e-3]),
momentum=0).minimize(bceloss_mean)
return {
"of_bce_with_logits_loss": bceloss,
"of_bce_with_logits_loss_mean": bceloss_mean,
"of_bce_with_logits_loss_sum": bceloss_sum,
}
def CompareJob(x: tp.Numpy.Placeholder((5, ), dtype=flow.float32)) -> tp.Numpy:
x += flow.get_variable(
name="v1",
shape=(5, ),
dtype=flow.float,
initializer=flow.zeros_initializer(),
)
loss = 3 * x * x
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler([], [1e-4]),
momentum=0).minimize(loss)
flow.watch_diff(x, global_storage_setter("x2_diff"))
return loss
def load_synthetic(image_size, batch_size):
label = flow.data.decode_random(
shape=(),
dtype=flow.int32,
batch_size=batch_size,
initializer=flow.zeros_initializer(flow.int32),
)
image = flow.data.decode_random(shape=(3, image_size, image_size),
dtype=flow.float,
batch_size=batch_size)
return label, image
def do_gather_nd(x, index):
x_var = flow.get_variable(
"params", shape=(1,), dtype=x_dtype, initializer=flow.zeros_initializer(),
)
x = x + flow.cast_to_current_logical_view(x_var)
y = flow.gather_nd(x, index)
if need_grad:
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [1e-3]), momentum=0
).minimize(y)
if callable(comp_diff_fn):
flow.watch_diff(x, comp_diff_fn)
return y
def conv2d_layer(
name,
input,
filters,
kernel_size=1,
strides=1,
padding="VALID",
data_format="NCHW",
dilation_rate=1,
activation="Relu",
use_bias=True,
weight_initializer=flow.variance_scaling_initializer(
2, 'fan_out', 'random_normal', data_format="NCHW"),
bias_initializer=flow.zeros_initializer(),
bn=True,
):
weight_shape = (filters, input.shape[1], kernel_size, kernel_size)
weight = flow.get_variable(
name + "_weight",
shape=weight_shape,
dtype=input.dtype,
initializer=weight_initializer,
)
output = flow.nn.conv2d(input,
weight,
strides,
padding,
data_format,
dilation_rate,
name=name)
if use_bias:
bias = flow.get_variable(
name + "_bias",
shape=(filters, ),
dtype=input.dtype,
initializer=bias_initializer,
)
output = flow.nn.bias_add(output, bias, data_format)
if activation is not None:
if activation == "Relu":
if bn:
output = _batch_norm(output, name + "_bn")
# flow.watch(output)
output = flow.nn.relu(output)
else:
output = flow.nn.relu(output)
else:
raise NotImplementedError
return output
def loss_con(self, one_hot_labels, feature):
branch = feature
fc_part1 = flow.layers.dense(
flow.reshape(branch, (branch.shape[0], -1)),
units=8, # 车辆颜色类别8
use_bias=True,
kernel_initializer=flow.variance_scaling_initializer(
2, 'fan_in', 'random_normal'),
bias_initializer=flow.zeros_initializer(),
name="fc1",
)
loss_con = flow.nn.softmax_cross_entropy_with_logits(
one_hot_labels, fc_part1, name="softmax_loss1")
return loss_con
def load_synthetic(config):
batch_size = config.train_batch_size
image_size = 112
label = flow.data.decode_random(
shape=(),
dtype=flow.int32,
batch_size=batch_size,
initializer=flow.zeros_initializer(flow.int32),
)
image = flow.data.decode_random(
shape=(image_size, image_size, 3), dtype=flow.float, batch_size=batch_size,
)
return label, image
def FlowJob(
value: oft.Numpy.Placeholder(value.shape),
bias: oft.Numpy.Placeholder(bias.shape),
):
with flow.scope.placement(device_type, "0:0"):
value += flow.get_variable(
name="v1",
shape=(1, ),
dtype=flow.float,
initializer=flow.zeros_initializer(),
)
bias += flow.get_variable(
name="v2",
shape=(1, ),
dtype=flow.float,
initializer=flow.zeros_initializer(),
)
loss = flow.nn.bias_add(value, bias, *flow_args)
flow.losses.add_loss(loss)
flow.watch_diff(value, test_global_storage.Setter("value_diff"))
flow.watch_diff(bias, test_global_storage.Setter("bias_diff"))
return loss
def load_synthetic(args):
total_device_num = args.num_nodes * args.gpu_num_per_node
batch_size = total_device_num * args.batch_size_per_device
label = flow.data.decode_random(
shape=(),
dtype=flow.int32,
batch_size=batch_size,
initializer=flow.zeros_initializer(flow.int32),
)
image = flow.data.decode_random(
shape=(args.image_size, args.image_size, 3), dtype=flow.float, batch_size=batch_size
)
return label, image
def instance_norm(input, name_prefix, trainable=True):
(mean, variance) = flow.nn.moments(input, [2, 3], keepdims=True)
gamma = flow.get_variable(name_prefix + "_gamma",
shape=(1, input.shape[1], 1, 1),
dtype=input.dtype,
initializer=flow.ones_initializer(),
trainable=trainable)
beta = flow.get_variable(name_prefix + "_beta",
shape=(1, input.shape[1], 1, 1),
dtype=input.dtype,
initializer=flow.zeros_initializer(),
trainable=trainable)
epsilon = 1e-3
normalized = (input - mean) / flow.math.sqrt(variance + epsilon)
return gamma * normalized + beta
def oneflow_Xmum(
of_input_1: tp.ListNumpy.Placeholder(shape=data_shape),
of_input_2: tp.ListNumpy.Placeholder(shape=data_shape),
) -> tp.ListNumpy:
with flow.scope.placement(device_type, "0:0"):
v1 = flow.get_variable(
shape=(1, ),
dtype=flow.float32,
initializer=flow.zeros_initializer(),
name="x1_var",
)
v1 = flow.cast_to_current_logical_view(v1)
x1_var = of_input_1 + v1
v2 = flow.get_variable(
shape=(1, ),
dtype=flow.float32,
initializer=flow.zeros_initializer(),
name="x2_var",
)
v2 = flow.cast_to_current_logical_view(v2)
x2_var = of_input_2 + v2
flow.watch_diff(x1_var,
assert_prediction_grad) # Only Compare input1 Grad
if compare_type == "maximum":
of_Xmum_out = flow.math.maximum(x1_var, x2_var)
elif compare_type == "minimum":
of_Xmum_out = flow.math.minimum(x1_var, x2_var)
with flow.scope.placement(device_type, "0:0"):
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [1e-3]),
momentum=0).minimize(of_Xmum_out)
return of_Xmum_out
def op_function(x: tp.Numpy.Placeholder(input.shape, dtype=value_type)):
with flow.scope.placement(device_type, "0:0"):
x += flow.get_variable(
name="input",
shape=input.shape,
dtype=value_type,
initializer=flow.zeros_initializer(),
)
out = flow.reflection_pad2d(x, padding)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0]), momentum=0
).minimize(out)
flow.watch_diff(x, _compare_diff)
return out
def FlowJob(x: oft.Numpy.Placeholder(x.shape)):
with flow.scope.placement(device_type, "0:0"):
x += flow.get_variable(
name="v1",
shape=(1, ),
dtype=flow.float,
initializer=flow.zeros_initializer(),
)
loss = flow_op(x, *flow_args)
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler([],
[0]),
momentum=0).minimize(loss)
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
return loss
def op_function(x: tp.Numpy.Placeholder(input.shape, dtype=flow.float32)):
with flow.scope.placement(device_type, "0:0"):
x += flow.get_variable(
name="input",
shape=input.shape,
dtype=flow.float32,
initializer=flow.zeros_initializer(),
)
y_int32 = flow.replication_pad2d(x, padding)
y_fp32 = flow.cast(y_int32, dtype=flow.float32)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0]), momentum=0
).minimize(y_fp32)
flow.watch_diff(x, _compare_diff)
return y_fp32
def oneflow_kldivloss(
of_input: tp.Numpy.Placeholder(shape=input.shape),
of_target: tp.Numpy.Placeholder(shape=target.shape),
) -> Dict[str, tp.Numpy]:
with flow.scope.placement(device_type, "0:0"):
v = flow.get_variable(
shape=input.shape,
dtype=flow.float32,
initializer=flow.zeros_initializer(),
name="x_var",
)
of_input = of_input + v
flow.watch_diff(of_input, assert_prediction_grad)
of_kldivloss = flow.nn.KLDivLoss(
of_input,
of_target,
log_target=log_target,
reduction="none",
name="kldivloss",
)
of_kldivloss_mean = flow.nn.KLDivLoss(
of_input,
of_target,
log_target=log_target,
reduction="mean",
name="kldivloss_mean",
)
of_kldivloss_sum = flow.nn.KLDivLoss(
of_input,
of_target,
log_target=log_target,
reduction="sum",
name="kldivloss_sum",
)
with flow.scope.placement(device_type, "0:0"):
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [1e-3]),
momentum=0).minimize(of_kldivloss_mean)
return {
"of_kldivloss": of_kldivloss,
"of_kldivloss_mean": of_kldivloss_mean,
"of_kldivloss_sum": of_kldivloss_sum,
}
def ReduceMaxJob(x: oft.Numpy.Placeholder(input_shape, dtype=flow.float)):
with flow.scope.placement(device_type, "0:0"):
x += flow.get_variable(
name="v1",
shape=input_shape,
dtype=flow.float,
initializer=flow.zeros_initializer(),
)
loss = flow.math.reduce_max(x, axis=axis, keepdims=keepdims)
loss = flow.identity(loss)
flow.losses.add_loss(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return loss
def PolyValJob(x: tp.Numpy.Placeholder(shape=in_shape)):
with flow.scope.placement(device_type, "0:0"):
x += flow.get_variable(
name="x",
shape=in_shape,
dtype=flow_data_type,
initializer=flow.zeros_initializer(),
trainable=True,
)
flow.watch_diff(x, assert_prediction_grad)
out = flow.math.polyval(coeffs, x)
with flow.scope.placement(device_type, "0:0"):
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [1e-4]),
momentum=0).minimize(out)
return out
def _build_dense(self, x, unit, name="dense_"):
"""
Build Dense Layer
:param x:
:return:
"""
self.init_range = 0.2
self.init = flow.truncated_normal_initializer(self.init_range)
self.reg = flow.regularizers.l2(0.01)
return flow.layers.dense(x,
units=unit,
kernel_initializer=self.init,
kernel_regularizer=self.reg,
bias_initializer=flow.zeros_initializer(),
bias_regularizer=self.reg,
name=name+"w")
def cast_to_static_shape_fn(x: flow.typing.ListNumpy.Placeholder(
shape=shape, dtype=dtype)) -> flow.typing.ListNumpy:
x_var = flow.get_variable(
name="x_var",
shape=(1, ),
dtype=flow.float32,
initializer=flow.zeros_initializer(),
)
x = x + flow.cast(x_var, dtype=dtype)
y = flow.cast_to_static_shape(x)
test_case.assertFalse(y.is_dynamic)
if require_grad:
flow.watch_diff(x, compare_diff_fn)
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [1e-3]),
momentum=0).minimize(y)
return y
def DynamicConvJob(x: oft.ListNumpy.Placeholder((10, 3, 100, 100))):
with flow.scope.placement(device_type, "0:0"):
x_var = flow.get_variable(
name="v1",
shape=(1, ),
dtype=flow.float,
initializer=flow.zeros_initializer(),
)
x_var = flow.cast_to_current_logical_view(x_var)
x += x_var
if data_format == "NCHW":
weight_shape = (filters, x_shape[1] // groups, kernel_size,
kernel_size)
else:
weight_shape = (filters, kernel_size, kernel_size,
x_shape[3] // groups)
weight = flow.get_variable(
"conv-weight",
shape=weight_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=0,
maxval=100),
)
weight = flow.cast_to_current_logical_view(weight)
loss = flow.nn.conv2d(
x,
weight,
strides=[stride, stride],
padding=of_padding,
data_format=data_format,
dilations=[1, 1],
groups=groups,
)
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [1e-4]),
momentum=0).minimize(loss)
flow.watch(x, global_storage_setter("x"))
flow.watch_diff(x, global_storage_setter("x_diff"))
flow.watch(weight, global_storage_setter("weight"))
flow.watch_diff(weight, global_storage_setter("weight_diff"))
flow.watch(loss, global_storage_setter("loss"))
flow.watch_diff(loss, global_storage_setter("loss_diff"))
return loss
def _batch_norm(inputs, momentum, epsilon, name, training=True):
return flow.layers.batch_normalization(
inputs=inputs,
axis=-1,
momentum=momentum,
epsilon=epsilon,
center=True,
scale=True,
# beta_initializer=flow.zeros_initializer(),
# gamma_initializer=flow.ones_initializer(),
# beta_regularizer=flow.zeros_initializer(),
# gamma_regularizer=flow.ones_initializer(),
moving_mean_initializer=flow.zeros_initializer(),
moving_variance_initializer=flow.ones_initializer(),
trainable=True,
training=training,
name=name)
def oneflow_ones() -> tp.Numpy:
with flow.scope.placement(device_type, "0:0"):
v = flow.get_variable(
shape=np_out_ones.shape,
dtype=flow.float32,
initializer=flow.zeros_initializer(),
name="x_var",
)
of_ones = flow.ones(shape=input_shape, dtype=flow.float32)
of_out = of_ones + v
with flow.scope.placement(device_type, "0:0"):
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [1e-3]),
momentum=0).minimize(of_out)
return of_ones
