def _test_linear_graph_save_load_cpu(test_case):
_test_linear_graph_save_load_global(test_case, flow.device("cpu"))
def main(args):
train_aug = Compose([
HorizontalFlip(p=0.5),
Rotate(15),
])
train_data_loader = face_seg(args.dataset_path,
args.batch_size,
augmentation=train_aug)
print(len(train_data_loader))
# oneflow init
start_t = time.time()
linknet34_module = LinkNet34(pretrained=True,
pretrained_model_path=args.model_path)
end_t = time.time()
linknet34_module.to(flow.device("cuda"))
of_sgd = flow.optim.SGD(linknet34_module.parameters(),
lr=args.learning_rate,
momentum=args.mom)
cosine = flow.optim.lr_scheduler.CosineAnnealingLR(of_sgd, 2)
of_losses = []
criterion = LossBinary(jaccard_weight=1)
for epoch in range(args.epochs):
linknet34_module.train()
train_data_loader.shuffle_data()
epoch_loss = 0
for b in range(len(train_data_loader)):
of_sgd.zero_grad()
image_nd, label_nd = train_data_loader[b]
# oneflow train
start_t = time.time()
image = flow.Tensor(image_nd)
label = flow.Tensor(label_nd,
dtype=flow.float32,
requires_grad=False)
image = image.to(flow.device("cuda"))
label = label.to(flow.device("cuda"))
logits = linknet34_module(image)
mask_probs_flat = flow.reshape(logits, shape=[-1])
true_masks_flat = flow.reshape(label, shape=[-1])
loss = criterion(mask_probs_flat, true_masks_flat)
epoch_loss += loss.numpy()
loss.backward()
of_sgd.step()
cosine.step()
end_t = time.time()
l = loss.numpy()
of_losses.append(l)
sys.stdout.write(
f"\rEpoch: {epoch} ---- Loss: {numpy.round(epoch_loss / (b + 1), 4)} ----- num: {b} "
)
sys.stdout.flush()
flow.save(linknet34_module.state_dict(), args.save_model_name)
writer = open("of_losses.txt", "w")
for o in of_losses:
writer.write("%f\n" % o)
writer.close()
def _test_instancenorm3d(test_case, device):
input_arr = np.array(
[
[
[
[
[1.04569761, 0.22863248, 1.42439335, 1.62249689],
[-0.80578825, -0.27276461, 1.04556507, 0.56864134],
[-1.24085419, -1.23960097, 0.33451416, -1.84820402],
],
[
[-1.511261, 1.06157517, -0.26715858, -1.32888141],
[1.17976881, -0.07931171, 0.33910684, -1.93458573],
[-1.72659647, 0.79049652, 0.39102785, -1.16264882],
],
],
[
[
[0.30067973, -1.2912226, -0.61508225, 0.56454001],
[0.87074187, -1.69257376, 0.36119148, -0.31014289],
[0.20776964, 1.26195488, -1.37122193, -0.17945234],
],
[
[-0.31112407, -0.80682631, 0.8233194, 0.6384975],
[0.57617527, 0.45505028, 1.68286151, -1.09590744],
[-1.18127546, -1.07529277, 0.52779943, 1.21755926],
],
],
],
[
[
[
[-0.12832351, 1.05625455, -0.23253249, -0.64747611],
[-0.00738123, -1.41390089, -1.92664144, -0.21427625],
[-0.94631219, -0.86493989, 0.21026905, 0.24989732],
],
[
[1.3859182, 1.72002107, 0.50091892, 1.04198896],
[0.71694594, 1.66417023, -1.63030052, 0.77182641],
[0.71545083, 1.96458366, -1.99031931, 1.3196714],
],
],
[
[
[1.80091702, 0.02834973, 0.82259214, -1.05597501],
[-0.58212207, 0.44205949, -0.14740003, -0.994508],
[1.14678114, -0.39196097, 1.2554798, -0.41829324],
],
[
[-1.0153903, -0.25755713, -1.81756333, -1.06781159],
[1.79680841, -1.9107133, -0.64325796, -1.94640775],
[1.30671156, 1.20445339, -1.26262901, -0.79494188],
],
],
],
],
dtype=np.float32,
)
output_arr = np.array(
[
[
[
[
[1.067, 0.3324, 1.4075, 1.5856],
[-0.5976, -0.1184, 1.0669, 0.6381],
[-0.9888, -0.9877, 0.4276, -1.5349],
],
[
[-1.2319, 1.0813, -0.1134, -1.068],
[1.1876, 0.0555, 0.4317, -1.6126],
[-1.4256, 0.8376, 0.4784, -0.9185],
],
],
[
[
[0.3447, -1.3751, -0.6446, 0.6298],
[0.9606, -1.8087, 0.4101, -0.3152],
[0.2444, 1.3833, -1.4615, -0.174],
],
[
[-0.3162, -0.8518, 0.9094, 0.7097],
[0.6424, 0.5115, 1.838, -1.1641],
[-1.2563, -1.1418, 0.5901, 1.3353],
],
],
],
[
[
[
[-0.2327, 0.8016, -0.3236, -0.6859],
[-0.1271, -1.3551, -1.8028, -0.3077],
[-0.9469, -0.8758, 0.063, 0.0976],
],
[
[1.0895, 1.3812, 0.3167, 0.7892],
[0.5054, 1.3324, -1.5441, 0.5533],
[0.5041, 1.5947, -1.8584, 1.0316],
],
],
[
[
[1.7507, 0.1901, 0.8894, -0.7645],
[-0.3473, 0.5544, 0.0354, -0.7104],
[1.1748, -0.1799, 1.2705, -0.2031],
],
[
[-0.7288, -0.0616, -1.435, -0.7749],
[1.7471, -1.517, -0.4012, -1.5485],
[1.3156, 1.2256, -0.9465, -0.5347],
],
],
],
],
dtype=np.float32,
)
m = flow.nn.InstanceNorm3d(num_features=2, eps=1e-05, momentum=0.1).to(
device=flow.device(device)
)
x = flow.tensor(input_arr, dtype=flow.float32, device=flow.device(device))
y = m(x)
test_case.assertTrue(np.allclose(y.numpy(), output_arr, 0.0001, 0.0001))
m.eval()
y = m(x)
test_case.assertTrue(np.allclose(y.numpy(), output_arr, 0.0001, 0.0001))
def _test_randn(test_case, device, shape):
y1 = flow.randn(*shape, device=flow.device(device))
y2 = flow.randn(*shape, device=flow.device(device))
test_case.assertTrue(
not np.allclose(y1.numpy(), y2.numpy(), atol=1e-4, rtol=1e-4))
test_case.assertTrue(shape == y1.shape)
def _test_randn_tuple_shape(test_case, device, shape):
y1 = flow.randn(shape, device=flow.device(device))
y2 = flow.randn(shape, device=flow.device(device))
test_case.assertTrue(not np.array_equal(y1.numpy(), y2.numpy()))
test_case.assertTrue(shape == y1.shape)
def new_ones_op(x,
size=None,
dtype=None,
device=None,
placement=None,
sbp=None,
requires_grad=False):
if isinstance(device, str):
device = flow.device(device)
if size != None:
size = _single(size)
new_size = size
new_dtype = dtype
new_device = device
new_placement = placement
new_sbp = sbp
new_requires_grad = requires_grad
if size is None:
new_size = x.shape
if dtype is None:
new_dtype = x.dtype
if device is None:
new_device = x.device if x.is_local else None
if placement is None:
new_placement = x.placement if x.is_global else None
if sbp is None:
new_sbp = x.sbp if x.is_global else None
if new_placement is not None:
assert device is None
assert new_sbp is not None
assert isinstance(
new_size,
(int, tuple, flow.Size)), f"size parameter not correct, please check!"
assert isinstance(
new_dtype, flow.dtype), f"dtype parameter not correct, please check!"
if new_placement is not None:
assert isinstance(
new_placement,
flow.placement), f"device parameter not correct, please check!"
assert isinstance(
new_sbp,
flow.sbp.sbp), f"device parameter not correct, please check!"
else:
assert isinstance(
new_device,
(str, flow.device)), f"device parameter not correct, please check!"
assert isinstance(
new_requires_grad,
bool), f"requires_grad parameter not correct, please check!"
if placement is not None:
res = flow._C.global_constant(new_size,
1.0,
dtype=new_dtype,
placement=placement,
sbp=sbp)
else:
res = flow._C.constant(new_size,
1.0,
dtype=new_dtype,
device=new_device)
res.requires_grad = new_requires_grad
return res
def test_non_default_device(test_case):
x = flow.randn(2, 3, device="cuda:1")
test_case.assertEqual(x.device, flow.device("cuda:1"))
def test_tensor_to_cast_h2d(test_case):
input = flow.Tensor(np.random.randn(2, 3, 4, 5))
output = input.to(device=flow.device("cuda"), dtype=flow.int)
test_case.assertEqual(output.dtype, flow.int)
test_case.assertEqual(output.device, flow.device("cuda"))
def _test_interpolate_trilinear_3d_align_corners(test_case, device):
input = flow.tensor(
np.arange(1, 9).reshape((1, 1, 2, 2, 2)),
device=flow.device(device),
dtype=flow.float32,
requires_grad=True,
)
of_out = flow.nn.functional.interpolate(input,
scale_factor=2.0,
mode="trilinear",
align_corners=True)
np_out = np.array([[[
[
[1.0, 1.3333332538604736, 1.6666667461395264, 2.0],
[
1.6666666269302368,
2.0,
2.3333334922790527,
2.6666665077209473,
],
[
2.3333332538604736,
2.6666665077209473,
3.0,
3.3333334922790527,
],
[3.0, 3.3333332538604736, 3.6666667461395264, 4.0],
],
[
[
2.3333334922790527,
2.6666665077209473,
3.0,
3.3333332538604736,
],
[3.0, 3.3333330154418945, 3.6666665077209473, 4.0],
[
3.6666665077209473,
4.0,
4.333333492279053,
4.6666669845581055,
],
[4.333333492279053, 4.666666030883789, 5.0, 5.3333330154418945],
],
[
[3.6666667461395264, 4.0, 4.333333492279053, 4.666666507720947],
[4.333333492279053, 4.666666507720947, 5.0, 5.3333330154418945],
[5.0, 5.333333492279053, 5.6666669845581055, 6.0],
[
5.6666669845581055,
6.0,
6.333333492279053,
6.6666669845581055,
],
],
[
[5.0, 5.3333330154418945, 5.666666507720947, 6.0],
[
5.666666507720947,
5.999999523162842,
6.3333330154418945,
6.666666507720947,
],
[6.333333492279053, 6.666666030883789, 7.0, 7.333333492279053],
[7.0, 7.3333330154418945, 7.6666669845581055, 8.0],
],
]]])
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 0.0001, 0.0001))
of_out = of_out.sum()
of_out.backward()
np_grad = [[[
[[7.999999523162842, 8.0], [7.999999523162842, 8.0]],
[[8.0, 8.0], [8.0, 8.0]],
]]]
test_case.assertTrue(np.allclose(input.grad.numpy(), np_grad, 1e-05,
1e-05))
def forward(self, x):
assert len(x.shape) > len(
self.normalized_shape
), "Input tensor dim must greater than normalized dim!"
self.begin_norm_axis = len(x.shape) - len(self.normalized_shape)
self.begin_params_axis = len(x.shape) - len(self.normalized_shape)
if x.device == flow.device("cpu"):
reduce_axis = []
for dim in range(len(x.shape)):
if dim >= self.begin_norm_axis:
reduce_axis.append(dim)
mean = x.mean(dim=reduce_axis, keepdim=True)
variance = x.var(dim=reduce_axis, keepdim=True)
axis = self.begin_norm_axis
params_shape = x.shape[self.begin_params_axis:]
weight = self.weight
bias = self.bias
if len(mean.shape) == 1:
nd_params_shape = [1] * len(x.shape)
nd_params_shape[self.begin_norm_axis] = params_shape[0]
mean = mean.reshape(shape=nd_params_shape)
variance = variance.reshape(shape=nd_params_shape)
if self.weight and params_shape[0] == self.weight.nelement():
weight = self.weight.reshape(shape=nd_params_shape)
if self.bias and params_shape[0] == self.bias.nelement():
bias = self.bias.reshape(shape=nd_params_shape)
elif len(mean.shape) == len(x.shape):
pass
else:
raise ValueError(
"shape of mean and variance should be 1D or has number of axes and x's"
)
variance += self.epsilon
normalized = (x - mean) * variance.rsqrt()
if self.weight:
normalized = normalized * weight
if self.bias:
normalized = normalized + bias
affined = normalized
nd_params_shape = [1] * (len(x.shape) -
len(params_shape)) + list(params_shape)
if self.elementwise_affine:
affined = affined * self.weight
affined = affined + self.bias
return affined
else:
if self.elementwise_affine:
res = self._op(
x,
self.weight,
self.bias,
center=True,
scale=True,
begin_norm_axis=self.begin_norm_axis,
begin_params_axis=self.begin_params_axis,
epsilon=self.epsilon,
)[0]
else:
res = self._op2(
x,
center=False,
scale=False,
begin_norm_axis=self.begin_norm_axis,
begin_params_axis=self.begin_params_axis,
epsilon=self.epsilon,
)[0]
return res
def test_tensor_to_h2h(test_case):
input = flow.Tensor(np.random.randn(2, 3, 4, 5))
output = input.to(device=flow.device("cpu"))
test_case.assertEqual(output.device, flow.device("cpu"))
test_case.assertTrue(
np.allclose(input.numpy(), output.numpy(), rtol=1e-04, atol=1e-04))
def _test_embedding_scale_by_freq(test_case, device):
weight = np.array(
[
[0.68258786, 0.6957856, 1.1829041],
[1.0154, -1.0616943, 0.50303376],
[0.29679507, 0.65562993, 1.0424724],
[-0.42980736, -0.35347632, -0.15600166],
[0.6763601, -0.24286619, -2.0873115],
[-0.13371214, -0.5589277, 1.9173933],
[0.08762296, 1.0264007, -0.67938024],
[0.32019204, -0.26137325, -1.3534237],
[-1.1555519, -0.67776406, 0.27372134],
[1.0615997, -0.59715784, 1.9855849],
],
dtype=np.float32,
)
output = np.array(
[
[
[1.0154, -1.0616943, 0.50303376],
[0.29679507, 0.65562993, 1.0424724],
[0.6763601, -0.24286619, -2.0873115],
[-0.13371214, -0.5589277, 1.9173933],
],
[
[0.6763601, -0.24286619, -2.0873115],
[-0.42980736, -0.35347632, -0.15600166],
[0.29679507, 0.65562993, 1.0424724],
[1.0615997, -0.59715784, 1.9855849],
],
],
dtype=np.float32,
)
indices = flow.tensor(
[[1, 2, 4, 5], [4, 3, 2, 9]],
dtype=flow.int,
device=flow.device(device),
requires_grad=False,
)
m = flow.nn.Embedding(10,
3,
scale_grad_by_freq=True,
_weight=flow.Tensor(weight))
m = m.to(device)
y = m(indices)
test_case.assertTrue(np.allclose(y.numpy(), output, 1e-05, 1e-05))
y = y.sum()
y.backward()
weight_grad_np = [
[0.0, 0.0, 0.0],
[1.0, 1.0, 1.0],
[1.0, 1.0, 1.0],
[1.0, 1.0, 1.0],
[1.0, 1.0, 1.0],
[1.0, 1.0, 1.0],
[0.0, 0.0, 0.0],
[0.0, 0.0, 0.0],
[0.0, 0.0, 0.0],
[1.0, 1.0, 1.0],
]
test_case.assertTrue(
np.allclose(m.weight.grad.numpy(), weight_grad_np, 1e-05, 1e-05))
def test_scalar_train_graph_cpu(test_case):
_test_scalar_train_graph(test_case, flow.device("cpu"))
def test_scalar_train_graph_gpu(test_case):
_test_scalar_train_graph(test_case, flow.device("cuda"))
def __init__(self):
super().__init__()
self.para0 = flow.nn.Parameter(
flow.Tensor(init_value, device=flow.device(device)))
def _test_interpolate_area_3d(test_case, device):
input = flow.tensor(
np.array([[[
[
[
-1.077571799600885,
-0.7804538890365837,
-1.2627538752119443,
0.9993507145120477,
],
[
2.0222532489157516,
1.103451377699465,
-0.4377324754879578,
1.890491810587517,
],
[
-0.5593861899064654,
-0.4949520241526519,
-0.18536721363519787,
-0.6098969866775772,
],
[
-1.6536215260171816,
-1.0392583540436786,
0.3686776597613967,
-0.5356882834951805,
],
],
[
[
-1.2617900664449953,
-1.4390921091631532,
0.20654399652431357,
0.8186472101906713,
],
[
-0.3033378863400014,
-0.8173269764076293,
-0.3767515097625614,
-0.11021655039337777,
],
[
-0.22977043608192885,
1.2717196366649905,
-0.4790851297878291,
-1.4495369404727856,
],
[
-1.2802093286977783,
-0.11184514806663474,
1.7022167087210984,
-1.7354837287725355,
],
],
[
[
2.4706497991773606,
-0.6549702631973298,
-0.9318107079571676,
1.4652904271682428,
],
[
1.1419864234341397,
1.389909081086008,
0.9657841900525568,
-0.8563114264976619,
],
[
0.19515087084250754,
-0.37808457398571094,
0.2938625398496183,
0.9279930510353327,
],
[
-0.9374118277994007,
0.3341831730452431,
-0.2792542765303833,
0.38029090707066726,
],
],
[
[
0.5918686659736041,
-0.7870631089938902,
-0.9534344874245392,
0.31341612954718795,
],
[
0.7509029444145228,
-0.9299288398562323,
-0.7343054052782476,
-0.8806481590696694,
],
[
-0.4707853016353985,
0.12253641652645629,
0.5088022039832846,
0.520391789327562,
],
[
-0.0861300651163632,
0.30291348404866386,
-0.6268565873680123,
-0.27469204305759976,
],
],
]]]),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
of_out_1 = flow.nn.functional.interpolate(input,
size=(2, 2, 2),
mode="area")
of_out_2 = flow.nn.functional.interpolate(input,
scale_factor=0.5,
mode="area")
np_out = np.array([[[
[
[-0.3192335125472539, 0.2159474151198386],
[-0.5121654212876662, -0.3655204892948264],
],
[
[0.4966693377547728, -0.2015024299324123],
[-0.11470347800925032, 0.18131719803880864],
],
]]])
test_case.assertTrue(np.allclose(of_out_1.numpy(), np_out, 1e-05, 1e-05))
test_case.assertTrue(np.allclose(of_out_2.numpy(), np_out, 1e-05, 1e-05))
of_out_1 = of_out_1.sum()
of_out_1.backward()
np_grad = np.array([[[
[
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
],
[
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
],
[
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
],
[
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
[0.125, 0.125, 0.125, 0.125],
],
]]])
test_case.assertTrue(np.allclose(input.grad.numpy(), np_grad, 1e-05,
1e-05))
def _test_inplace_add(test_case, shape, device):
np_x = np.random.randn(*shape)
of_x = flow.tensor(
np_x, dtype=flow.float32, device=flow.device(device), requires_grad=True
)
of_x_inplace = of_x + 1
id_old = id(of_x_inplace)
of_x_inplace.add_(5)
test_case.assertEqual(id_old, id(of_x_inplace))
np_out = np_x + 1 + 5
test_case.assertTrue(np.allclose(of_x_inplace.numpy(), np_out, 1e-05, 1e-05))
of_x_inplace = of_x_inplace.sum()
of_x_inplace.backward()
test_case.assertTrue(np.allclose(of_x.grad.numpy(), np.ones(shape), 1e-05, 1e-05))
of_x = flow.tensor(
np_x, dtype=flow.float32, device=flow.device(device), requires_grad=True
)
of_y = flow.tensor(
np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device),
requires_grad=False,
)
of_x_inplace = of_x + 1
id_old = id(of_x_inplace)
of_x_inplace.add_(of_y)
test_case.assertEqual(id_old, id(of_x_inplace))
np_out = np_x + 1 + of_y.numpy()
test_case.assertTrue(np.allclose(of_x_inplace.numpy(), np_out, 1e-05, 1e-05))
of_x_inplace = of_x_inplace.sum()
of_x_inplace.backward()
test_case.assertTrue(np.allclose(of_x.grad.numpy(), np.ones(shape), 1e-05, 1e-05))
of_x = flow.tensor(
np_x, dtype=flow.float32, device=flow.device(device), requires_grad=True
)
of_y = flow.tensor(
np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device),
requires_grad=False,
)
of_x_inplace = of_x + 1
id_old = id(of_x_inplace)
of_x_inplace += of_y
test_case.assertEqual(id_old, id(of_x_inplace))
np_out = np_x + 1 + of_y.numpy()
test_case.assertTrue(np.allclose(of_x_inplace.numpy(), np_out, 1e-05, 1e-05))
of_x_inplace = of_x_inplace.sum()
of_x_inplace.backward()
test_case.assertTrue(np.allclose(of_x.grad.numpy(), np.ones(shape), 1e-05, 1e-05))
of_x = flow.tensor(
np_x, dtype=flow.float32, device=flow.device(device), requires_grad=True
)
of_y = flow.tensor(
np.array([5.0]),
dtype=flow.float32,
device=flow.device(device),
requires_grad=False,
)
of_x_inplace = of_x + 1
id_old = id(of_x_inplace)
of_x_inplace.add_(of_y)
test_case.assertEqual(id_old, id(of_x_inplace))
np_out = np_x + 6
test_case.assertTrue(np.allclose(of_x_inplace.numpy(), np_out, 1e-05, 1e-05))
of_x_inplace = of_x_inplace.sum()
of_x_inplace.backward()
test_case.assertTrue(np.allclose(of_x.grad.numpy(), np.ones(shape), 1e-05, 1e-05))
of_x = flow.tensor(
np_x, dtype=flow.float32, device=flow.device(device), requires_grad=True
)
np_y = np.random.randn(*shape[:-1], 1)
of_y = flow.tensor(
np_y, dtype=flow.float32, device=flow.device(device), requires_grad=False
)
of_x_inplace = of_x + 1
id_old = id(of_x_inplace)
of_x_inplace.add_(of_y)
test_case.assertEqual(id_old, id(of_x_inplace))
np_out = np_x + 1 + np_y
test_case.assertTrue(np.allclose(of_x_inplace.numpy(), np_out, 1e-05, 1e-05))
of_x_inplace = of_x_inplace.sum()
of_x_inplace.backward()
test_case.assertTrue(np.allclose(of_x.grad.numpy(), np.ones(shape), 1e-05, 1e-05))
def _test_tensor_scalar_logical_and(test_case, shape, scalar, dtype, device):
np_input = np.random.randint(3, size=shape)
input = flow.tensor(np_input, dtype=dtype, device=flow.device(device))
of_out = input.logical_and(scalar)
np_out = np.logical_and(np_input, scalar)
test_case.assertTrue(np.array_equal(of_out.numpy(), np_out))
def new_zeros_op(x,
size=None,
dtype=None,
device=None,
placement=None,
sbp=None,
requires_grad=False):
if isinstance(device, str):
device = flow.device(device)
if size is None or len(size) == 0:
new_size = x.shape
else:
new_size = _handle_size_arg(size)
new_dtype = dtype
new_device = device
new_placement = placement
new_sbp = sbp
new_requires_grad = requires_grad
if dtype is None:
new_dtype = x.dtype
if device is None:
new_device = x.device if x.is_local else None
if placement is None:
new_placement = x.placement if x.is_global else None
if sbp is None:
new_sbp = x.sbp if x.is_global else None
if new_placement is not None:
assert (
device is None
), "argument 'device' must be None when argument 'placement' exist"
assert (
new_sbp is not None
), "argument 'sbp' must not be None when argument 'placement' exist"
assert isinstance(
new_size, (int, tuple, list, flow.Size)
), f"argument 'size' must be tuple of ints, not %s" % (type(new_size))
assert isinstance(
new_dtype,
flow.dtype), f"argument 'dtype' must be flow.dtype, not %s" % (
type(new_dtype))
if new_placement is not None:
assert isinstance(
new_placement, flow.placement
), f"argument 'placement' must be flow.placement, not %s" % (
type(new_placement))
assert isinstance(
new_sbp, (flow.sbp.sbp, tuple)
), f"argument 'sbp' must be flow.sbp.sbp, not %s" % (type(new_sbp))
else:
assert isinstance(new_device, (
str,
flow.device)), f"argument 'device' must be flow.device, not %s" % (
type(new_device))
assert isinstance(
new_requires_grad,
bool), f"argument 'requires_grad' must be bool, not %s" % (
type(new_requires_grad))
if new_placement is not None:
res = flow._C.global_constant(new_size,
0.0,
dtype=new_dtype,
placement=new_placement,
sbp=new_sbp)
else:
res = flow._C.constant(new_size,
0.0,
dtype=new_dtype,
device=new_device)
res.requires_grad = new_requires_grad
return res
def test_mock_device(test_case):
device = flow.device("mock")
test_case.assertEqual(device.type, "mock")
def test_with_generator(test_case):
gen = flow.Generator("cuda")
x = flow.randn(2, 3, device="cuda", generator=gen)
test_case.assertEqual(x.device,
flow.device(f"cuda:{flow.env.get_rank()}"))
def _test_div_impl(test_case, shape, device):
x = flow.tensor(np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device))
y = flow.tensor(np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device))
of_out = flow.div(x, y)
np_out = np.divide(x.numpy(), y.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 0.0001, 0.0001))
x = 5
y = flow.tensor(np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device))
of_out = flow.div(x, y)
np_out = np.divide(x, y.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 0.0001, 0.0001))
x = flow.tensor(np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device))
y = 5
of_out = flow.div(x, y)
np_out = np.divide(x.numpy(), y)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 0.0001, 0.0001))
x = flow.tensor(np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device))
y = flow.tensor(np.random.randn(1, 1),
dtype=flow.float32,
device=flow.device(device))
of_out = flow.div(x, y)
np_out = np.divide(x.numpy(), y.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 0.0001, 0.0001))
x = flow.tensor(np.array([5.0]),
dtype=flow.float32,
device=flow.device(device))
y = flow.tensor(np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device))
of_out = flow.div(x, y)
np_out = np.divide(x.numpy(), y.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 0.0001, 0.0001))
x = flow.tensor(
np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
y = flow.tensor(
np.array([5.0]),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
of_out = flow.div(x, y)
np_out = np.divide(x.numpy(), y.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 0.0001, 0.0001))
of_out = of_out.sum()
of_out.backward()
np_grad_x = np.full(shape, 0.2)
test_case.assertTrue(np.allclose(x.grad.numpy(), np_grad_x, 0.0001,
0.0001))
def _test_backward(test_case, device, shape):
x = flow.randn(*shape, device=flow.device(device), requires_grad=True)
y = x.sum()
y.backward()
test_case.assertTrue(
np.allclose(np.ones(shape), x.grad.numpy(), atol=1e-4, rtol=1e-4))
def test_non_default_device(test_case):
x = flow.tensor([2, 3], dtype=flow.float, device="cuda:1")
y = flow._C.dropout(x)
test_case.assertEqual(y.device, flow.device("cuda:1"))
def _test_randn_with_flow_size(test_case, device, shape):
y1 = flow.randn(flow.Size(shape), device=flow.device(device))
y2 = flow.randn(flow.Size(shape), device=flow.device(device))
test_case.assertTrue(not np.array_equal(y1.numpy(), y2.numpy()))
test_case.assertTrue(shape == y1.shape)
def _test_cast_int2float(test_case, device, shape):
np_arr = np.random.randn(*shape).astype(np.int8)
input = flow.tensor(np_arr, dtype=flow.int8, device=flow.device(device))
output = flow.cast(input, flow.float32)
np_out = np_arr.astype(np.float32)
test_case.assertTrue(np.array_equal(output.numpy(), np_out))
def _test_clamp_integral(test_case, shape, device):
input = flow.tensor(np.random.randint(3, 10, shape),
device=flow.device(device))
of_out = flow.clamp(input, 1, 5)
np_out = np.clip(input.numpy(), 1, 5)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-05, 1e-05))
def compare_with_numpy_adagrad(
test_case,
device,
x_shape,
learning_rate,
train_iters,
lr_decay,
weight_decay,
initial_accumulator_value,
eps,
):
random_grad_seq = []
for _ in range(train_iters):
random_grad_seq.append(
np.random.uniform(size=x_shape).astype(np.float32))
init_value = np.random.uniform(size=x_shape).astype(np.float32)
class CustomModule(flow.nn.Module):
def __init__(self):
super().__init__()
self.para0 = flow.nn.Parameter(
flow.Tensor(init_value, device=flow.device(device)))
def forward(self, mask):
return self.para0 * mask
simp_module = CustomModule()
simp_module.to(device)
simp_module.train()
adam0 = flow.optim.Adagrad(
[{
"params": simp_module.parameters(),
"lr": learning_rate,
"eps": eps,
"weight_decay": weight_decay,
}],
lr_decay=lr_decay,
initial_accumulator_value=initial_accumulator_value,
)
class CustomAdagradGraph(flow.nn.Graph):
def __init__(self):
super().__init__()
self.m = simp_module
self.add_optimizer(adam0)
def build(self, mask_tensor):
loss = flow.sum(self.m(mask_tensor))
loss.backward()
return loss
of_res_list = []
adagrad_graph = CustomAdagradGraph()
for i in range(train_iters):
mask_tensor = flow.tensor(random_grad_seq[i],
requires_grad=False,
device=flow.device(device))
adagrad_x = adagrad_graph(mask_tensor)
of_res_list.append(copy.copy(simp_module.para0.numpy()))
np_res_list = []
def train_by_numpy():
x = init_value
st = np.ones_like(x) * initial_accumulator_value
def train_one_iter(iter, grad):
grad = grad + weight_decay * x
lr = learning_rate / (1 + (iter - 1) * lr_decay)
s = st + grad * grad
param = x - lr / (np.sqrt(s) + eps) * grad
return (param, s)
for i in range(1, train_iters + 1):
(x, st) = train_one_iter(i, random_grad_seq[i - 1])
np_res_list.append(x)
return x
train_by_numpy()
test_case.assertTrue(
np.allclose(of_res_list, np_res_list, rtol=0.001, atol=0.001))
def _test_instancenorm3d_backward(test_case, device):
input_arr = np.array(
[
[
[
[
[1.04569761, 0.22863248, 1.42439335, 1.62249689],
[-0.80578825, -0.27276461, 1.04556507, 0.56864134],
[-1.24085419, -1.23960097, 0.33451416, -1.84820402],
],
[
[-1.511261, 1.06157517, -0.26715858, -1.32888141],
[1.17976881, -0.07931171, 0.33910684, -1.93458573],
[-1.72659647, 0.79049652, 0.39102785, -1.16264882],
],
],
[
[
[0.30067973, -1.2912226, -0.61508225, 0.56454001],
[0.87074187, -1.69257376, 0.36119148, -0.31014289],
[0.20776964, 1.26195488, -1.37122193, -0.17945234],
],
[
[-0.31112407, -0.80682631, 0.8233194, 0.6384975],
[0.57617527, 0.45505028, 1.68286151, -1.09590744],
[-1.18127546, -1.07529277, 0.52779943, 1.21755926],
],
],
],
[
[
[
[-0.12832351, 1.05625455, -0.23253249, -0.64747611],
[-0.00738123, -1.41390089, -1.92664144, -0.21427625],
[-0.94631219, -0.86493989, 0.21026905, 0.24989732],
],
[
[1.3859182, 1.72002107, 0.50091892, 1.04198896],
[0.71694594, 1.66417023, -1.63030052, 0.77182641],
[0.71545083, 1.96458366, -1.99031931, 1.3196714],
],
],
[
[
[1.80091702, 0.02834973, 0.82259214, -1.05597501],
[-0.58212207, 0.44205949, -0.14740003, -0.994508],
[1.14678114, -0.39196097, 1.2554798, -0.41829324],
],
[
[-1.0153903, -0.25755713, -1.81756333, -1.06781159],
[1.79680841, -1.9107133, -0.64325796, -1.94640775],
[1.30671156, 1.20445339, -1.26262901, -0.79494188],
],
],
],
],
dtype=np.float32,
)
m = flow.nn.InstanceNorm3d(num_features=2, eps=1e-05, momentum=0.1).to(
device=flow.device(device)
)
x = flow.tensor(input_arr, device=flow.device(device), requires_grad=True)
y = m(x)
z = y.sum()
z.backward()
test_case.assertTrue(
np.allclose(x.grad.numpy(), np.zeros(shape=input_arr.shape), 1e-05, 1e-05)
)
def test_linear_graph_save_load_gpu(test_case):
_test_linear_graph_save_load_global(test_case, flow.device("cuda"))