def test_layernorm_gpu(test_case):
input_arr = np.array(
[
[
[[-0.16046895, -1.03667831], [-0.34974465, 0.26505867]],
[[-1.24111986, -0.53806001], [1.72426331, 0.43572459]],
],
[
[[-0.77390957, -0.42610624], [0.16398858, -1.35760343]],
[[1.07541728, 0.11008703], [0.26361224, -0.48663723]],
],
],
dtype=np.float32,
)
output = np.array(
[
[
[[-0.0544118, -1.0509688], [-0.2696846, 0.4295622]],
[[-1.2834904, -0.4838651], [2.0891891, 0.6236691]],
],
[
[[-0.8555527, -0.3554582], [0.4930190, -1.6948260]],
[[1.8035311, 0.4155158], [0.6362644, -0.4424936]],
],
],
dtype=np.float32,
)
x = flow.Tensor(input_arr).to(device=flow.device("cuda:0"))
m = flow.nn.LayerNorm(x.size()[1:]).to(device=flow.device("cuda:0"))
y = m(x)
test_case.assertTrue(np.allclose(y.numpy(), output, 1e-05, 1e-05))
def _test_masked_fill_backward(test_case, device):
input_arr = np.array([
[
[-0.13169311, 0.97277078, 1.23305363, 1.56752789],
[-1.51954275, 1.87629473, -0.53301206, 0.53006478],
[-1.38244183, -2.63448052, 1.30845795, -0.67144869],
],
[
[0.41502161, 0.14452418, 0.38968, -1.76905653],
[0.34675095, -0.7050969, -0.7647731, -0.73233418],
[-1.90089858, 0.01262963, 0.74693893, 0.57132389],
],
])
fill_value = -3.1415 # random value e.g. -1e9 3.14
x = flow.Tensor(input_arr,
dtype=flow.float32,
device=flow.device(device),
requires_grad=True)
mask = flow.Tensor((input_arr > 0).astype(np.int8),
dtype=flow.int,
device=flow.device(device))
y = flow.masked_fill(x, mask, value=fill_value)
z = y.sum()
z.backward()
test_case.assertTrue(
np.array_equal(x.grad.numpy(), (input_arr < 0).astype(np.float32)))
def _test_batchnorm1d_2d_input(test_case, device):
input_arr = np.array(
[
[0.1438, 1.1229, -0.0480, -1.6834, -0.8262],
[0.5836, 0.1350, -0.8860, -1.7878, 1.0592],
[0.7252, -1.1488, -0.0274, 1.4051, 0.1018],
[-0.3595, -0.1801, 0.1146, -1.5712, -1.9291],
],
dtype=np.float32,
)
output_arr = np.array(
[
[-0.3056, 1.4066, 0.4151, -0.5783, -0.3864],
[0.7326, 0.1884, -1.7100, -0.6563, 1.3170],
[1.0668, -1.3949, 0.4674, 1.7292, 0.4521],
[-1.4938, -0.2002, 0.8275, -0.4945, -1.3827],
],
dtype=np.float32,
)
m = flow.nn.BatchNorm1d(num_features=5, eps=1e-5,
momentum=0.1).to(device=flow.device(device))
x = flow.Tensor(input_arr, device=flow.device(device))
y = m(x)
test_case.assertTrue(
np.allclose(y.numpy(), output_arr, rtol=1e-04, atol=1e-04))
def _test_instancenorm1d_backward(test_case, device):
input_arr = np.array(
[
[
[-0.1091, 2.0041, 0.8850, -0.0412],
[-1.2055, 0.7442, 2.3300, 1.2411],
[-1.2466, 0.3667, 1.2267, 0.3043],
],
[
[-0.2484, -1.1407, 0.3352, 0.6687],
[-0.2975, -0.0227, -0.2302, -0.3762],
[-0.7759, -0.6789, 1.1444, 1.8077],
],
],
dtype=np.float32,
)
m = flow.nn.InstanceNorm1d(num_features=2, eps=1e-5, momentum=0.1).to(
device=flow.device(device)
)
x = flow.Tensor(input_arr, device=flow.device(device), requires_grad=True)
y = m(x)
z = y.sum()
z.backward()
test_case.assertTrue(
np.allclose(x.grad.numpy(), np.zeros(shape=input_arr.shape), 1e-5, 1e-5)
)
def _test_nllloss_sum_backward(test_case, device):
x = np.array([
[0.88103855, 0.9908683, 0.6226845],
[0.53331435, 0.07999352, 0.8549948],
[0.25879037, 0.39530203, 0.698465],
[0.73427284, 0.63575995, 0.18827209],
[0.05689114, 0.0862954, 0.6325046],
]).astype(np.float32)
y = np.array([0, 2, 1, 1, 0]).astype(np.int)
input = flow.Tensor(x,
dtype=flow.float32,
device=flow.device(device),
requires_grad=True)
target = flow.Tensor(y, dtype=flow.int64, device=flow.device(device))
nll_loss = flow.nn.NLLLoss(reduction="sum")
nll_loss = nll_loss.to(device)
of_out = nll_loss(input, target)
of_out = of_out.sum()
of_out.backward()
np_grad = [
[-1.0, 0.0, 0.0],
[0.0, 0.0, -1.0],
[0.0, -1.0, 0.0],
[0.0, -1.0, 0.0],
[-1.0, 0.0, 0.0],
]
test_case.assertTrue(
np.allclose(input.grad.numpy(), np_grad, atol=1e-5, rtol=1e-5))
def _test_matmul_backward_y_grad(test_case, device):
input1 = flow.Tensor(
[
[-1.8604081869125366, -2.0019688606262207],
[1.0511547327041626, -2.263841390609741],
],
dtype=flow.float32,
device=flow.device(device),
requires_grad=False,
)
input2 = flow.Tensor(
[
[-0.13973912596702576, 0.8478717803955078],
[-0.2144828885793686, -1.7145386934280396],
],
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
of_out = flow.matmul(input1, input2)
of_out = of_out.sum()
of_out.backward()
print(input2.grad.numpy().tolist())
np_grad = [
[-0.809253454208374, -0.809253454208374],
[-4.265810012817383, -4.265810012817383],
]
test_case.assertTrue(
np.allclose(input2.grad.numpy(), np_grad, atol=1e-05, rtol=1e-05))
def _test_concat_with_three_tensor_backward(test_case, device):
input1 = flow.Tensor(
np.random.randn(2, 6, 5, 3),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
input2 = flow.Tensor(
np.random.randn(2, 6, 5, 3),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
input3 = flow.Tensor(
np.random.randn(2, 6, 5, 3),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
of_out = flow.cat([input1, input2, input3], dim=1)
of_out = of_out.sum()
of_out.backward()
test_case.assertTrue(
np.allclose(input1.grad.numpy(), np.ones((2, 6, 5, 3)), 1e-4, 1e-4))
test_case.assertTrue(
np.allclose(input2.grad.numpy(), np.ones((2, 6, 5, 3)), 1e-4, 1e-4))
test_case.assertTrue(
np.allclose(input3.grad.numpy(), np.ones((2, 6, 5, 3)), 1e-4, 1e-4))
def _test_batch_matmul_backward(test_case, device):
input1 = flow.Tensor(
[
[
[
-0.0036776792258024216, 1.9946473836898804,
-0.423959881067276
],
[
1.0892143249511719, 0.04005361348390579,
-0.27883127331733704
],
],
[
[
-0.970306396484375, 0.017771577462553978,
0.019596196711063385
],
[
0.27402883768081665, -0.8192587494850159,
-0.3135920464992523
],
],
],
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
input2 = flow.Tensor(
[
[
[1.118346929550171, -0.930071234703064],
[1.1238232851028442, 1.373764157295227],
[0.17178462445735931, -1.1010534763336182],
],
[
[0.6694859862327576, 0.9250285029411316],
[-1.0835869312286377, 0.4192655086517334],
[1.2616937160491943, 0.33809131383895874],
],
],
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
of_out = flow.matmul(input1, input2)
of_out = of_out.sum()
of_out.backward()
np_grad = [
[
[0.18827569484710693, 2.4975874423980713, -0.9292688369750977],
[0.18827569484710693, 2.4975874423980713, -0.9292688369750977],
],
[
[1.5945144891738892, -0.6643214225769043, 1.5997850894927979],
[1.5945144891738892, -0.6643214225769043, 1.5997850894927979],
],
]
test_case.assertTrue(
np.allclose(input1.grad.numpy(), np_grad, atol=1e-05, rtol=1e-05))
def test_tensor_to_d2d(test_case):
input = flow.Tensor(np.random.randn(2, 3, 4, 5),
device=flow.device("cuda"))
output = input.to(device=flow.device("cuda:0"))
test_case.assertEqual(output.device, flow.device("cuda:0"))
test_case.assertTrue(
np.allclose(input.numpy(), output.numpy(), rtol=1e-04, atol=1e-04))
def _test_meshgrid_forawd_3tensor(test_case, device):
input1 = flow.Tensor(
np.array([1, 2, 3]),
dtype=flow.float32,
device=flow.device(device),
)
input2 = flow.Tensor(
np.array([4, 5, 6]),
dtype=flow.float32,
device=flow.device(device),
)
input3 = flow.Tensor(
np.array([7, 8, 9]),
dtype=flow.float32,
device=flow.device(device),
)
np_x, np_y, np_z = np.meshgrid(input1.numpy(),
input2.numpy(),
input3.numpy(),
indexing="ij")
of_x, of_y, of_z = flow.meshgrid(input1, input2, input3)
test_case.assertTrue(np.allclose(of_x.numpy(), np_x, 1e-4, 1e-4))
test_case.assertTrue(np.allclose(of_y.numpy(), np_y, 1e-4, 1e-4))
test_case.assertTrue(np.allclose(of_z.numpy(), np_z, 1e-4, 1e-4))
def _test_where_backward(test_case, device):
x = flow.Tensor(
np.array([[-0.4620, 0.3139], [0.3898, -0.7197], [0.0478, -0.1657]]),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
y = flow.Tensor(
np.ones(shape=(3, 2)),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
condition = flow.Tensor(np.array([[0, 1], [1, 0], [1, 0]]),
dtype=flow.int32,
device=flow.device(device))
of_out = flow.where(condition, x, y)
of_out = of_out.sum()
of_out.backward()
test_case.assertTrue(
np.allclose(x.grad.numpy(),
condition.numpy() == 1, 1e-5, 1e-5))
test_case.assertTrue(
np.allclose(y.grad.numpy(),
condition.numpy() == 0, 1e-5, 1e-5))
def _test_bcewithlogitsloss_impl(test_case, device, shape, reduction):
x = np.random.randn(*shape).astype(np.float32)
y = np.random.randint(0, 2, [*shape]).astype(np.float32)
w = np.random.randn(*shape).astype(np.float32)
pw = np.random.randn([*shape][-1]).astype(np.float32)
input = flow.Tensor(
x, dtype=flow.float32, requires_grad=True, device=flow.device(device)
)
target = flow.Tensor(y, dtype=flow.float32, device=flow.device(device))
weight = flow.Tensor(w, dtype=flow.float32, device=flow.device(device))
pos_weight = flow.Tensor(pw, dtype=flow.float32, device=flow.device(device))
bcewithlogits_loss = flow.nn.BCEWithLogitsLoss(
weight=weight, pos_weight=pos_weight, reduction=reduction
)
of_out = bcewithlogits_loss(input, target)
np_out = _np_bcewithlogitsloss(
x, y, np_weight=w, np_pos_weight=pw, reduction=reduction
)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
# Backward test with np:
of_out = of_out.sum()
of_out.backward()
np_grad = _np_bcewithlogitsloss_grad(x, y, np_weight=w, np_pos_weight=pw,)[
reduction
]
test_case.assertTrue(np.allclose(input.grad.numpy(), np_grad, 1e-5, 1e-5))
def _test_nllloss_sum(test_case, device):
x = np.array(
[
[0.88103855, 0.9908683, 0.6226845],
[0.53331435, 0.07999352, 0.8549948],
[0.25879037, 0.39530203, 0.698465],
[0.73427284, 0.63575995, 0.18827209],
[0.05689114, 0.0862954, 0.6325046],
]
).astype(np.float32)
y = np.array([0, 2, 1, 1, 0]).astype(np.int)
input = flow.Tensor(x, dtype=flow.float32, device=flow.device(device))
target = flow.Tensor(y, dtype=flow.int64, device=flow.device(device))
nll_loss = flow.nn.NLLLoss(reduction="sum")
nll_loss = nll_loss.to(device)
of_out = nll_loss(input, target)
np_out = nll_loss_1d(input.numpy(), target.numpy(), reduction="sum")
test_case.assertTrue(np.allclose(of_out.numpy(), np_out))
nll_loss = flow.nn.NLLLoss(reduction="sum", ignore_index=1)
nll_loss = nll_loss.to(device)
of_out = nll_loss(input, target)
np_out = nll_loss_1d(input.numpy(), target.numpy(), reduction="sum", ignore_index=1)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out))
def _test_marginrankingloss_mean(test_case, shape, margin, device):
input1 = flow.Tensor(np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device))
input2 = flow.Tensor(np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device))
target_pos = flow.Tensor(np.ones(shape),
dtype=flow.float32,
device=flow.device(device))
target_neg = flow.Tensor(-1 * np.ones(shape),
dtype=flow.float32,
device=flow.device(device))
margin_ranking_loss = flow.nn.MarginRankingLoss(margin=margin,
reduction="mean")
margin_ranking_loss = margin_ranking_loss.to(device)
of_out_pos = margin_ranking_loss(input1, input2, target_pos)
np_out_pos = np_margin_ranking_loss(margin,
input1.numpy(),
input2.numpy(),
target_pos.numpy(),
reduction="mean")
test_case.assertTrue(
np.allclose(of_out_pos.numpy(), np_out_pos, 1e-5, 1e-5))
of_out_neg = margin_ranking_loss(input1, input2, target_neg)
np_out_neg = np_margin_ranking_loss(margin,
input1.numpy(),
input2.numpy(),
target_neg.numpy(),
reduction="mean")
test_case.assertTrue(
np.allclose(of_out_neg.numpy(), np_out_neg, 1e-5, 1e-5))
def _test_marginrankingloss_grad(test_case, shape, margin, device):
input1 = flow.Tensor(
np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
input2 = flow.Tensor(
np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
target = flow.Tensor(np.ones(shape),
dtype=flow.float32,
device=flow.device(device))
margin_ranking_loss = flow.nn.MarginRankingLoss(margin=margin,
reduction="sum")
margin_ranking_loss = margin_ranking_loss.to(device)
of_out = margin_ranking_loss(input1, input2, target)
of_out.backward()
np_out_grad1, np_out_grad2 = np_margin_ranking_loss_grad(
margin, input1.numpy(), input2.numpy(), target.numpy())
test_case.assertTrue(
np.allclose(input1.grad.numpy(), np_out_grad1, 1e-5, 1e-5))
test_case.assertTrue(
np.allclose(input2.grad.numpy(), np_out_grad2, 1e-5, 1e-5))
def _test_matmul_backward_x_grad(test_case, device):
input1 = flow.Tensor(
[
[-1.8604081869125366, -2.0019688606262207],
[1.0511547327041626, -2.263841390609741],
],
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
input2 = flow.Tensor(
[
[-0.13973912596702576, 0.8478717803955078],
[-0.2144828885793686, -1.7145386934280396],
],
dtype=flow.float32,
device=flow.device(device),
requires_grad=False,
)
of_out = flow.matmul(input1, input2)
of_out = of_out.sum()
of_out.backward()
np_grad = [
[0.7081326246261597, -1.9290215969085693],
[0.7081326246261597, -1.9290215969085693],
]
test_case.assertTrue(
np.allclose(input1.grad.numpy(), np_grad, atol=1e-05, rtol=1e-05))
def test_tensor_device(test_case):
shape = (2, 3, 4, 5)
x = flow.Tensor(*shape)
test_case.assertTrue(not x.is_cuda)
x = flow.Tensor(*shape, device=flow.device("cuda"))
test_case.assertTrue(x.is_cuda)
x = flow.Tensor(*shape, device=flow.device("cpu"))
test_case.assertTrue(not x.is_cuda)
def _test_pow_elementwise_impl(test_case, shape, scalar, device):
np_input_x = 10 * np.random.rand(*shape)
np_input_y = np.random.randint(1, 3, shape) + np.random.randn(*shape)
of_input_x = flow.Tensor(np_input_x, dtype=flow.float32, device=flow.device(device))
of_input_y = flow.Tensor(np_input_y, dtype=flow.float32, device=flow.device(device))
of_out = flow.pow(of_input_x, of_input_y)
np_out = np.power(np_input_x, np_input_y)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
def _test_gather_tensor_function(test_case, device):
input = np.array([[1, 2], [3, 4]])
index = np.array([[0, 0], [1, 0]])
np_out = np.take_along_axis(input, index, 1)
input = flow.Tensor(input, device=flow.device(device))
index = flow.Tensor(index, dtype=flow.int, device=flow.device(device))
output = input.gather(index, dim=1)
test_case.assertTrue(np.array_equal(output.numpy(), np_out))
def _test_zeros(test_case, device, shape):
y = flow.zeros(shape, device=flow.device(device))
test_case.assertTrue(np.array_equal(np.zeros(shape), y.numpy()))
y2 = flow.zeros(10, device=flow.device(device))
test_case.assertTrue(np.array_equal(np.zeros(10), y2.numpy()))
y3 = flow.zeros(10, dtype=flow.int, device=flow.device(device))
test_case.assertTrue(np.array_equal(np.zeros(10, dtype=int), y3.numpy()))
def _test_flatten_backward(test_case, device):
m = flow.nn.Flatten().to(flow.device(device))
x = flow.Tensor(2, 3, 4, 5, device=flow.device(device), requires_grad=True)
flow.nn.init.uniform_(x)
y = m(x)
z = y.sum()
z.backward()
test_case.assertTrue(
np.array_equal(np.ones(shape=(2, 3, 4, 5)), x.grad.numpy()))
def _test_ones(test_case, device, shape):
y = flow.ones(shape, device=flow.device(device))
test_case.assertTrue(np.array_equal(np.ones(shape), y.numpy()))
y2 = flow.ones(10, device=flow.device(device))
test_case.assertTrue(np.array_equal(np.ones(10), y2.numpy()))
y3 = flow.ones(10, dtype=flow.float64, device=flow.device(device))
test_case.assertTrue(
np.array_equal(np.ones(10, dtype=np.float64), y3.numpy()))
def _test_less_normal(test_case, device):
input1 = flow.Tensor(np.random.randn(2, 6, 5, 3),
dtype=flow.float32,
device=flow.device(device))
input2 = flow.Tensor(np.random.randn(2, 6, 5, 3),
dtype=flow.float32,
device=flow.device(device))
of_out = flow.lt(input1, input2)
np_out = np.less(input1.numpy(), input2.numpy())
test_case.assertTrue(np.array_equal(of_out.numpy(), np_out))
def _test_batch_matmul(test_case, device):
input1 = flow.Tensor(np.random.randn(10, 3, 4),
dtype=flow.float32,
device=flow.device(device))
input2 = flow.Tensor(np.random.randn(10, 4, 5),
dtype=flow.float32,
device=flow.device(device))
of_out = flow.matmul(input1, input2)
np_out = np.matmul(input1.numpy(), input2.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
def _test_nllloss_bert_sum(test_case, device):
x = np.array([[[0.12, 0.36, 0.22, 0.66], [0.13, 0.34, 0.52,
-0.96]]]).astype(np.float32)
input = flow.Tensor(x, dtype=flow.float32, device=flow.device(device))
y = np.array([[1, 0, 0, 1]]).astype(np.int)
target = flow.Tensor(y, dtype=flow.int64, device=flow.device(device))
nll_loss = flow.nn.NLLLoss(reduction="sum")
nll_loss = nll_loss.to(device)
of_out = nll_loss(input, target)
np_out = nll_loss_bert(input.numpy(), target.numpy(), reduction="sum")
test_case.assertTrue(np.allclose(of_out.numpy(), np_out))
def _test_concat_with_axis_one(test_case, device):
input1 = flow.Tensor(np.random.randn(2, 6, 5, 3),
dtype=flow.float32,
device=flow.device(device))
input2 = flow.Tensor(np.random.randn(2, 6, 5, 3),
dtype=flow.float32,
device=flow.device(device))
of_out = flow.cat([input1, input2], dim=1)
np_out = np.concatenate((input1.numpy(), input2.numpy()), axis=1)
test_case.assertTrue(np.array_equal(of_out.numpy(), np_out))
def _test_eq(test_case, shape, device):
arr1 = np.random.randn(*shape)
arr2 = np.random.randn(*shape)
input = flow.Tensor(arr1, dtype=flow.float32, device=flow.device(device))
other = flow.Tensor(arr2, dtype=flow.float32, device=flow.device(device))
of_out = flow.eq(input, other)
of_out2 = flow.equal(input, other)
np_out = np.equal(arr1, arr2)
test_case.assertTrue(np.array_equal(of_out.numpy(), np_out))
test_case.assertTrue(np.array_equal(of_out2.numpy(), np_out))
def _test_nllloss_segmentation_none(test_case, device):
x = np.array([[[[0.12, 0.36], [0.22, 0.66]],
[[0.13, 0.34], [0.52, -0.96]]]]).astype(np.float32)
input = flow.Tensor(x, dtype=flow.float32, device=flow.device(device))
y = np.array([[[1, 0], [0, 1]]]).astype(np.int)
target = flow.Tensor(y, dtype=flow.int64, device=flow.device(device))
nll_loss = flow.nn.NLLLoss()
nll_loss = nll_loss.to(device)
of_out = nll_loss(input, target)
np_out = nll_loss_2d(input.numpy(), target.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out))
def _test_atan2_forward(test_case, shape, scalar, device):
np_input_x = 10 * np.random.rand(*shape)
np_input_y = 10 * np.random.randn(*shape)
of_input_x = flow.Tensor(np_input_x,
dtype=flow.float32,
device=flow.device(device))
of_input_y = flow.Tensor(np_input_y,
dtype=flow.float32,
device=flow.device(device))
of_out = flow.atan2(of_input_x, of_input_y)
np_out = np.arctan2(np_input_x, np_input_y)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
def test_conv2d_kernel(test_case):
for device in ["cuda", "cpu"]:
conv = flow.nn.Conv2d(1, 1, (3, 5),
bias=False).to(flow.device(device))
conv.to(flow.device("cuda"))
_test_conv2d(
test_case,
conv,
test_conv2d_kernel_data,
test_conv2d_kernel_weight,
test_conv2d_kernel_output,
)
