def test_eq(test_case):
arr1 = np.array([
2,
3,
4,
5,
])
arr2 = np.array([2, 3, 4, 1])
input = flow.Tensor(arr1, dtype=flow.float32)
other = flow.Tensor(arr2, dtype=flow.float32)
of_out = flow.eq(input, other)
of_out2 = flow.equal(input, other)
np_out = np.equal(arr1, arr2)
test_case.assertTrue(np.array_equal(of_out.numpy(), np_out))
test_case.assertTrue(np.array_equal(of_out2.numpy(), np_out))
def test_softplus(test_case):
m = flow.nn.Softplus()
arr = np.random.randn(2, 3, 4, 5)
np_out = np.where(arr > 20, arr, np.log(1.0 + np.exp(1.0 * arr)))
x = flow.Tensor(arr)
of_out = m(x)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, rtol=1e-05))
def test_hardtanh_min_max(test_case):
m = flow.nn.Hardtanh(min_val=-2.0, max_val=2.3)
arr = np.random.randn(2, 3, 4, 5)
np_out = np.maximum(-2.0, np.minimum(2.3, arr))
x = flow.Tensor(arr)
of_out = m(x)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, rtol=1e-05))
def _test_linear_backward_with_bias(test_case, device):
linear = flow.nn.Linear(3, 8)
linear = linear.to(device)
x = flow.Tensor(
[
[-0.94630778, -0.83378579, -0.87060891],
[2.0289922, -0.28708987, -2.18369248],
[0.35217619, -0.67095644, -1.58943879],
[0.08086036, -1.81075924, 1.20752494],
[0.8901075, -0.49976737, -1.07153746],
[-0.44872912, -1.07275683, 0.06256855],
[-0.22556897, 0.74798368, 0.90416439],
[0.48339456, -2.32742195, -0.59321527],
],
device=flow.device(device),
requires_grad=True,
)
flow.nn.init.constant_(linear.weight, 2.068758)
flow.nn.init.constant_(linear.bias, 0.23)
of_out = linear(x)
of_out = of_out.sum()
of_out.backward()
np_grad = np.array([
[16.5501, 16.5501, 16.5501],
[16.5501, 16.5501, 16.5501],
[16.5501, 16.5501, 16.5501],
[16.5501, 16.5501, 16.5501],
[16.5501, 16.5501, 16.5501],
[16.5501, 16.5501, 16.5501],
[16.5501, 16.5501, 16.5501],
[16.5501, 16.5501, 16.5501],
])
test_case.assertTrue(np.allclose(np_grad, x.grad.numpy(), 1e-4, 1e-4))
def _test_meshgrid_forawd_scalr(test_case, device):
input1 = flow.Tensor(
np.array(1.0),
dtype=flow.float32,
device=flow.device(device),
)
input2 = flow.Tensor(
np.array(2.0),
dtype=flow.float32,
device=flow.device(device),
)
np_x, np_y = np.meshgrid(input1.numpy(), input2.numpy(), indexing="ij")
of_x, of_y = flow.meshgrid(input1, input2)
test_case.assertTrue(np.allclose(of_x.numpy(), np_x, 1e-4, 1e-4))
test_case.assertTrue(np.allclose(of_y.numpy(), np_y, 1e-4, 1e-4))
def _test_where_scalar(test_case, device):
x = 0.5
y = 2.0
condition = flow.Tensor(np.array([1]), dtype=flow.int32)
of_out = flow.where(condition, x, y)
np_out = np.array([0.5])
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
def _test_repeat_new_dim_backward(test_case, device):
input = flow.Tensor(
np.random.randn(2, 4, 1, 3),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
sizes = (4, 3, 2, 3, 3)
of_out = input.repeat(sizes=sizes)
of_out = of_out.sum()
of_out.backward()
np_grad = [
[
[[216.0, 216.0, 216.0]],
[[216.0, 216.0, 216.0]],
[[216.0, 216.0, 216.0]],
[[216.0, 216.0, 216.0]],
],
[
[[216.0, 216.0, 216.0]],
[[216.0, 216.0, 216.0]],
[[216.0, 216.0, 216.0]],
[[216.0, 216.0, 216.0]],
],
]
test_case.assertTrue(np.array_equal(input.grad.numpy(), np_grad))
def test_arcsin(test_case):
input = flow.Tensor(np.random.random((4, 5, 6)) - 0.5,
dtype=flow.float32)
of_out = input.arcsin()
np_out = np.arcsin(input.numpy())
test_case.assertTrue(
np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
def _test_cast_tensor_function(test_case):
shape = (2, 3, 4, 5)
np_arr = np.random.randn(*shape).astype(np.float32)
input = flow.Tensor(np_arr, dtype=flow.float32)
output = input.cast(flow.int8)
np_out = np_arr.astype(np.int8)
test_case.assertTrue(np.array_equal(output.numpy(), np_out))
def test_add_tensor_method(test_case):
x = flow.Tensor(np.random.randn(1, 1))
y = flow.Tensor(np.random.randn(2, 3))
of_out = x + y
np_out = np.add(x.numpy(), y.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
x = flow.Tensor(np.random.randn(2, 3))
of_out = x + 3
np_out = np.add(x.numpy(), 3)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
x = flow.Tensor(np.random.randn(2, 3))
of_out = 3 + x
np_out = np.add(3, x.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
def test_sub_tensor_method(test_case):
x = flow.Tensor(np.random.randn(1, 1))
y = flow.Tensor(np.random.randn(2, 3))
of_out = x - y
np_out = np.subtract(x.numpy(), y.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
x = flow.Tensor(np.random.randn(2, 3))
of_out = x - 3
np_out = np.subtract(x.numpy(), 3)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
x = flow.Tensor(np.random.randn(2, 3))
of_out = 3 - x
np_out = np.subtract(3, x.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
def test_mul(test_case):
x = flow.Tensor(np.random.randn(1, 1))
y = flow.Tensor(np.random.randn(2, 3))
of_out = x * y
np_out = np.multiply(x.numpy(), y.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
x = flow.Tensor(np.random.randn(2, 3))
of_out = x * 3
np_out = np.multiply(x.numpy(), 3)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
x = flow.Tensor(np.random.randn(2, 3))
of_out = 3 * x
np_out = np.multiply(3, x.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
def test_numpy_and_default_dtype(test_case):
shape = (2, 3, 4, 5)
tensor = flow.Tensor(*shape)
flow.nn.init.ones_(tensor)
test_case.assertTrue(tensor.dtype == flow.float32)
test_case.assertTrue(
np.array_equal(tensor.numpy(), np.ones(shape, dtype=np.float32)))
def test_creating_consistent_tensor(test_case):
shape = (2, 3)
x = flow.Tensor(*shape, placement=flow.placement("gpu", ["0:0"], None))
x.set_placement(flow.placement("cpu", ["0:0"], None))
x.set_is_consistent(True)
test_case.assertTrue(not x.is_cuda)
x.determine()
def test_layernorm_v3(test_case):
input_arr = np.array(
[
[
[[-0.16046895, -1.03667831], [-0.34974465, 0.26505867]],
[[-1.24111986, -0.53806001], [1.72426331, 0.43572459]],
],
[
[[-0.77390957, -0.42610624], [0.16398858, -1.35760343]],
[[1.07541728, 0.11008703], [0.26361224, -0.48663723]],
],
],
dtype=np.float32,
)
output = np.array(
[
[
[[0.9999740, -0.9999740], [-0.9999470, 0.9999470]],
[[-0.9999595, 0.9999595], [0.9999880, -0.9999880]],
],
[
[[-0.9998344, 0.9998341], [0.9999914, -0.9999914]],
[[0.9999787, -0.9999787], [0.9999645, -0.9999645]],
],
],
dtype=np.float32,
)
x = flow.Tensor(input_arr)
m = flow.nn.LayerNorm(2, elementwise_affine=True)
y = m(x)
test_case.assertTrue(np.allclose(y.numpy(), output))
def test_sqrt_tensor_function(test_case):
input_arr = np.random.randn(1, 6, 3, 8)
np_out = np.sqrt(input_arr)
x = flow.Tensor(input_arr)
of_out = x.sqrt()
test_case.assertTrue(
np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
def _test_expand_backward_same_dim(test_case, device):
input_shape = (2, 4, 1, 1)
expand_dim = [2, 4, 2, 1]
input = np.array([
[
[[0.9876952171325684]],
[[0.8772538304328918]],
[[0.9200366735458374]],
[[0.2810221314430237]],
],
[
[[0.3037724494934082]],
[[0.7783719897270203]],
[[0.08884672075510025]],
[[0.17156553268432617]],
],
])
of_input = flow.Tensor(input,
dtype=flow.float32,
device=flow.device(device),
requires_grad=True)
of_out = of_input.expand(2, 4, 2, 1)
y = of_out.sum().backward()
np_grad = [
[[[2.0]], [[2.0]], [[2.0]], [[2.0]]],
[[[2.0]], [[2.0]], [[2.0]], [[2.0]]],
]
test_case.assertTrue(np.array_equal(of_input.grad.numpy(), np_grad))
def test_rsqrt_tensor_function(test_case):
np_arr = np.random.randn(3, 2, 5, 7)
np_out = 1 / np.sqrt(np_arr)
x = flow.Tensor(np_arr)
of_out = flow.rsqrt(input=x)
test_case.assertTrue(
np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
def test_nllloss_mean(test_case):
x = np.array([
[0.88103855, 0.9908683, 0.6226845],
[0.53331435, 0.07999352, 0.8549948],
[0.25879037, 0.39530203, 0.698465],
[0.73427284, 0.63575995, 0.18827209],
[0.05689114, 0.0862954, 0.6325046],
]).astype(np.float32)
y = np.array([0, 2, 1, 1, 0]).astype(np.int)
input = flow.Tensor(x, dtype=flow.float32)
target = flow.Tensor(y, dtype=flow.int64)
nll_loss = flow.nn.NLLLoss(reduction="mean")
of_out = nll_loss(input, target)
np_out = nll_loss_1d(input.numpy(), target.numpy(), reduction="mean")
test_case.assertTrue(np.allclose(of_out.numpy(), np_out))
def test_square_tensor_function(test_case):
np_arr = np.random.randn(2, 7, 7, 3)
np_out = np.square(np_arr)
x = flow.Tensor(np_arr)
of_out = x.square()
test_case.assertTrue(
np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
def _test_softplus(test_case, device):
m = flow.nn.Softplus()
arr = np.random.randn(2, 3, 4, 5)
np_out = np.where(arr > 20, arr, np.log(1.0 + np.exp(1.0 * arr)))
x = flow.Tensor(arr, device=flow.device(device))
of_out = m(x)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
def train_one_iter(grad):
grad_tensor = flow.Tensor(grad, requires_grad=False)
loss = x * grad_tensor
loss = flow.sum(x * grad_tensor)
loss.backward()
adam.step()
adam.zero_grad()
def _test_linear_with_bias(test_case, device):
linear = flow.nn.Linear(3, 8)
linear = linear.to(device)
input_arr = np.array(
[
[-0.94630778, -0.83378579, -0.87060891],
[2.0289922, -0.28708987, -2.18369248],
[0.35217619, -0.67095644, -1.58943879],
[0.08086036, -1.81075924, 1.20752494],
[0.8901075, -0.49976737, -1.07153746],
[-0.44872912, -1.07275683, 0.06256855],
[-0.22556897, 0.74798368, 0.90416439],
[0.48339456, -2.32742195, -0.59321527],
],
dtype=np.float32,
)
np_weight = np.ones((3, 8)).astype(np.float32)
np_weight.fill(2.068758)
np_bias = np.ones((8))
np_bias.fill(0.23)
x = flow.Tensor(input_arr, device=flow.device(device))
flow.nn.init.constant_(linear.weight, 2.068758)
flow.nn.init.constant_(linear.bias, 0.23)
of_out = linear(x)
np_out = np.matmul(input_arr, np_weight)
np_out += np_bias
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
def train_by_oneflow():
x = Parameter(flow.Tensor(init_value, device=flow.device(device)))
adam = flow.optim.Adam(
[
{
"params": [x],
"lr": learning_rate,
"betas": betas,
"eps": eps,
"weight_decay": weight_decay,
"scale": scale,
}
]
)
def train_one_iter(grad):
grad_tensor = flow.Tensor(
grad, requires_grad=False, device=flow.device(device)
)
loss = flow.sum(x * grad_tensor)
loss.backward()
adam.step()
adam.zero_grad()
for i in range(train_iters):
train_one_iter(random_grad_seq[i])
return x
def test_logsigmoid(test_case):
m = flow.nn.LogSigmoid()
arr = np.random.randn(2, 3, 4, 5)
np_out = np.log(1.0 / (1.0 + np.exp(-arr)))
x = flow.Tensor(arr)
of_out = m(x)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, rtol=1e-05))
def test_layernorm(test_case):
input_arr = np.array(
[
[
[[-0.16046895, -1.03667831], [-0.34974465, 0.26505867]],
[[-1.24111986, -0.53806001], [1.72426331, 0.43572459]],
],
[
[[-0.77390957, -0.42610624], [0.16398858, -1.35760343]],
[[1.07541728, 0.11008703], [0.26361224, -0.48663723]],
],
],
dtype=np.float32,
)
output = np.array(
[
[
[[-0.0544118, -1.0509688], [-0.2696846, 0.4295622]],
[[-1.2834904, -0.4838651], [2.0891891, 0.6236691]],
],
[
[[-0.8555527, -0.3554582], [0.4930190, -1.6948260]],
[[1.8035311, 0.4155158], [0.6362644, -0.4424936]],
],
],
dtype=np.float32,
)
x = flow.Tensor(input_arr)
m = flow.nn.LayerNorm(x.size()[1:])
y = m(x)
test_case.assertTrue(np.allclose(y.numpy(), output))
def test_hardtanh(test_case):
m = flow.nn.Hardtanh()
arr = np.random.randn(2, 3, 4, 5)
np_out = np.maximum(-1, np.minimum(1, arr))
x = flow.Tensor(arr)
of_out = m(x)
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, rtol=1e-05))
def test_layernorm_v2(test_case):
input_arr = np.array(
[
[
[[-0.16046895, -1.03667831], [-0.34974465, 0.26505867]],
[[-1.24111986, -0.53806001], [1.72426331, 0.43572459]],
],
[
[[-0.77390957, -0.42610624], [0.16398858, -1.35760343]],
[[1.07541728, 0.11008703], [0.26361224, -0.48663723]],
],
],
dtype=np.float32,
)
output = np.array(
[
[
[[0.3406544, -1.5249983], [-0.0623574, 1.2467014]],
[[-1.2004623, -0.5688803], [1.4634399, 0.3059027]],
],
[
[[-0.3180245, 0.3122248], [1.3815271, -1.3757277]],
[[1.4972910, -0.2341234], [0.0412391, -1.3044068]],
],
],
dtype=np.float32,
)
x = flow.Tensor(input_arr)
m = flow.nn.LayerNorm([2, 2], eps=1e-5)
y = m(x)
test_case.assertTrue(np.allclose(y.numpy(), output))
def _test_body_tanh_v3(test_case, input_arr):
x = flow.Tensor(input_arr)
y = x.tanh()
z = np.tanh(input_arr)
test_case.assertTrue(np.allclose(y.numpy(), z, rtol=1e-4, atol=1e-4))
def _test_eq_float(test_case, shape, device):
arr = np.random.randn(*shape)
input = flow.Tensor(arr, dtype=flow.float32, device=flow.device(device))
num = 1
of_out = flow.eq(input, num)
np_out = np.equal(arr, num)
test_case.assertTrue(np.array_equal(of_out.numpy(), np_out))