def test_eq(test_case): arr1 = np.array([ 2, 3, 4, 5, ]) arr2 = np.array([2, 3, 4, 1]) input = flow.Tensor(arr1, dtype=flow.float32) other = flow.Tensor(arr2, dtype=flow.float32) of_out = flow.eq(input, other) of_out2 = flow.equal(input, other) np_out = np.equal(arr1, arr2) test_case.assertTrue(np.array_equal(of_out.numpy(), np_out)) test_case.assertTrue(np.array_equal(of_out2.numpy(), np_out))
def test_softplus(test_case): m = flow.nn.Softplus() arr = np.random.randn(2, 3, 4, 5) np_out = np.where(arr > 20, arr, np.log(1.0 + np.exp(1.0 * arr))) x = flow.Tensor(arr) of_out = m(x) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, rtol=1e-05))
def test_hardtanh_min_max(test_case): m = flow.nn.Hardtanh(min_val=-2.0, max_val=2.3) arr = np.random.randn(2, 3, 4, 5) np_out = np.maximum(-2.0, np.minimum(2.3, arr)) x = flow.Tensor(arr) of_out = m(x) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, rtol=1e-05))
def _test_linear_backward_with_bias(test_case, device): linear = flow.nn.Linear(3, 8) linear = linear.to(device) x = flow.Tensor( [ [-0.94630778, -0.83378579, -0.87060891], [2.0289922, -0.28708987, -2.18369248], [0.35217619, -0.67095644, -1.58943879], [0.08086036, -1.81075924, 1.20752494], [0.8901075, -0.49976737, -1.07153746], [-0.44872912, -1.07275683, 0.06256855], [-0.22556897, 0.74798368, 0.90416439], [0.48339456, -2.32742195, -0.59321527], ], device=flow.device(device), requires_grad=True, ) flow.nn.init.constant_(linear.weight, 2.068758) flow.nn.init.constant_(linear.bias, 0.23) of_out = linear(x) of_out = of_out.sum() of_out.backward() np_grad = np.array([ [16.5501, 16.5501, 16.5501], [16.5501, 16.5501, 16.5501], [16.5501, 16.5501, 16.5501], [16.5501, 16.5501, 16.5501], [16.5501, 16.5501, 16.5501], [16.5501, 16.5501, 16.5501], [16.5501, 16.5501, 16.5501], [16.5501, 16.5501, 16.5501], ]) test_case.assertTrue(np.allclose(np_grad, x.grad.numpy(), 1e-4, 1e-4))
def _test_meshgrid_forawd_scalr(test_case, device): input1 = flow.Tensor( np.array(1.0), dtype=flow.float32, device=flow.device(device), ) input2 = flow.Tensor( np.array(2.0), dtype=flow.float32, device=flow.device(device), ) np_x, np_y = np.meshgrid(input1.numpy(), input2.numpy(), indexing="ij") of_x, of_y = flow.meshgrid(input1, input2) test_case.assertTrue(np.allclose(of_x.numpy(), np_x, 1e-4, 1e-4)) test_case.assertTrue(np.allclose(of_y.numpy(), np_y, 1e-4, 1e-4))
def _test_where_scalar(test_case, device): x = 0.5 y = 2.0 condition = flow.Tensor(np.array([1]), dtype=flow.int32) of_out = flow.where(condition, x, y) np_out = np.array([0.5]) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
def _test_repeat_new_dim_backward(test_case, device): input = flow.Tensor( np.random.randn(2, 4, 1, 3), dtype=flow.float32, device=flow.device(device), requires_grad=True, ) sizes = (4, 3, 2, 3, 3) of_out = input.repeat(sizes=sizes) of_out = of_out.sum() of_out.backward() np_grad = [ [ [[216.0, 216.0, 216.0]], [[216.0, 216.0, 216.0]], [[216.0, 216.0, 216.0]], [[216.0, 216.0, 216.0]], ], [ [[216.0, 216.0, 216.0]], [[216.0, 216.0, 216.0]], [[216.0, 216.0, 216.0]], [[216.0, 216.0, 216.0]], ], ] test_case.assertTrue(np.array_equal(input.grad.numpy(), np_grad))
def test_arcsin(test_case): input = flow.Tensor(np.random.random((4, 5, 6)) - 0.5, dtype=flow.float32) of_out = input.arcsin() np_out = np.arcsin(input.numpy()) test_case.assertTrue( np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
def _test_cast_tensor_function(test_case): shape = (2, 3, 4, 5) np_arr = np.random.randn(*shape).astype(np.float32) input = flow.Tensor(np_arr, dtype=flow.float32) output = input.cast(flow.int8) np_out = np_arr.astype(np.int8) test_case.assertTrue(np.array_equal(output.numpy(), np_out))
def test_add_tensor_method(test_case): x = flow.Tensor(np.random.randn(1, 1)) y = flow.Tensor(np.random.randn(2, 3)) of_out = x + y np_out = np.add(x.numpy(), y.numpy()) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4)) x = flow.Tensor(np.random.randn(2, 3)) of_out = x + 3 np_out = np.add(x.numpy(), 3) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4)) x = flow.Tensor(np.random.randn(2, 3)) of_out = 3 + x np_out = np.add(3, x.numpy()) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
def test_sub_tensor_method(test_case): x = flow.Tensor(np.random.randn(1, 1)) y = flow.Tensor(np.random.randn(2, 3)) of_out = x - y np_out = np.subtract(x.numpy(), y.numpy()) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4)) x = flow.Tensor(np.random.randn(2, 3)) of_out = x - 3 np_out = np.subtract(x.numpy(), 3) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4)) x = flow.Tensor(np.random.randn(2, 3)) of_out = 3 - x np_out = np.subtract(3, x.numpy()) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
def test_mul(test_case): x = flow.Tensor(np.random.randn(1, 1)) y = flow.Tensor(np.random.randn(2, 3)) of_out = x * y np_out = np.multiply(x.numpy(), y.numpy()) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4)) x = flow.Tensor(np.random.randn(2, 3)) of_out = x * 3 np_out = np.multiply(x.numpy(), 3) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4)) x = flow.Tensor(np.random.randn(2, 3)) of_out = 3 * x np_out = np.multiply(3, x.numpy()) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-4, 1e-4))
def test_numpy_and_default_dtype(test_case): shape = (2, 3, 4, 5) tensor = flow.Tensor(*shape) flow.nn.init.ones_(tensor) test_case.assertTrue(tensor.dtype == flow.float32) test_case.assertTrue( np.array_equal(tensor.numpy(), np.ones(shape, dtype=np.float32)))
def test_creating_consistent_tensor(test_case): shape = (2, 3) x = flow.Tensor(*shape, placement=flow.placement("gpu", ["0:0"], None)) x.set_placement(flow.placement("cpu", ["0:0"], None)) x.set_is_consistent(True) test_case.assertTrue(not x.is_cuda) x.determine()
def test_layernorm_v3(test_case): input_arr = np.array( [ [ [[-0.16046895, -1.03667831], [-0.34974465, 0.26505867]], [[-1.24111986, -0.53806001], [1.72426331, 0.43572459]], ], [ [[-0.77390957, -0.42610624], [0.16398858, -1.35760343]], [[1.07541728, 0.11008703], [0.26361224, -0.48663723]], ], ], dtype=np.float32, ) output = np.array( [ [ [[0.9999740, -0.9999740], [-0.9999470, 0.9999470]], [[-0.9999595, 0.9999595], [0.9999880, -0.9999880]], ], [ [[-0.9998344, 0.9998341], [0.9999914, -0.9999914]], [[0.9999787, -0.9999787], [0.9999645, -0.9999645]], ], ], dtype=np.float32, ) x = flow.Tensor(input_arr) m = flow.nn.LayerNorm(2, elementwise_affine=True) y = m(x) test_case.assertTrue(np.allclose(y.numpy(), output))
def test_sqrt_tensor_function(test_case): input_arr = np.random.randn(1, 6, 3, 8) np_out = np.sqrt(input_arr) x = flow.Tensor(input_arr) of_out = x.sqrt() test_case.assertTrue( np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
def _test_expand_backward_same_dim(test_case, device): input_shape = (2, 4, 1, 1) expand_dim = [2, 4, 2, 1] input = np.array([ [ [[0.9876952171325684]], [[0.8772538304328918]], [[0.9200366735458374]], [[0.2810221314430237]], ], [ [[0.3037724494934082]], [[0.7783719897270203]], [[0.08884672075510025]], [[0.17156553268432617]], ], ]) of_input = flow.Tensor(input, dtype=flow.float32, device=flow.device(device), requires_grad=True) of_out = of_input.expand(2, 4, 2, 1) y = of_out.sum().backward() np_grad = [ [[[2.0]], [[2.0]], [[2.0]], [[2.0]]], [[[2.0]], [[2.0]], [[2.0]], [[2.0]]], ] test_case.assertTrue(np.array_equal(of_input.grad.numpy(), np_grad))
def test_rsqrt_tensor_function(test_case): np_arr = np.random.randn(3, 2, 5, 7) np_out = 1 / np.sqrt(np_arr) x = flow.Tensor(np_arr) of_out = flow.rsqrt(input=x) test_case.assertTrue( np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
def test_nllloss_mean(test_case): x = np.array([ [0.88103855, 0.9908683, 0.6226845], [0.53331435, 0.07999352, 0.8549948], [0.25879037, 0.39530203, 0.698465], [0.73427284, 0.63575995, 0.18827209], [0.05689114, 0.0862954, 0.6325046], ]).astype(np.float32) y = np.array([0, 2, 1, 1, 0]).astype(np.int) input = flow.Tensor(x, dtype=flow.float32) target = flow.Tensor(y, dtype=flow.int64) nll_loss = flow.nn.NLLLoss(reduction="mean") of_out = nll_loss(input, target) np_out = nll_loss_1d(input.numpy(), target.numpy(), reduction="mean") test_case.assertTrue(np.allclose(of_out.numpy(), np_out))
def test_square_tensor_function(test_case): np_arr = np.random.randn(2, 7, 7, 3) np_out = np.square(np_arr) x = flow.Tensor(np_arr) of_out = x.square() test_case.assertTrue( np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5, equal_nan=True))
def _test_softplus(test_case, device): m = flow.nn.Softplus() arr = np.random.randn(2, 3, 4, 5) np_out = np.where(arr > 20, arr, np.log(1.0 + np.exp(1.0 * arr))) x = flow.Tensor(arr, device=flow.device(device)) of_out = m(x) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
def train_one_iter(grad): grad_tensor = flow.Tensor(grad, requires_grad=False) loss = x * grad_tensor loss = flow.sum(x * grad_tensor) loss.backward() adam.step() adam.zero_grad()
def _test_linear_with_bias(test_case, device): linear = flow.nn.Linear(3, 8) linear = linear.to(device) input_arr = np.array( [ [-0.94630778, -0.83378579, -0.87060891], [2.0289922, -0.28708987, -2.18369248], [0.35217619, -0.67095644, -1.58943879], [0.08086036, -1.81075924, 1.20752494], [0.8901075, -0.49976737, -1.07153746], [-0.44872912, -1.07275683, 0.06256855], [-0.22556897, 0.74798368, 0.90416439], [0.48339456, -2.32742195, -0.59321527], ], dtype=np.float32, ) np_weight = np.ones((3, 8)).astype(np.float32) np_weight.fill(2.068758) np_bias = np.ones((8)) np_bias.fill(0.23) x = flow.Tensor(input_arr, device=flow.device(device)) flow.nn.init.constant_(linear.weight, 2.068758) flow.nn.init.constant_(linear.bias, 0.23) of_out = linear(x) np_out = np.matmul(input_arr, np_weight) np_out += np_bias test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
def train_by_oneflow(): x = Parameter(flow.Tensor(init_value, device=flow.device(device))) adam = flow.optim.Adam( [ { "params": [x], "lr": learning_rate, "betas": betas, "eps": eps, "weight_decay": weight_decay, "scale": scale, } ] ) def train_one_iter(grad): grad_tensor = flow.Tensor( grad, requires_grad=False, device=flow.device(device) ) loss = flow.sum(x * grad_tensor) loss.backward() adam.step() adam.zero_grad() for i in range(train_iters): train_one_iter(random_grad_seq[i]) return x
def test_logsigmoid(test_case): m = flow.nn.LogSigmoid() arr = np.random.randn(2, 3, 4, 5) np_out = np.log(1.0 / (1.0 + np.exp(-arr))) x = flow.Tensor(arr) of_out = m(x) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, rtol=1e-05))
def test_layernorm(test_case): input_arr = np.array( [ [ [[-0.16046895, -1.03667831], [-0.34974465, 0.26505867]], [[-1.24111986, -0.53806001], [1.72426331, 0.43572459]], ], [ [[-0.77390957, -0.42610624], [0.16398858, -1.35760343]], [[1.07541728, 0.11008703], [0.26361224, -0.48663723]], ], ], dtype=np.float32, ) output = np.array( [ [ [[-0.0544118, -1.0509688], [-0.2696846, 0.4295622]], [[-1.2834904, -0.4838651], [2.0891891, 0.6236691]], ], [ [[-0.8555527, -0.3554582], [0.4930190, -1.6948260]], [[1.8035311, 0.4155158], [0.6362644, -0.4424936]], ], ], dtype=np.float32, ) x = flow.Tensor(input_arr) m = flow.nn.LayerNorm(x.size()[1:]) y = m(x) test_case.assertTrue(np.allclose(y.numpy(), output))
def test_hardtanh(test_case): m = flow.nn.Hardtanh() arr = np.random.randn(2, 3, 4, 5) np_out = np.maximum(-1, np.minimum(1, arr)) x = flow.Tensor(arr) of_out = m(x) test_case.assertTrue(np.allclose(of_out.numpy(), np_out, rtol=1e-05))
def test_layernorm_v2(test_case): input_arr = np.array( [ [ [[-0.16046895, -1.03667831], [-0.34974465, 0.26505867]], [[-1.24111986, -0.53806001], [1.72426331, 0.43572459]], ], [ [[-0.77390957, -0.42610624], [0.16398858, -1.35760343]], [[1.07541728, 0.11008703], [0.26361224, -0.48663723]], ], ], dtype=np.float32, ) output = np.array( [ [ [[0.3406544, -1.5249983], [-0.0623574, 1.2467014]], [[-1.2004623, -0.5688803], [1.4634399, 0.3059027]], ], [ [[-0.3180245, 0.3122248], [1.3815271, -1.3757277]], [[1.4972910, -0.2341234], [0.0412391, -1.3044068]], ], ], dtype=np.float32, ) x = flow.Tensor(input_arr) m = flow.nn.LayerNorm([2, 2], eps=1e-5) y = m(x) test_case.assertTrue(np.allclose(y.numpy(), output))
def _test_body_tanh_v3(test_case, input_arr): x = flow.Tensor(input_arr) y = x.tanh() z = np.tanh(input_arr) test_case.assertTrue(np.allclose(y.numpy(), z, rtol=1e-4, atol=1e-4))
def _test_eq_float(test_case, shape, device): arr = np.random.randn(*shape) input = flow.Tensor(arr, dtype=flow.float32, device=flow.device(device)) num = 1 of_out = flow.eq(input, num) np_out = np.equal(arr, num) test_case.assertTrue(np.array_equal(of_out.numpy(), np_out))