def make_model(dtype="float64"): return Model( layers=( TanhLayer(MA(6, 9, dtype=dtype), zeros(1, 9, dtype=dtype)), TanhLayer(MB(9, 10, dtype=dtype), zeros(1, 10, dtype=dtype)), TanhLayer(MC(10, 8, dtype=dtype), zeros(1, 8, dtype=dtype)), ) )
def test_module_linear_seq_bwd(_backend_fixture): backend = _backend_fixture backend_options = get_backend_options(args, backend) torch.manual_seed(123) inp = torch.Tensor(MA(2, 4, dtype=args.dtype)) model = Linear_Seq(4, 2, 3) target = torch.Tensor([2.5]) # Leave this here, it will be needed again for when we investigate how # to automatically remove duplicates from grads of sequential """ from myia.abstract.aliasing import find_aliases al = find_aliases(model, aliasable=tensor_pytorch_aliasable) print("alias", al) # """ def mse(value, target): diff = value - target return (diff * diff).sum() def cost(model, inp, target): value = model(inp) loss = mse(value, target) return loss pt_cost = cost(model, inp, target) @myia( backend=backend, backend_options=backend_options, alias_tracker=tensor_pytorch_aliasable, ) def step(model, inp, target): _cost, dmodel = value_and_grad(cost, "model")(model, inp, target) return _cost, dmodel loss, grad = step(model, inp, target) pt_cost = cost(model, inp, target) if model.f1.weight.grad is not None: model.f1.weight.grad.data.zero_() if model.f1.bias.grad is not None: model.f1.bias.grad.data.zero_() pt_cost.backward() for n, p in model.named_parameters(): m_p = grad for a in tuple(n.split(".")): m_p = getattr(m_p, a) assert torch.allclose(p.grad.data, m_p)
def test_module_2_layer_mlp_update__to_device(_backend_fixture): backend = _backend_fixture backend_options = get_backend_options(args, backend) torch.manual_seed(123) inp = torch.Tensor(MA(2, 4, dtype=args.dtype)) model = MLP_2_Layers(4, 2, 3) target = torch.Tensor([2.5]) inp = to_device(inp, backend, backend_options) model = to_device(model, backend, backend_options) target = to_device(target, backend, backend_options) def mse(value, target): diff = value - target return sum(diff * diff) def cost(model, inp, target): value = model(inp) loss = mse(value, target) return loss @myia(backend=backend, backend_options=backend_options) def step(model, inp, target): _cost, dmodel = value_and_grad(cost, "model")(model, inp, target) return _cost, model - dmodel loss, model = step(model, inp, target) assert loss == 42.759910583496094 expected_model = [ torch.Tensor([ [1.31208074, 7.52942896, -3.48841572, -2.12911177], [4.61794090, -5.96872425, 3.26280975, -16.41462517], ]), torch.Tensor([-2.16651487, -1.72582722]), torch.Tensor([ [0.39250314, -4.12741709], [3.85490060, -1.67493737], [1.51745880, -5.04526806], ]), torch.Tensor([7.15553093, 6.48739338, 9.37104797]), ] for p, ep in zip(model.parameters(), expected_model): assert torch.allclose(p, ep)
def test_module_2_layer_mlp_bwd(_backend_fixture): backend = _backend_fixture backend_options = get_backend_options(args, backend) torch.manual_seed(123) inp = torch.Tensor(MA(2, 4, dtype=args.dtype)) model = MLP_2_Layers(4, 2, 3) target = torch.Tensor([2.5]) def mse(value, target): diff = value - target return sum(diff * diff) def cost(model, inp, target): value = model(inp) loss = mse(value, target) return loss @myia(backend=backend, backend_options=backend_options) def step(model, inp, target): _cost, dmodel = value_and_grad(cost, "model")(model, inp, target) return _cost, dmodel loss, grad = step(model, inp, target) assert loss == 42.759910583496094 expected_grads = [ torch.Tensor([ [-1.51596880, -7.51286650, 3.24008656, 2.31766868], [-5.04396868, 6.33524609, -3.62623000, 16.01710510], ]), torch.Tensor([1.85057139, 1.95227396]), torch.Tensor([ [-0.65377355, 4.39202595], [-4.45504284, 1.24591899], [-1.77709150, 4.90630770], ]), torch.Tensor([-7.69495678, -6.02438641, -9.53780556]), ] for g, eg in zip(grad.parameters(), expected_grads): assert torch.allclose(g, eg)
def test_module_2_layer_mlp_seq_fwd(_backend_fixture): backend = _backend_fixture backend_options = get_backend_options(args, backend) torch.manual_seed(123) inp = torch.Tensor(MA(2, 4, dtype=args.dtype)) model = MLP_2_Layers_Seq(4, 2, 3) @myia(backend=backend, backend_options=backend_options) def step(model, inp): return model(inp) output = step(model, inp) output_expected = torch.Tensor([ [-0.55702960, 0.85518718, 0.13796528], [-0.67215765, -0.09247651, -0.38900381], ]) assert torch.allclose(output, output_expected)
def test_module_matmul_bwd(_backend_fixture): backend = _backend_fixture backend_options = get_backend_options(args, backend) torch.manual_seed(123) inp = torch.Tensor(MA(2, 4, dtype=args.dtype)) model = Tiny(4, 3) target = torch.Tensor([2.5]) def mse(value, target): diff = value - target return sum(diff * diff) def cost(model, inp, target): value = model(inp) loss = mse(value, target) return loss @myia(backend=backend, backend_options=backend_options) def step(model, inp, target): _cost, dmodel = value_and_grad(cost, "model")(model, inp, target) return _cost, dmodel loss, grad = step(model, inp, target) loss_expected = 161.0585479736328 assert np.isclose(loss, loss_expected) grad_expected = torch.Tensor([ [-15.79940414, 34.22111893, -16.79670525], [82.41101074, -39.50494003, 100.31848145], [-40.12714767, 23.00367165, -48.50212097], [66.75647736, -107.74736023, 74.33836365], ]) assert torch.allclose(grad.W, grad_expected)
def test_module_matmul_update(_backend_fixture): backend = _backend_fixture backend_options = get_backend_options(args, backend) torch.manual_seed(123) inp = torch.Tensor(MA(2, 4, dtype=args.dtype)) model = Tiny(4, 3) target = torch.Tensor([2.5]) def mse(value, target): diff = value - target return sum(diff * diff) def cost(model, inp, target): value = model(inp) loss = mse(value, target) return loss @myia(backend=backend, backend_options=backend_options) def step(model, inp, target): _cost, dmodel = value_and_grad(cost, "model")(model, inp, target) return _cost, model - dmodel loss, model = step(model, inp, target) loss_expected = 161.0585479736328 assert np.isclose(loss, loss_expected) model_expected = torch.Tensor([ [15.42187691, -34.19045258, 16.33688736], [-82.06187439, 38.71609116, -99.63981628], [39.45422363, -23.73973656, 47.91710663], [-66.33718109, 107.40527344, -73.99190521], ]) assert torch.allclose(model.W, model_expected)
def format(): return {} def f(): raise ValueError("test") register_backend(name, f, format) with pytest.raises(LoadingError): load_backend(name) del _backends[name] @run(MA(2, 3)) def test_reshape2(x): return reshape(x, (6, )) @mt(run(MA(2, 3)), run(MA(1, 3))) def test_array_reduce(x): return array_reduce(scalar_add, x, (1, 3)) @run(MA(2, 3)) def test_array_reduce2(x): return array_reduce(scalar_add, x, (3, )) @run_gpu(MA(1, 1))
}, result=InferenceError), # Generic broadcasting tests infer_scalar([f64], f64, result=[f64]), infer_scalar([[f64]], [[f64]], result=[[f64]]), infer_scalar((i64, i64), i64, result=(i64, i64)), infer_scalar(i64, (i64, i64), result=(i64, i64)), infer_scalar(Point(i64, i64), i64, result=Point(i64, i64)), # Various errors infer_scalar((i64, i64), (i64, i64, i64), result=InferenceError), infer_scalar(Point(i64, i64), Point3D(i64, i64, i64), result=InferenceError), infer_scalar((i64, i64), [i64], result=InferenceError), # Full tests run(MA(2, 3), MB(2, 3)), run(Point(1, 2), Point(3, 4)), run((MA(2, 3), 7.5, MB(1, 3)), 3.5), ) def test_hyper_map(x, y): return hyper_map(scalar_add, x, y) @mt( infer_scalar((i64, f64), (i64, f64), result=InferenceError), infer_scalar([f64], f64, result=[f64]), ) def test_hyper_map_notuple(x, y): return hyper_map_notuple(scalar_add, x, y)
def test_optim_setitem(_backend_fixture): from myia.abstract import macro from myia.operations import primitives as P from myia.ir import sexp_to_node from myia.lib import setter_from_getter def update_sgd(p, g): return p - 0.01 * g @macro async def update(info, model_ref, dmodel_ref, update_rule_ref): new_model = model_ref.node dmodel = dmodel_ref.node update_rule = update_rule_ref.node p = new_model g = dmodel p = (P.record_getitem, p, "W") g = (P.record_getitem, g, "W") p_node = sexp_to_node(p, info.graph) g_node = sexp_to_node(g, info.graph) pn = info.graph.apply(update_rule, p_node, g_node) new_model = sexp_to_node(setter_from_getter(p, pn), info.graph) return new_model backend = _backend_fixture backend_options = get_backend_options(args, backend) torch.manual_seed(123) inp = torch.Tensor(MA(2, 4, dtype=args.dtype)) model = Tiny(4, 3) target = torch.Tensor([2.5]) def mse(value, target): diff = value - target return sum(diff * diff) def cost(model, inp, target): value = model(inp) loss = mse(value, target) return loss @myia(backend=backend, backend_options=backend_options) def step(model, inp, target): _cost, dmodel = value_and_grad(cost, "model")(model, inp, target) return _cost, update(model, dmodel, update_sgd) loss, model_new = step(model, inp, target) expected_loss = torch.Tensor([161.05856323242188]) assert torch.allclose(loss, expected_loss) expected_param = torch.Tensor([ [-0.21953332, -0.31154382, -0.29184943], [-0.47497076, -0.39380032, -0.32451797], [-0.27165186, -0.96610248, -0.09999254], [-0.24826682, 0.73539025, -0.39692938], ]) assert torch.allclose(model_new.W, expected_param)
def __init__(self, in_f, out_f): super(Tiny, self).__init__() self.W = nn.Parameter(torch.Tensor(MA(in_f, out_f, dtype=args.dtype))) self.W = nn.init.xavier_uniform_(self.W)
@mt( run(-2.7, result=-2), run(7.8, result=7), run(np.float16(7.8), result=7), run(np.float32(7.8), result=7), run(np.float64(7.8), result=7), ) def test_trunc(a): return math.trunc(a) @mt( # sin seems not supported for pytorch/CPU/float16 run(MA(3, 3, dtype="float32"), result=np.sin(MA(3, 3, dtype="float32"))), run(MA(3, 3, dtype="float64"), result=np.sin(MA(3, 3, dtype="float64"))), ) def test_elemwise_sin(a): return np.sin(a) @mt( # cos seems not supported for pytorch/CPU/float16 run(MA(3, 3, dtype="float32"), result=np.cos(MA(3, 3, dtype="float32"))), run(MA(3, 3, dtype="float64"), result=np.cos(MA(3, 3, dtype="float64"))), ) def test_elemwise_cos(a): return np.cos(a)
@mark.xfail(reason="A DummyInferrer ends up being called") @gradient(4.5, 6.7, backend=backend_all) def test_closures_in_tuples(x, y): def f(): return x * y def g(): return x + y tup = f, g ff, gg = tup return ff() + gg() @gradient(MA(2, 3), MB(2, 3), backend=backend_all) def test_array_operations(xs, ys): div = array_map(scalar_div, xs, ys) sm = array_reduce(scalar_add, div, ()) return array_to_scalar(sm) @gradient(3.1, 7.6, backend=backend_all) def test_array_operations_distribute(x, y): xs = distribute(scalar_to_array(x, AA), (4, 3)) ys = distribute(scalar_to_array(y, AA), (4, 3)) div = array_map(scalar_div, xs, ys) sm = array_reduce(scalar_add, div, ()) return array_to_scalar(sm)
args = Args() class Tiny(nn.Module): def __init__(self, in_f, out_f): super(Tiny, self).__init__() self.W = nn.Parameter(torch.Tensor(MA(in_f, out_f, dtype=args.dtype))) self.W = nn.init.xavier_uniform_(self.W) def forward(self, input): return input @ self.W @run(Tiny(4, 3), torch.tensor(MA(2, 4, dtype="float32"))) def test_module_matmul_fwd(model, inp): return model(inp) def test_module_matmul_bwd(_backend_fixture): backend = _backend_fixture backend_options = get_backend_options(args, backend) torch.manual_seed(123) inp = torch.Tensor(MA(2, 4, dtype=args.dtype)) model = Tiny(4, 3) target = torch.Tensor([2.5]) def mse(value, target):
return tagged(z) @mt(run("hey", 2), run("idk", 5)) def test_string_eq(s, x): if s == "idk": x = x + 1 return x @mt(run("hey", 2), run("idk", 5)) def test_string_ne(s, x): if s != "idk": x = x + 1 return x @run("hey") def test_string_return(s): return s @run(MA(4, 5)) def test_array_getitem(x): return array_getitem(x, (0, 1), (3, 5), (2, 3)) @run(MA(4, 5), MB(2, 2)) def test_array_setitem(x, v): return array_setitem(x, (0, 1), (3, 5), (2, 3), v)
from myia.lib import InferenceError from myia.operations import dtype, scalar_cast from myia.testing.common import MA, Ty, af32_of, f32, i64, to_abstract_test from myia.testing.multitest import infer, mt, run @mt( infer(i64, result=InferenceError), infer(af32_of(4, 5), result=Ty(to_abstract_test(f32))), ) def test_dtype(arr): return dtype(arr) @mt(infer(af32_of(4, 5), i64, result=f32), run(MA(2, 3), 7, result=7.0)) def test_cast_to_dtype(arr, x): return scalar_cast(x, dtype(arr))
import pytest from myia.lib import InferenceError from myia.operations import einsum from myia.testing.common import MA, MB from myia.testing.multitest import infer, mt, run_debug @run_debug(MA(1, 4)[0]) def test_einsum_view1d(a): return einsum("i", a) @run_debug(MA(1, 4)[0]) def test_einsum_sum1d(a): return einsum("i->", a) @run_debug(MA(4, 1)[0], MB(4, 1)[0]) def test_einsum_elemwise1d(a, b): return einsum("i,i->i", a, b) @run_debug(MA(3, 3)[0], MA(3, 3)[1]) def test_einsum_inner(a, b): return einsum("j,j", a, b) @run_debug(MA(3, 3)[0], MA(3, 3)[1]) def test_einsum_outer(a, b): return einsum("i,k->ik", a, b)
fwd_and_bwd_no_numpy_compat = _fwd_and_bwd.configure(backend=backend_all, numpy_compat=False) run = basic_run.configure(numpy_compat=True) run_no_numpy_compat = basic_run.configure(numpy_compat=False) # THIS TEST ALL OPS that are in dir of "torch" or "torch.tensor" # all_torch_ops = dir(torch) # all_torch_tensor_ops = dir(torch.Tensor([5.49670])) torch.manual_seed(123) single_tensor_arg_tests = ( fwd_and_bwd(nn.Parameter(torch.Tensor([2.1]).reshape(()))), fwd_and_bwd(nn.Parameter(torch.Tensor([2.1]))), fwd_and_bwd(nn.Parameter(torch.Tensor([-2.2]))), fwd_and_bwd(nn.Parameter(torch.Tensor(MA(2, 3)))), ) @mt(*single_tensor_arg_tests) def test_torch_abs(x): return torch.abs(x) @mt(*single_tensor_arg_tests) def test_torch_exp(x): return torch.exp(x) @mt(*single_tensor_arg_tests) def test_torch_log(x):