def test__convert_arg_init_AbstractScalar(backend_opt):
b = backend_opt
model1 = 3.0
m1 = to_device(model1, b)
assert isinstance(m1, float)
assert m1 == 3.0
model2 = 14
m2 = to_device(model2, b)
assert isinstance(m2, int)
assert m2 == 14
def test__convert_arg_init_AbstractList(backend_opt):
b = backend_opt
model = [91.0, 51.0]
m = to_device(model, b)
assert isinstance(m, list)
assert isinstance(m[0], float)
assert isinstance(m[1], float)
assert m[0] == 91.0
assert m[1] == 51.0
model2 = []
m2 = to_device(model2, b)
assert m2 == []
def test_module_2_layer_mlp_update__to_device(_backend_fixture):
backend = _backend_fixture
backend_options = get_backend_options(args, backend)
torch.manual_seed(123)
inp = torch.Tensor(MA(2, 4, dtype=args.dtype))
model = MLP_2_Layers(4, 2, 3)
target = torch.Tensor([2.5])
inp = to_device(inp, backend, backend_options)
model = to_device(model, backend, backend_options)
target = to_device(target, backend, backend_options)
def mse(value, target):
diff = value - target
return sum(diff * diff)
def cost(model, inp, target):
value = model(inp)
loss = mse(value, target)
return loss
@myia(backend=backend, backend_options=backend_options)
def step(model, inp, target):
_cost, dmodel = value_and_grad(cost, "model")(model, inp, target)
return _cost, model - dmodel
loss, model = step(model, inp, target)
assert loss == 42.759910583496094
expected_model = [
torch.Tensor([
[1.31208074, 7.52942896, -3.48841572, -2.12911177],
[4.61794090, -5.96872425, 3.26280975, -16.41462517],
]),
torch.Tensor([-2.16651487, -1.72582722]),
torch.Tensor([
[0.39250314, -4.12741709],
[3.85490060, -1.67493737],
[1.51745880, -5.04526806],
]),
torch.Tensor([7.15553093, 6.48739338, 9.37104797]),
]
for p, ep in zip(model.parameters(), expected_model):
assert torch.allclose(p, ep)
def run_helper(epochs, n, batch_size, layer_sizes):
"""Run a model with the specified layer sizes on n random batches.
Arguments:
epochs: How many epochs to run.
n: Number of training batches to generate.
batch_size: Number of samples per batch.
layer_sizes: Sizes of the model's layers.
"""
layers = []
for W, b in mlp_parameters(*layer_sizes):
layers.append(Linear(W, b))
layers.append(Tanh())
model = Sequential(tuple(layers))
model = to_device(model, backend, backend_options)
data = generate_data(n, batch_size, layer_sizes[0], layer_sizes[-1])
for _ in range(epochs):
costs = []
t0 = time.time()
for inp, target in data:
cost, model = step(model, inp, target, lr)
costs.append(cost)
costs = [float(c.from_device()) for c in costs]
c = sum(costs) / n
t = time.time() - t0
print(f'Cost: {c:15.10f}\tTime: {t:15.10f}')
def test__convert_arg_init_AbstractTuple(backend_opt):
b = backend_opt
model = (9, 5.0)
m = to_device(model, b)
assert isinstance(m, tuple)
assert isinstance(m[0], int)
assert isinstance(m[1], float)
assert m[0] == 9
assert m[1] == 5.0
def test__convert_arg_init_AbstractClass(backend_opt):
b = backend_opt
@dataclass(frozen=True)
class A():
s: 'scalar number'
def apply(self, input):
"""Apply the layer."""
return (input, self.s)
model = A(2.0)
m = to_device(model, b)
assert isinstance(m, A)
assert isinstance(m.s, float)
assert m.s == 2.0
def generate_data(n,
batch_size,
input_size,
target_size,
sequence_size,
backend,
backend_options,
*,
seed=91):
"""Generate inputs and targets.
Generates n batches of samples of size input_size, matched with
a single target.
"""
R = RandomState(seed=seed)
"""
return ([to_device(param(R, batch_size, input_size), backend, backend_options) for i in range(sequence_size)],
to_device(param(R, batch_size, target_size), backend, backend_options))
#"""
return to_device(([param(R, batch_size, input_size) for i in range(sequence_size)], param(R, batch_size, target_size)), \
backend, backend_options)
def test_to_canonical():
def _convert(data, typ):
return to_canonical(data, to_abstract_test(typ))
# Leaves
assert _convert(True, Bool) is True
assert _convert(False, Bool) is False
assert _convert(10, i64) == 10
assert _convert(1.5, f64) == 1.5
assert _convert([], []) == ()
with pytest.raises(TypeError):
_convert([], [f64])
with pytest.raises(TypeError):
_convert([1, 2], [])
# Class -> Tuple conversion
pt = Point(1, 2)
pt3 = Point3D(1, 2, 3)
assert list(_convert(pt, Point(i64, i64))) == [1, 2]
with pytest.raises(TypeError):
_convert((1, 2), Point(i64, i64))
assert list(_convert(
(pt, pt), (Point(i64, i64), Point(i64, i64)))) == [(1, 2), (1, 2)]
li = _convert([1], [i64])
assert (isinstance(li, tuple) and li[0] == 1
and isinstance(li[1], TaggedValue) and li[1].value == ())
# Arrays
fmat = np.ones((5, 8))
imat = np.ones((5, 8), dtype='int32')
assert _convert(fmat, af64_of(5, 8)) is fmat
assert _convert(imat, ai32_of(5, 8)) is imat
with pytest.raises(TypeError):
_convert(imat, ai64_of(5, 8))
with pytest.raises(TypeError):
_convert(imat, ai32_of(4, 8))
# Misc errors
with pytest.raises(TypeError):
_convert(10, f64)
with pytest.raises(TypeError):
_convert(1.5, i64)
with pytest.raises(TypeError):
_convert(10, (i64, i64))
with pytest.raises(TypeError):
_convert((1, ), (i64, i64))
with pytest.raises(TypeError):
_convert((1, 2, 3), (i64, i64))
with pytest.raises(TypeError):
_convert((1, 2, 3), [i64])
with pytest.raises(TypeError):
_convert(pt3, Point(i64, i64))
with pytest.raises(TypeError):
_convert(10, ai64_of())
with pytest.raises(TypeError):
_convert(10, ai64_of())
with pytest.raises(TypeError):
_convert(1, Bool)
with pytest.raises(TypeError):
_convert(1, D(x=i64))
with pytest.raises(TypeError):
_convert({'x': 2.0}, D(x=i64))
with pytest.raises(TypeError):
_convert({'x': 2.0, 'y': 1}, D(x=i64))
with pytest.raises(TypeError):
_convert({'y': 2.0}, D(x=i64))
with pytest.raises(TypeError):
_convert('x', 1.0)
with pytest.raises(TypeError):
_convert(1.0, to_abstract_test('x'))
with pytest.raises(TypeError):
_convert(1.0, to_abstract_test(HandleInstance(1.0)))
v = to_device(22, None)
with pytest.raises(TypeError):
_convert(v, f64)
def run_helper(args, n, batch_size, layer_sizes):
"""Run a model with the specified layer sizes on n random batches.
The first layer is an LSTM layer, the rest are linear+tanh.
Arguments:
iters: How many iters to run.
n: Number of training batches to generate.
batch_size: Number of samples per batch.
layer_sizes: Sizes of the model's layers.
"""
i = 0
j = 0
layers = []
lstmp, *linp = lstm_parameters(*layer_sizes, batch_size=batch_size)
layers.append(LSTMLayer(*lstmp))
for W, b in linp:
layers.append(Linear(W, b))
layers.append(Tanh())
model = Sequential(tuple(layers))
if args.break_bm:
print("break_bm")
return
model = to_device(model, backend, backend_options)
if args.break_mod_on_d:
print("break_mod_on_d")
return
lr = getattr(numpy, dtype)(args.lr)
lr = to_device(lr, backend, backend_options)
inp, target = generate_data(n, batch_size, layer_sizes[0], layer_sizes[-1],
args.timesteps, backend, backend_options)
if args.break_mod_on_d_and_gen_data:
print("break_mod_on_d_and_gen_data")
return
times = []
if args.cuda_sync:
import torch
for _ in range(args.iters):
costs = []
if args.cuda_sync:
torch.cuda.synchronize()
t0 = time.time()
#cost, model = step(model, inp, target, lr)
cost, model = step(model, inp, target)
if args.break_cr1:
print("break_cr1", time.time() - t0)
break
if args.break_cr2 and i == 1:
print("break_cr2", time.time() - t0)
break
if args.print_all_iters:
print(i, cost.array)
if args.cuda_sync:
torch.cuda.synchronize()
t_diff = time.time() - t0
if args.break_cr1 or args.break_cr2:
break
i += 1
if args.print_all_iters:
print(i, t_diff, cost)
if i > args.warmup: # first 99 steps are warmup
times.append(t_diff)
if not args.break_cr1 and not args.break_cr2:
print("times", times)
print("\nMyia LSTM Stats")
print("Avg Time: ", sum(times) / len(times))
print("Min Time: ", min(times))
print("Max Time: ", max(times))
if args.save_txt:
filename = ""
for arg in vars(args):
filename += str(arg)
filename += "="
filename += str(getattr(args, arg))
filename += "."
str_out = "Myia LSTM Stats\n" + \
"Avg Time: " + str(sum(times)/len(times)) + \
"Min Time: " + str(min(times)) + \
"Max Time: " + str(max(times)) + \
"Times" + str(times)
f = open(filename + ".txt", "w+")
f.write(str_out)
f.close()
def convert_args(self, args):
return tuple(to_device(arg, self.backend) for arg in args)
def test__convert_arg_init_AbstractArray(backend_opt):
b = backend_opt
m = to_device(MA(2, 3), b)
assert isinstance(m, ArrayWrapper)
np.testing.assert_allclose(b.to_numpy(m.array), MA(2, 3))
def run_model(args):
i = 0
j = 0
layers = []
for W, b in mlp_parameters():
layers.append(Linear(W, b))
layers.append(Tanh())
model = Sequential(tuple(layers))
model = to_device(model, backend, backend_options)
inp, target = mlp_data()
inp = to_device(inp, backend, backend_options)
target = to_device(target, backend, backend_options)
lr = getattr(numpy, dtype)(args.lr)
lr = to_device(lr, backend, backend_options)
if args.break_mod_on_d_and_gen_data:
print("break_mod_on_d_and_gen_data")
return
times = []
if args.cuda_sync:
import torch
for _ in range(args.iters):
if args.cuda_sync:
torch.cuda.synchronize()
t0 = time.time()
cost, model = step(model, inp, target)
#cost, model = step(model, inp, target, lr)
if args.cuda_sync:
torch.cuda.synchronize()
t_diff = time.time() - t0
if args.break_cr1:
print("break_cr1", time.time() - t0)
break
i += 1
if args.break_cr2 and i == 2:
print("break_cr2", time.time() - t0)
break
#"""
if args.print_all_iters:
print(i, t_diff, cost.array)
#"""
if i > args.warmup: # first 99 steps are warmup
times.append(t_diff)
print("times", times)
print("\nMyia MLP Stats")
print("Avg Time: ", sum(times) / len(times))
print("Min Time: ", min(times))
print("Max Time: ", max(times))
if args.save_txt:
filename = ""
for arg in vars(args):
filename += str(arg)
filename += "="
filename += str(getattr(args, arg))
filename += "."
str_out = "Myia MLP Stats\n" + \
"Avg Time: " + str(sum(times)/len(times)) + \
"Min Time: " + str(min(times)) + \
"Max Time: " + str(max(times)) + \
"Times" + str(times)
f = open(filename + ".txt", "w+")
f.write(str_out)
f.close()
return x
layer_sizes = (4, 24, 2)
mlp = mlp_parameters(*layer_sizes)
model = Sequential(
(
Linear(mlp[0][0], mlp[0][1]),
Tanh(),
Linear(mlp[1][0], mlp[1][1]),
Softmax(1),
)
)
model = to_device(model, backend, backend_options, broaden=False)
use_cuda = torch.cuda.is_available()
FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor
@myia(
backend=backend,
backend_options=backend_options,
specialize_values=["model"],
)
def step_eval(model, data):
output = model.apply(data)
return output
