def main(pytorch: bool = False, gpu_id: int = -1):
global CONFIG
fix_random_seed(0)
if gpu_id >= 0:
require_gpu(gpu_id)
print("Set GPU", gpu_id)
backends = {"pytorch": pytorch}
for name, use_backend in backends.items():
if not use_backend:
print(f"Skipping {name}")
continue
set_backend(name, gpu_id)
C = registry.resolve(Config().from_str(CONFIG))
model = C["model"]
X, Y = get_dummy_data(**C["data"])
print("Copy to device")
X = [model.ops.asarray(x) for x in X]
Y = [model.ops.asarray(y) for y in Y]
print("Begin init", len(X))
model.initialize(X=X[:5])
print("Pre-batch")
n_words = sum(len(x) for x in X)
X = [
model.layers[0].predict(batch)
for batch in model.ops.minibatch(16, X)
]
model.layers.pop(0)
print("Start")
start_time = timer()
end_time = timer()
print(name, n_words, end_time - start_time)
def check_transform(transform, in_data, out_data):
model = registry.resolve({"config": {"@layers": transform}})["config"]
input_checker = get_data_checker(in_data)
output_checker = get_data_checker(out_data)
model.initialize(in_data, out_data)
Y, backprop = model(in_data, is_train=True)
output_checker(Y, out_data)
dX = backprop(Y)
input_checker(dX, in_data)
def test_layers_with_residual(name, kwargs, in_data, out_data):
cfg = {"@layers": "residual.v1", "layer": {"@layers": name, **kwargs}}
model = registry.resolve({"config": cfg})["config"]
if "LSTM" in name:
model = with_padded(model)
model.initialize(in_data, out_data)
Y, backprop = model(in_data, is_train=True)
assert_data_match(Y, out_data)
dX = backprop(Y)
assert_data_match(dX, in_data)
def test_optimizer_schedules_from_config(schedule_valid):
lr, lr_next1, lr_next2, lr_next3 = schedule_valid
cfg = {"@optimizers": "Adam.v1", "learn_rate": lr}
optimizer = registry.resolve({"cfg": cfg})["cfg"]
assert optimizer.learn_rate == lr_next1
optimizer.step_schedules()
assert optimizer.learn_rate == lr_next2
optimizer.step_schedules()
assert optimizer.learn_rate == lr_next3
optimizer.learn_rate = 1.0
assert optimizer.learn_rate == 1.0
def test_layers_from_config(name, kwargs, in_data, out_data):
cfg = {"@layers": name, **kwargs}
model = registry.resolve({"config": cfg})["config"]
if "LSTM" in name:
model = with_padded(model)
valid = True
with data_validation(valid):
model.initialize(in_data, out_data)
Y, backprop = model(in_data, is_train=True)
assert_data_match(Y, out_data)
dX = backprop(Y)
assert_data_match(dX, in_data)
def test_layers_batching_all(name, kwargs, in_data, out_data):
cfg = {"@layers": name, **kwargs}
model = registry.resolve({"config": cfg})["config"]
if "expand_window" in name:
return
if "LSTM" in name:
model = with_padded(model)
util_batch_unbatch_list(model, in_data, out_data)
else:
if isinstance(in_data, OPS.xp.ndarray) and in_data.ndim == 2:
if isinstance(out_data, OPS.xp.ndarray) and out_data.ndim == 2:
util_batch_unbatch_array(model, in_data, out_data)
if isinstance(in_data, Ragged):
if isinstance(out_data, OPS.xp.ndarray) and out_data.ndim == 2:
util_batch_unbatch_ragged(model, in_data, out_data)
def test_layers_from_config(name, kwargs, in_data, out_data):
cfg = {"@layers": name, **kwargs}
model = registry.resolve({"config": cfg})["config"]
if "LSTM" in name:
model = with_padded(model)
valid = True
with data_validation(valid):
model.initialize(in_data, out_data)
Y, backprop = model(in_data, is_train=True)
assert_data_match(Y, out_data)
dX = backprop(Y)
assert_data_match(dX, in_data)
# Test that during predictions, no dropout is applied
model._to_ops(NoDropoutOps())
model.predict(in_data)
def main(numpy: bool = False,
pytorch: bool = False,
generic: bool = False,
gpu_id: int = -1):
global CONFIG
fix_random_seed(0)
if gpu_id >= 0:
require_gpu(gpu_id)
print("Set GPU", gpu_id)
backends = {"pytorch": pytorch, "numpy": numpy, "generic": generic}
for name, use_backend in backends.items():
if not use_backend:
print(f"Skipping {name}")
continue
set_backend(name, gpu_id)
print("Getting data")
C = registry.resolve(Config().from_str(CONFIG))
model = C["model"]
X, Y = get_dummy_data(**C["data"])
print("Copy to device")
X = [model.ops.asarray(x) for x in X]
Y = [model.ops.asarray(y) for y in Y]
print("Begin init", len(X))
model.initialize(X=X[:5])
print("Pre-batch")
n_words = sum(len(x) for x in X)
batches = model.ops.multibatch(16, X, Y)
batches = [(model.layers[0].predict(x), y) for x, y in batches]
model.layers.pop(0)
print("Start")
start_time = timer()
total = run_forward(model, [x for x, y in batches])
end_time = timer()
print(name, n_words, total, end_time - start_time)
start_time = timer()
total = run_forward_backward(model, batches)
end_time = timer()
print(name, n_words, total, end_time - start_time)
def test_optimizers_from_config(name):
learn_rate = 0.123
cfg = {"@optimizers": name, "learn_rate": learn_rate}
optimizer = registry.resolve({"config": cfg})["config"]
assert optimizer.learn_rate == learn_rate
