def test_script_mode(user_script, training_fn, capture_error):
channel = test.Channel.create(name="training")
features = [1, 2, 3, 4]
labels = [0, 1, 0, 1]
np.savez(os.path.join(channel.path, "training_data"),
features=features,
labels=labels)
module = test.UserModule(test.File(name="user_script.py",
data=user_script))
hyperparameters = dict(
training_data_file=os.path.join(channel.path, "training_data.npz"),
sagemaker_program="user_script.py",
epochs=10,
batch_size=64,
model_dir=env.model_dir,
)
test.prepare(user_module=module,
hyperparameters=hyperparameters,
channels=[channel])
assert execute_an_wrap_exit(
training_fn, capture_error=capture_error) == trainer.SUCCESS_CODE
model_path = os.path.join(env.model_dir, "saved_model")
model = fake_ml_framework.Model.load(model_path)
assert model.epochs == 10
assert model.batch_size == 64
assert model.loss == "elastic"
assert model.optimizer == "SGD"
def test_script_mode_client_import_error(training_fn, capture_error):
channel = test.Channel.create(name="training")
requirements_file = test.File("requirements.txt", "invalid/module")
user_script = test.File(name="user_script", data="invalid/module")
module = test.UserModule(user_script).add_file(setup_file).add_file(
requirements_file).upload()
hyperparameters = dict(sagemaker_program="user_script")
test.prepare(user_module=module,
hyperparameters=hyperparameters,
channels=[channel])
with pytest.raises(errors.InstallModuleError) as e:
training_fn(capture_error)
message = str(e.value)
assert "InstallModuleError:" in message
# fmt: off
if capture_error:
assert "Invalid requirement" in message
assert "It looks like a path" in message
def test_training_framework(user_script):
with pytest.raises(ImportError):
importlib.import_module(modules.DEFAULT_MODULE_NAME)
channel = test.Channel.create(name='training')
features = [1, 2, 3, 4]
labels = [0, 1, 0, 1]
np.savez(os.path.join(channel.path, 'training_data'),
features=features,
labels=labels)
module = test.UserModule(test.File(name='user_script.py',
data=user_script))
hyperparameters = dict(training_data_file='training_data.npz',
sagemaker_program='user_script.py',
epochs=10,
batch_size=64,
optimizer='Adam')
test.prepare(user_module=module,
hyperparameters=hyperparameters,
channels=[channel])
assert execute_an_wrap_exit(framework_training_fn) == trainer.SUCCESS_CODE
model_path = os.path.join(env.model_dir, 'saved_model')
model = fake_ml_framework.Model.load(model_path)
assert model.epochs == 10
assert model.batch_size == 64
assert model.optimizer == 'Adam'
def test_trainer_report_success(user_script, sagemaker_program):
channel = test.Channel.create(name="training")
features = [1, 2, 3, 4]
labels = [0, 1, 0, 1]
np.savez(os.path.join(channel.path, "training_data"),
features=features,
labels=labels)
module = test.UserModule(
test.File(name=sagemaker_program, data=user_script))
hyperparameters = dict(
training_data_file="training_data.npz",
sagemaker_program=sagemaker_program,
epochs=10,
batch_size=64,
)
test.prepare(user_module=module,
hyperparameters=hyperparameters,
channels=[channel])
assert execute_an_wrap_exit(trainer.train) == trainer.SUCCESS_CODE
model_path = os.path.join(env.model_dir, "saved_model")
model = fake_ml_framework.Model.load(model_path)
assert model.epochs == 10
assert model.batch_size == 64
assert model.optimizer == "SGD"
assert os.path.exists(os.path.join(env.output_dir, "success"))
def test_script_mode_local_directory(user_script, training_fn, capture_error, tmpdir):
channel = test.Channel.create(name='training')
features = [1, 2, 3, 4]
labels = [0, 1, 0, 1]
np.savez(os.path.join(channel.path, 'training_data'), features=features, labels=labels)
tmp_code_dir = str(tmpdir)
module = test.UserModule(test.File(name='user_script.py', data=user_script))
module.create_tmp_dir_with_files(tmp_code_dir)
hyperparameters = dict(training_data_file=os.path.join(channel.path, 'training_data.npz'),
sagemaker_program='user_script.py', sagemaker_submit_directory=tmp_code_dir,
epochs=10, batch_size=64, model_dir=env.model_dir)
test.prepare(user_module=module, hyperparameters=hyperparameters, channels=[channel], local=True)
assert execute_an_wrap_exit(training_fn, capture_error=capture_error) == trainer.SUCCESS_CODE
model_path = os.path.join(env.model_dir, 'saved_model')
model = fake_ml_framework.Model.load(model_path)
assert model.epochs == 10
assert model.batch_size == 64
assert model.loss == 'elastic'
assert model.optimizer == 'SGD'
def test_training_framework_network_isolation(user_script, capture_error):
with pytest.raises(ImportError):
importlib.import_module(modules.DEFAULT_MODULE_NAME)
channel_train = test.Channel.create(name='training')
channel_code = test.Channel.create(name='code')
features = [1, 2, 3, 4]
labels = [0, 1, 0, 1]
np.savez(os.path.join(channel_train.path, 'training_data'), features=features, labels=labels)
with open(os.path.join(channel_code.path, 'user_script.py'), 'w') as f:
f.write(user_script)
f.close()
module = test.UserModule(test.File(name='user_script.py', data="")) # dummy module for hyperparameters
submit_dir = env.input_dir + '/data/code'
hyperparameters = dict(training_data_file='training_data.npz', sagemaker_program='user_script.py',
sagemaker_submit_directory=submit_dir, epochs=10, batch_size=64, optimizer='Adam')
test.prepare(user_module=module, hyperparameters=hyperparameters, channels=[channel_train, channel_code],
local=True)
assert execute_an_wrap_exit(framework_training_fn) == trainer.SUCCESS_CODE
model_path = os.path.join(env.model_dir, 'saved_model')
model = fake_ml_framework.Model.load(model_path)
assert model.epochs == 10
assert model.batch_size == 64
assert model.optimizer == 'Adam'
def test_trainer_report_failure():
channel = test.Channel.create(name="training")
features = [1, 2, 3, 4]
labels = [0, 1, 0, 1]
np.savez(os.path.join(channel.path, "training_data"), features=features, labels=labels)
module = test.UserModule(
test.File(name="user_script.py", data=USER_SCRIPT_WITH_EXCEPTION)
).add_file(setup_file)
hyperparameters = dict(
training_data_file="training_data.npz",
sagemaker_program="user_script.py",
epochs=10,
batch_size=64,
)
test.prepare(user_module=module, hyperparameters=hyperparameters, channels=[channel])
os.environ["SAGEMAKER_TRAINING_MODULE"] = "test.functional.simple_framework:train"
assert execute_an_wrap_exit(trainer.train) == errno.ENOENT
failure_file = os.path.join(environment.output_dir, "failure")
assert os.path.exists(failure_file)
message = failure_message()
assert message.startswith("Framework Error:")
assert "No such file or directory" in message
def test_training_framework(user_script, capture_error):
with pytest.raises(ImportError):
importlib.import_module(modules.DEFAULT_MODULE_NAME)
channel = test.Channel.create(name="training")
features = [1, 2, 3, 4]
labels = [0, 1, 0, 1]
np.savez(os.path.join(channel.path, "training_data"), features=features, labels=labels)
file = test.File(name="user_script.py", data=user_script)
module = test.UserModule(file).add_file(setup_file)
hyperparameters = dict(
training_data_file="training_data.npz",
sagemaker_program="user_script.py",
epochs=10,
batch_size=64,
optimizer="Adam",
)
test.prepare(user_module=module, hyperparameters=hyperparameters, channels=[channel])
assert execute_an_wrap_exit(framework_training_fn) == trainer.SUCCESS_CODE
model_path = os.path.join(environment.model_dir, "saved_model")
model = fake_ml_framework.Model.load(model_path)
assert model.epochs == 10
assert model.batch_size == 64
assert model.optimizer == "Adam"
def test_trainer_report_failure():
channel = test.Channel.create(name='training')
features = [1, 2, 3, 4]
labels = [0, 1, 0, 1]
np.savez(os.path.join(channel.path, 'training_data'),
features=features,
labels=labels)
module = test.UserModule(
test.File(name='user_script.py', data=USER_SCRIPT_WITH_EXCEPTION))
hyperparameters = dict(training_data_file='training_data.npz',
sagemaker_program='user_script.py',
epochs=10,
batch_size=64)
test.prepare(user_module=module,
hyperparameters=hyperparameters,
channels=[channel])
os.environ[
'SAGEMAKER_TRAINING_MODULE'] = 'test.functional.simple_framework:train'
assert execute_an_wrap_exit(trainer.train) == errno.ENOENT
failure_file = os.path.join(env.output_dir, 'failure')
assert os.path.exists(failure_file)
message = failure_message()
assert message.startswith('framework error:')
assert 'No such file or directory' in message
def test_script_mode(user_script):
channel = test.Channel.create(name='training')
features = [1, 2, 3, 4]
labels = [0, 1, 0, 1]
np.savez(os.path.join(channel.path, 'training_data'),
features=features,
labels=labels)
module = test.UserModule(test.File(name='user_script.py',
data=user_script))
hyperparameters = dict(training_data_file=os.path.join(
channel.path, 'training_data.npz'),
sagemaker_program='user_script.py',
epochs=10,
batch_size=64,
model_dir=env.model_dir)
test.prepare(user_module=module,
hyperparameters=hyperparameters,
channels=[channel])
assert execute_an_wrap_exit(
framework_training_with_script_mode_fn) == trainer.SUCCESS_CODE
model_path = os.path.join(env.model_dir, 'saved_model')
model = fake_ml_framework.Model.load(model_path)
assert model.epochs == 10
assert model.batch_size == 64
assert model.loss == 'elastic'
assert model.optimizer == 'SGD'
def test_parameter_server():
module = test.UserModule(test.File(name='user_script.py', data=PARAMETER_SERVER_SCRIPT))
hyperparameters = dict(sagemaker_program='user_script.py')
test.prepare(user_module=module, hyperparameters=hyperparameters, channels=[test.Channel.create(name='training')])
training_env = sagemaker_containers.training_env()
process = entry_point.run(training_env.module_dir, training_env.user_entry_point,
training_env.to_cmd_args(), training_env.to_env_vars(), wait=False)
# confirm the ps process is still hanging
assert process.poll() is None
process.kill()
def test_script_mode_client_error():
channel = test.Channel.create(name="training")
module = test.UserModule(test.File(name="user_script.py", data=USER_MODE_SCRIPT_WITH_ERROR))
hyperparameters = dict(sagemaker_program="user_script.py")
test.prepare(user_module=module, hyperparameters=hyperparameters, channels=[channel])
with pytest.raises(errors.ExecuteUserScriptError) as e:
framework_training_with_script_mode_fn(capture_error=True)
message = str(e.value)
assert "ExecuteUserScriptError" in message
assert "ZeroDivisionError" in message
def test_script_mode_client_error(training_fn, capture_error):
channel = test.Channel.create(name='training')
module = test.UserModule(test.File(name='user_script.py', data=USER_MODE_SCRIPT_WITH_ERROR))
hyperparameters = dict(sagemaker_program='user_script.py')
test.prepare(user_module=module, hyperparameters=hyperparameters, channels=[channel])
with pytest.raises(errors.ExecuteUserScriptError) as e:
training_fn(capture_error)
message = str(e.value)
assert 'ExecuteUserScriptError' in message
if capture_error:
assert 'ZeroDivisionError' in message
def test_script_mode_client_import_error():
channel = test.Channel.create(name='training')
requirements_file = test.File('requirements.txt', '42/0')
user_script = test.File(name='user_script.py', data='42/0')
module = test.UserModule(user_script).add_file(requirements_file).upload()
hyperparameters = dict(sagemaker_program='user_script.py')
test.prepare(user_module=module,
hyperparameters=hyperparameters,
channels=[channel])
with pytest.raises(errors.InstallModuleError) as e:
framework_training_with_script_mode_fn()
message = str(e.value)
assert 'InstallModuleError:' in message
def test_script_mode_client_import_error(training_fn, capture_error):
channel = test.Channel.create(name='training')
requirements_file = test.File('requirements.txt', '42/0')
user_script = test.File(name='user_script', data='42/0')
module = test.UserModule(user_script).add_file(setup_file).add_file(requirements_file).upload()
hyperparameters = dict(sagemaker_program='user_script')
test.prepare(user_module=module, hyperparameters=hyperparameters, channels=[channel])
with pytest.raises(errors.InstallModuleError) as e:
training_fn(capture_error)
message = str(e.value)
assert 'InstallModuleError:' in message
if capture_error:
assert "Invalid requirement: \'42/0\'" in message
assert "It looks like a path. File \'42/0\' does not exist." in message
def test_script_mode_local_directory(tmpdir):
channel = test.Channel.create(name="training")
features = [1, 2, 3, 4]
labels = [0, 1, 0, 1]
np.savez(os.path.join(channel.path, "training_data"),
features=features,
labels=labels)
tmp_code_dir = str(tmpdir)
module = test.UserModule(
test.File(name="user_script.py", data=USER_MODE_SCRIPT))
module.create_tmp_dir_with_files(tmp_code_dir)
hyperparameters = dict(
training_data_file=os.path.join(channel.path, "training_data.npz"),
sagemaker_program="user_script.py",
sagemaker_submit_directory=tmp_code_dir,
epochs=10,
batch_size=64,
model_dir=environment.model_dir,
)
test.prepare(user_module=module,
hyperparameters=hyperparameters,
channels=[channel],
local=True)
assert (execute_an_wrap_exit(framework_training_with_script_mode_fn,
capture_error=False) == trainer.SUCCESS_CODE)
model_path = os.path.join(environment.model_dir, "saved_model")
model = fake_ml_framework.Model.load(model_path)
assert model.epochs == 10
assert model.batch_size == 64
assert model.loss == "elastic"
assert model.optimizer == "SGD"
def test_trainer_report_success(user_script):
with pytest.raises(ImportError):
importlib.import_module(modules.DEFAULT_MODULE_NAME)
channel = test.Channel.create(name='training')
features = [1, 2, 3, 4]
labels = [0, 1, 0, 1]
np.savez(os.path.join(channel.path, 'training_data'),
features=features,
labels=labels)
module = test.UserModule(test.File(name='user_script.py',
data=user_script))
hyperparameters = dict(training_data_file='training_data.npz',
sagemaker_program='user_script.py',
epochs=10,
batch_size=64,
optimizer='SGD')
test.prepare(user_module=module,
hyperparameters=hyperparameters,
channels=[channel])
os.environ[
'SAGEMAKER_TRAINING_MODULE'] = 'test.functional.simple_framework:train'
assert execute_an_wrap_exit(trainer.train) == trainer.SUCCESS_CODE
model_path = os.path.join(env.model_dir, 'saved_model')
model = fake_ml_framework.Model.load(model_path)
assert model.epochs == 10
assert model.batch_size == 64
assert model.optimizer == 'SGD'
assert os.path.exists(os.path.join(env.output_dir, 'success'))
test_only = False
test_report = False
test_noclean = False
if "test" in sys.argv: test_only = True
if "report" in sys.argv: test_report = True
if "noclean" in sys.argv: test_noclean = True
if not test_only:
build_conf("Minimal config monochrome", defines.minimal_monochrome)
build_conf("Minimal config, 16 bit color depth", defines.minimal_16bit)
build_conf("Normal config, 16 bit color depth swapped",
defines.normal_16bit_swap)
build_conf("Full config, 32 bit color depth", defines.full_32bit)
files = test.prepare()
if test_noclean == False:
build.clean()
for f in files:
name = f[:-2] #test_foo.c -> test_foo
build.build_test(defines.test, name)
if test_report:
print("")
print("Generating report")
print("-----------------------", flush=True)
os.system("rm -r report")
os.system("mkdir report")
os.system(
"gcovr -r ../ --html-details -o report/index.html --exclude-directories '\.\./examples' --exclude-directories 'src/.*' --exclude-directories 'unity' --exclude 'lv_test_.*\.c'"
from __future__ import division
import caffe
from caffe import tools
import test
from util import Timer
import numpy as np
import os
save_format, snapshot_prefix = test.prepare()
weights = '/home/jonlong/x/caffe/models/VGG/VGG16_full_conv.caffemodel'
caffe.set_device(6)
solver = caffe.SGDSolver('solver.prototxt')
solver.set_snapshot_prefix(snapshot_prefix)
solver.net.set_phase_train()
solver.net.set_mode_gpu()
solver.net.copy_from(weights)
for _ in range(1000):
solver.step(200)
test.seg_tests(solver, save_format, 'pascal')
from __future__ import division
import caffe
from caffe import tools
import test
from util import Timer
import numpy as np
import os
save_format, snapshot_prefix = test.prepare()
weights = "/home/jonlong/x/caffe/models/VGG/VGG16_full_conv.caffemodel"
caffe.set_device(6)
solver = caffe.SGDSolver("solver.prototxt")
solver.set_snapshot_prefix(snapshot_prefix)
solver.net.set_phase_train()
solver.net.set_mode_gpu()
solver.net.copy_from(weights)
for _ in range(1000):
solver.step(200)
test.seg_tests(solver, save_format, "pascal")
