def test_xgboost_boston_single_machine(docker_image, opt_ml):
customer_script = 'single_machine_customer_script.py'
hyperparameters = {
'objective': 'reg:linear',
'colsample-bytree': 0.3,
'learning-rate': 0.1,
'max-depth': 5,
'reg-alpha': 10,
'n-estimators': 10
}
local_mode.train(customer_script,
data_dir,
docker_image,
opt_ml,
hyperparameters=hyperparameters,
source_dir=boston_path)
files = [
'model/xgb-boston.model', 'output/data/cv_results.csv',
'output/data/feature-importance-plot.png'
]
assert not local_mode.file_exists(opt_ml,
'output/failure'), 'Failure happened'
test_utils.files_exist(opt_ml, files)
def test_chainer_mnist_distributed(docker_image, sagemaker_local_session,
instance_type, customer_script, tmpdir):
if instance_type == 'local_gpu':
pytest.skip('Local Mode does not support distributed GPU training.')
# pure_nccl communicator hangs when only one gpu is available.
cluster_size = 2
hyperparameters = {
'sagemaker_process_slots_per_host': 1,
'sagemaker_num_processes': cluster_size,
'batch-size': 10000,
'epochs': 1,
'communicator': 'hierarchical'
}
estimator = Chainer(entry_point=customer_script,
source_dir=mnist_path,
role=role,
image_name=docker_image,
train_instance_count=cluster_size,
train_instance_type=instance_type,
sagemaker_session=sagemaker_local_session,
hyperparameters=hyperparameters,
output_path='file://{}'.format(tmpdir))
estimator.fit({
'train': 'file://{}'.format(os.path.join(data_dir, 'train')),
'test': 'file://{}'.format(os.path.join(data_dir, 'test'))
})
success_files = {
'model': ['model.npz'],
'output': [
'success', 'data/accuracy.png', 'data/cg.dot', 'data/log',
'data/loss.png'
],
}
test_utils.files_exist(str(tmpdir), success_files)
request_data = np.zeros((100, 784), dtype='float32')
test_utils.predict_and_assert_response_length(estimator, request_data,
instance_type)
test_utils.predict_and_assert_response_length(estimator, request_data,
instance_type,
json_serializer,
json_deserializer,
'application/json')
test_utils.predict_and_assert_response_length(estimator, request_data,
instance_type,
csv_serializer,
csv_deserializer, 'text/csv')
def test_chainer_mnist_single_machine(docker_image, sagemaker_local_session,
instance_type, tmpdir):
customer_script = 'single_machine_customer_script.py'
hyperparameters = {'batch-size': 10000, 'epochs': 1}
estimator = Chainer(entry_point=customer_script,
source_dir=mnist_path,
role=role,
image_name=docker_image,
train_instance_count=1,
train_instance_type=instance_type,
sagemaker_session=sagemaker_local_session,
hyperparameters=hyperparameters,
output_path='file://{}'.format(tmpdir))
estimator.fit({
'train': 'file://{}'.format(os.path.join(data_dir, 'train')),
'test': 'file://{}'.format(os.path.join(data_dir, 'test'))
})
success_files = {
'model': ['model.npz'],
'output': [
'success', 'data/accuracy.png', 'data/cg.dot', 'data/log',
'data/loss.png'
],
}
test_utils.files_exist(str(tmpdir), success_files)
request_data = np.zeros((100, 784), dtype='float32')
test_utils.predict_and_assert_response_length(estimator, request_data,
instance_type)
test_utils.predict_and_assert_response_length(estimator, request_data,
instance_type,
csv_serializer,
csv_deserializer, 'text/csv')
test_arrays = [
np.zeros((100, 784), dtype='float32'),
np.zeros((100, 1, 28, 28), dtype='float32'),
np.zeros((100, 28, 28), dtype='float32')
]
with test_utils.local_mode_lock():
try:
predictor = _json_predictor(estimator, instance_type)
for array in test_arrays:
response = predictor.predict(array)
assert len(response) == len(array)
finally:
predictor.delete_endpoint()
def test_chainer_mnist_single_machine(docker_image, opt_ml, use_gpu):
customer_script = 'single_machine_customer_script.py'
hyperparameters = {'batch-size': 10000, 'epochs': 1}
local_mode.train(customer_script,
data_dir,
docker_image,
opt_ml,
hyperparameters=hyperparameters,
source_dir=mnist_path,
use_gpu=use_gpu)
files = [
'model/model.npz', 'output/success', 'output/data/algo-1/accuracy.png',
'output/data/algo-1/cg.dot', 'output/data/algo-1/log',
'output/data/algo-1/loss.png'
]
test_utils.files_exist(opt_ml, files)
assert not local_mode.file_exists(opt_ml,
'output/failure'), 'Failure happened'
script_path = os.path.join(mnist_path, customer_script)
with local_mode.serve(script_path,
model_dir=None,
image_name=docker_image,
opt_ml=opt_ml,
use_gpu=use_gpu,
source_dir=mnist_path):
test_arrays = [
np.zeros((100, 784), dtype='float32'),
np.zeros((100, 1, 28, 28), dtype='float32'),
np.zeros((100, 28, 28), dtype='float32')
]
request_data = np.zeros((100, 784), dtype='float32')
data_as_list = request_data.tolist()
test_utils.predict_and_assert_response_length(data_as_list, 'text/csv')
for array in test_arrays:
# JSON and NPY can take multidimensional (n > 2) arrays
data_as_list = array.tolist()
test_utils.predict_and_assert_response_length(
data_as_list, 'application/json')
test_utils.predict_and_assert_response_length(
request_data, 'application/x-npy')
def test_single_machine_failure(docker_image, instance_type, sagemaker_local_session, tmpdir):
customer_script = 'failure_script.py'
estimator = Chainer(entry_point=customer_script,
source_dir=resource_path,
role=role,
image_name=docker_image,
train_instance_count=1,
train_instance_type=instance_type,
sagemaker_session=sagemaker_local_session,
output_path='file://{}'.format(tmpdir))
with pytest.raises(RuntimeError):
estimator.fit()
failure_files = {'output': ['failure', os.path.join('data', 'this_file_is_expected')]}
test_utils.files_exist(str(tmpdir), failure_files)
def test_chainer_mnist_distributed(docker_image, opt_ml, use_gpu,
customer_script):
cluster_size = 2
# pure_nccl communicator hangs when only one gpu is available.
hyperparameters = {
'sagemaker_process_slots_per_host': 1,
'sagemaker_num_processes': cluster_size,
'batch-size': 10000,
'epochs': 1,
'communicator': 'hierarchical'
}
local_mode.train(customer_script,
data_dir,
docker_image,
opt_ml,
hyperparameters=hyperparameters,
cluster_size=cluster_size,
source_dir=mnist_path,
use_gpu=use_gpu)
files = [
'model/model.npz', 'output/success', 'output/data/algo-1/accuracy.png',
'output/data/algo-1/cg.dot', 'output/data/algo-1/log',
'output/data/algo-1/loss.png'
]
test_utils.files_exist(opt_ml, files)
assert not local_mode.file_exists(opt_ml,
'output/failure'), 'Failure happened'
with local_mode.serve(os.path.join(mnist_path, customer_script),
model_dir=None,
image_name=docker_image,
opt_ml=opt_ml):
request_data = np.zeros((100, 784), dtype='float32')
data_as_list = request_data.tolist()
test_utils.predict_and_assert_response_length(data_as_list,
'application/json')
test_utils.predict_and_assert_response_length(data_as_list, 'text/csv')
test_utils.predict_and_assert_response_length(request_data,
'application/x-npy')
def test_xgboost_abalone_training_single_machine(docker_image, opt_ml):
customer_script = "abalone_distributed.py"
hyperparameters = get_abalone_default_hyperparameters()
local_mode.train(
customer_script,
data_dir,
docker_image,
opt_ml,
hyperparameters=hyperparameters,
source_dir=abalone_path,
)
files = ["model/xgboost-model"]
assert not local_mode.file_exists(opt_ml,
"output/failure"), "Failure happened"
test_utils.files_exist(opt_ml, files)
def test_xgboost_abalone_kfold(dataset, extra_hps, model_file_count,
docker_image, opt_ml):
hyperparameters = get_abalone_default_hyperparameters()
data_path = os.path.join(data_root, dataset, "data")
local_mode.train(
False,
data_path,
docker_image,
opt_ml,
hyperparameters={
**hyperparameters,
**extra_hps
},
)
files = [f"model/xgboost-model-{i}" for i in range(model_file_count)]
assert not local_mode.file_exists(opt_ml,
"output/failure"), "Failure happened"
test_utils.files_exist(opt_ml, files)
local_mode.file_exists(opt_ml, "output/data/predictions.csv")
def test_chainer_mnist_custom_loop(docker_image, sagemaker_local_session,
instance_type, tmpdir):
customer_script = 'single_machine_custom_loop.py'
hyperparameters = {'batch-size': 10000, 'epochs': 1}
estimator = Chainer(entry_point=customer_script,
source_dir=mnist_path,
role=role,
image_name=docker_image,
train_instance_count=1,
train_instance_type=instance_type,
sagemaker_session=sagemaker_local_session,
hyperparameters=hyperparameters,
output_path='file://{}'.format(tmpdir))
estimator.fit({
'train': 'file://{}'.format(os.path.join(data_dir, 'train')),
'test': 'file://{}'.format(os.path.join(data_dir, 'test'))
})
success_files = {
'model': ['model.npz'],
'output': ['success'],
}
test_utils.files_exist(str(tmpdir), success_files)
request_data = np.zeros((100, 784), dtype='float32')
test_utils.predict_and_assert_response_length(estimator, request_data,
instance_type)
test_utils.predict_and_assert_response_length(estimator, request_data,
instance_type,
json_serializer,
json_deserializer,
'application/json')
test_utils.predict_and_assert_response_length(estimator, request_data,
instance_type,
csv_serializer,
csv_deserializer, 'text/csv')
def test_all_processes_finish_with_mpi(docker_image, sagemaker_local_session, tmpdir):
"""
This test validates that all training processes finish before containers are shut down.
"""
customer_script = 'all_processes_finish_customer_script.py'
hyperparameters = {'sagemaker_use_mpi': True, 'sagemaker_process_slots_per_host': 2,
'sagemaker_num_processes': 4}
estimator = Chainer(entry_point=customer_script,
source_dir=resource_path,
role=role,
image_name=docker_image,
train_instance_count=2,
train_instance_type='local',
sagemaker_session=sagemaker_local_session,
hyperparameters=hyperparameters,
output_path='file://{}'.format(tmpdir))
estimator.fit()
completion_file = {'output': [os.path.join('data', 'algo-2', 'process_could_complete')]}
test_utils.files_exist(str(tmpdir), completion_file)
def test_distributed_failure(docker_image, sagemaker_local_session, tmpdir):
customer_script = 'failure_script.py'
cluster_size = 2
failure_node = 1
hyperparameters = {'sagemaker_process_slots_per_host': 1,
'sagemaker_num_processes': cluster_size, 'node_to_fail': failure_node}
estimator = Chainer(entry_point=customer_script,
source_dir=resource_path,
role=role,
image_name=docker_image,
train_instance_count=cluster_size,
train_instance_type='local',
sagemaker_session=sagemaker_local_session,
hyperparameters=hyperparameters,
output_path='file://{}'.format(tmpdir))
with pytest.raises(RuntimeError):
estimator.fit()
node_failure_file = os.path.join('data', 'file_from_node_{}'.format(failure_node))
failure_files = {'output': ['failure', node_failure_file]}
test_utils.files_exist(str(tmpdir), failure_files)
def test_chainer_mnist_custom_loop(docker_image, opt_ml, use_gpu):
customer_script = 'single_machine_custom_loop.py'
hyperparameters = {'batch-size': 10000, 'epochs': 1}
local_mode.train(customer_script,
data_dir,
docker_image,
opt_ml,
hyperparameters=hyperparameters,
source_dir=mnist_path,
use_gpu=use_gpu)
files = ['model/model.npz', 'output/success']
test_utils.files_exist(opt_ml, files)
assert not local_mode.file_exists(opt_ml,
'output/failure'), 'Failure happened'
script_path = os.path.join(mnist_path, customer_script)
with local_mode.serve(script_path,
model_dir=None,
image_name=docker_image,
opt_ml=opt_ml):
request_data = np.zeros((100, 784), dtype='float32')
data_as_list = request_data.tolist()
test_utils.predict_and_assert_response_length(data_as_list,
'application/json')
test_utils.predict_and_assert_response_length(data_as_list, 'text/csv')
test_utils.predict_and_assert_response_length(request_data,
'application/x-npy')
def test_training_jobs_do_not_stall(docker_image, sagemaker_local_session, tmpdir):
"""
This test validates that training does not stall.
https://github.com/chainer/chainermn/issues/236
"""
customer_script = 'training_jobs_do_not_stall_customer_script.py'
hyperparameters = {'sagemaker_use_mpi': True, 'sagemaker_process_slots_per_host': 1,
'sagemaker_num_processes': 2}
estimator = Chainer(entry_point=customer_script,
source_dir=resource_path,
role=role,
image_name=docker_image,
train_instance_count=2,
train_instance_type='local',
sagemaker_session=sagemaker_local_session,
hyperparameters=hyperparameters,
output_path='file://{}'.format(tmpdir))
with pytest.raises(RuntimeError):
estimator.fit()
failure_files = {'output': ['failure', os.path.join('data', 'this_file_is_expected')]}
test_utils.files_exist(str(tmpdir), failure_files)
