def test_chainer(strftime, time, sagemaker_session, chainer_version,
chainer_py_version):
chainer = Chainer(
entry_point=SCRIPT_PATH,
role=ROLE,
sagemaker_session=sagemaker_session,
instance_count=INSTANCE_COUNT,
instance_type=INSTANCE_TYPE,
framework_version=chainer_version,
py_version=chainer_py_version,
)
inputs = "s3://mybucket/train"
chainer.fit(inputs=inputs)
sagemaker_call_names = [c[0] for c in sagemaker_session.method_calls]
assert sagemaker_call_names == ["train", "logs_for_job"]
boto_call_names = [
c[0] for c in sagemaker_session.boto_session.method_calls
]
assert boto_call_names == ["resource"]
expected_train_args = _create_train_job(chainer_version,
chainer_py_version)
expected_train_args["input_config"][0]["DataSource"]["S3DataSource"][
"S3Uri"] = inputs
actual_train_args = sagemaker_session.method_calls[0][2]
assert actual_train_args == expected_train_args
model = chainer.create_model()
expected_image_base = "520713654638.dkr.ecr.us-west-2.amazonaws.com/sagemaker-chainer:{}-gpu-{}"
assert {
"Environment": {
"SAGEMAKER_SUBMIT_DIRECTORY":
"s3://mybucket/sagemaker-chainer-{}/source/sourcedir.tar.gz".
format(TIMESTAMP),
"SAGEMAKER_PROGRAM":
"dummy_script.py",
"SAGEMAKER_REGION":
"us-west-2",
"SAGEMAKER_CONTAINER_LOG_LEVEL":
"20",
},
"Image": expected_image_base.format(chainer_version,
chainer_py_version),
"ModelDataUrl": "s3://m/m.tar.gz",
} == model.prepare_container_def(GPU)
assert "cpu" in model.prepare_container_def(CPU)["Image"]
predictor = chainer.deploy(1, GPU)
assert isinstance(predictor, ChainerPredictor)
def test_chainer(strftime, sagemaker_session, chainer_version):
chainer = Chainer(entry_point=SCRIPT_PATH,
role=ROLE,
sagemaker_session=sagemaker_session,
train_instance_count=INSTANCE_COUNT,
train_instance_type=INSTANCE_TYPE,
py_version=PYTHON_VERSION,
framework_version=chainer_version)
inputs = 's3://mybucket/train'
chainer.fit(inputs=inputs)
sagemaker_call_names = [c[0] for c in sagemaker_session.method_calls]
assert sagemaker_call_names == ['train', 'logs_for_job']
boto_call_names = [
c[0] for c in sagemaker_session.boto_session.method_calls
]
assert boto_call_names == ['resource']
expected_train_args = _create_train_job(chainer_version)
expected_train_args['input_config'][0]['DataSource']['S3DataSource'][
'S3Uri'] = inputs
actual_train_args = sagemaker_session.method_calls[0][2]
assert actual_train_args == expected_train_args
model = chainer.create_model()
expected_image_base = '520713654638.dkr.ecr.us-west-2.amazonaws.com/sagemaker-chainer:{}-gpu-{}'
assert {
'Environment': {
'SAGEMAKER_SUBMIT_DIRECTORY':
's3://mybucket/sagemaker-chainer-{}/source/sourcedir.tar.gz'.
format(TIMESTAMP),
'SAGEMAKER_PROGRAM':
'dummy_script.py',
'SAGEMAKER_ENABLE_CLOUDWATCH_METRICS':
'false',
'SAGEMAKER_REGION':
'us-west-2',
'SAGEMAKER_CONTAINER_LOG_LEVEL':
'20'
},
'Image': expected_image_base.format(chainer_version, PYTHON_VERSION),
'ModelDataUrl': 's3://m/m.tar.gz'
} == model.prepare_container_def(GPU)
assert 'cpu' in model.prepare_container_def(CPU)['Image']
predictor = chainer.deploy(1, GPU)
assert isinstance(predictor, ChainerPredictor)
def _test_mnist(sagemaker_session, ecr_image, instance_type, instance_count,
script):
source_dir = 'test/resources/mnist'
with timeout(minutes=15):
data_path = 'test/resources/mnist/data'
chainer = Chainer(entry_point=script,
source_dir=source_dir,
role='SageMakerRole',
train_instance_count=instance_count,
train_instance_type=instance_type,
sagemaker_session=sagemaker_session,
image_name=ecr_image,
hyperparameters={
'batch-size': 10000,
'epochs': 1
})
prefix = 'chainer_mnist/{}'.format(sagemaker_timestamp())
train_data_path = os.path.join(data_path, 'train')
key_prefix = prefix + '/train'
train_input = sagemaker_session.upload_data(path=train_data_path,
key_prefix=key_prefix)
test_path = os.path.join(data_path, 'test')
test_input = sagemaker_session.upload_data(path=test_path,
key_prefix=prefix + '/test')
chainer.fit({'train': train_input, 'test': test_input})
with timeout_and_delete_endpoint(estimator=chainer, minutes=30):
predictor = chainer.deploy(initial_instance_count=1,
instance_type=instance_type)
batch_size = 100
data = np.zeros(shape=(batch_size, 1, 28, 28), dtype='float32')
output = predictor.predict(data)
assert len(output) == batch_size
def test_chainer(strftime, sagemaker_session, chainer_version):
chainer = Chainer(entry_point=SCRIPT_PATH, role=ROLE, sagemaker_session=sagemaker_session,
train_instance_count=INSTANCE_COUNT, train_instance_type=INSTANCE_TYPE, py_version=PYTHON_VERSION,
framework_version=chainer_version)
inputs = 's3://mybucket/train'
chainer.fit(inputs=inputs)
sagemaker_call_names = [c[0] for c in sagemaker_session.method_calls]
assert sagemaker_call_names == ['train', 'logs_for_job']
boto_call_names = [c[0] for c in sagemaker_session.boto_session.method_calls]
assert boto_call_names == ['resource']
expected_train_args = _create_train_job(chainer_version)
expected_train_args['input_config'][0]['DataSource']['S3DataSource']['S3Uri'] = inputs
actual_train_args = sagemaker_session.method_calls[0][2]
assert actual_train_args == expected_train_args
model = chainer.create_model()
expected_image_base = '520713654638.dkr.ecr.us-west-2.amazonaws.com/sagemaker-chainer:{}-gpu-{}'
assert {'Environment':
{'SAGEMAKER_SUBMIT_DIRECTORY':
's3://mybucket/sagemaker-chainer-{}/source/sourcedir.tar.gz'.format(TIMESTAMP),
'SAGEMAKER_PROGRAM': 'dummy_script.py',
'SAGEMAKER_ENABLE_CLOUDWATCH_METRICS': 'false',
'SAGEMAKER_REGION': 'us-west-2',
'SAGEMAKER_CONTAINER_LOG_LEVEL': '20'},
'Image': expected_image_base.format(chainer_version, PYTHON_VERSION),
'ModelDataUrl': 's3://m/m.tar.gz'} == model.prepare_container_def(GPU)
assert 'cpu' in model.prepare_container_def(CPU)['Image']
predictor = chainer.deploy(1, GPU)
assert isinstance(predictor, ChainerPredictor)
