def test_tar_and_upload_dir_s3(sagemaker_session):
bucket = 'mybucker'
s3_key_prefix = 'something/source'
script = 'mnist.py'
directory = 's3://m'
result = tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script, directory)
assert result == UploadedCode('s3://m', 'mnist.py')
def test_local_mode_serving_from_s3_model(sagemaker_local_session, mxnet_model):
local_mode_lock_fd = open(LOCK_PATH, 'w')
local_mode_lock = local_mode_lock_fd.fileno()
model_data = mxnet_model.model_data
boto_session = sagemaker_local_session.boto_session
default_bucket = sagemaker_local_session.default_bucket()
uploaded_data = tar_and_upload_dir(boto_session, default_bucket,
'test_mxnet_local_mode', '', model_data)
s3_model = MXNetModel(model_data=uploaded_data.s3_prefix, role='SageMakerRole',
entry_point=mxnet_model.entry_point, image=mxnet_model.image,
sagemaker_session=sagemaker_local_session)
predictor = None
try:
# Since Local Mode uses the same port for serving, we need a lock in order
# to allow concurrent test execution. The serving test is really fast so it still
# makes sense to allow this behavior.
fcntl.lockf(local_mode_lock, fcntl.LOCK_EX)
predictor = s3_model.deploy(initial_instance_count=1, instance_type='local')
data = numpy.zeros(shape=(1, 1, 28, 28))
predictor.predict(data)
finally:
if predictor:
predictor.delete_endpoint()
time.sleep(5)
fcntl.lockf(local_mode_lock, fcntl.LOCK_UN)
def test_input_fn_with_input_channels(docker_image, sagemaker_session, opt_ml, processor):
resource_path = os.path.join(SCRIPT_PATH, '../resources/iris')
copy_resource(resource_path, opt_ml, 'code')
copy_resource(resource_path, opt_ml, 'data', 'input/data')
s3_source_archive = fw_utils.tar_and_upload_dir(session=sagemaker_session.boto_session,
bucket=sagemaker_session.default_bucket(),
s3_key_prefix='test_job',
script='iris_train_input_fn_with_channels.py',
directory=os.path.join(resource_path, 'code'))
additional_hyperparameters = dict(training_steps=1, evaluation_steps=1)
create_config_files('iris_train_input_fn_with_channels.py', s3_source_archive.s3_prefix,
opt_ml, additional_hyperparameters)
os.makedirs(os.path.join(opt_ml, 'model'))
train(docker_image, opt_ml, processor)
assert file_exists(opt_ml, 'model/export/Servo'), 'model was not exported'
assert file_exists(opt_ml, 'model/checkpoint'), 'checkpoint was not created'
assert file_exists(opt_ml, 'output/success'), 'Success file was not created'
assert not file_exists(opt_ml, 'output/failure'), 'Failure happened'
with HostingContainer(image=docker_image, opt_ml=opt_ml,
script_name='iris_train_input_fn_with_channels.py', processor=processor) as c:
c.execute_pytest('test/integ/container_tests/estimator_classification.py')
def test_tar_and_upload_dir_not_s3(sagemaker_session):
bucket = 'mybucker'
s3_key_prefix = 'something/source'
script = os.path.basename(__file__)
directory = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
result = tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script, directory)
assert result == UploadedCode('s3://{}/{}/sourcedir.tar.gz'.format(bucket, s3_key_prefix), script)
def test_save_restore_assets(docker_image, sagemaker_session, opt_ml, processor):
resource_path = os.path.join(SCRIPT_PATH, '../resources/sentiment')
default_bucket = sagemaker_session.default_bucket()
copy_resource(resource_path, opt_ml, 'data', 'input/data')
s3_source_archive = fw_utils.tar_and_upload_dir(session=sagemaker_session.boto_session,
bucket=sagemaker_session.default_bucket(),
s3_key_prefix='test_job',
script='sentiment.py',
directory=os.path.join(resource_path, 'code'))
checkpoint_s3_path = 's3://{}/save_restore_assets/output-{}'.format(default_bucket, uuid.uuid4())
additional_hyperparameters = dict(
training_steps=1000,
evaluation_steps=100,
checkpoint_path=checkpoint_s3_path)
create_config_files('sentiment.py', s3_source_archive.s3_prefix, opt_ml, additional_hyperparameters)
os.makedirs(os.path.join(opt_ml, 'model'))
train(docker_image, opt_ml, processor)
with HostingContainer(opt_ml=opt_ml, image=docker_image, script_name='sentiment.py', processor=processor) as c:
c.execute_pytest('test/integ/container_tests/sentiment_classification.py')
def _upload_code(self, key_prefix):
local_code = get_config_value('local.local_code', self.sagemaker_session.config)
if self.sagemaker_session.local_mode and local_code:
self.uploaded_code = None
else:
self.uploaded_code = tar_and_upload_dir(session=self.sagemaker_session.boto_session,
bucket=self.bucket or self.sagemaker_session.default_bucket(),
s3_key_prefix=key_prefix,
script=self.entry_point,
directory=self.source_dir)
def test_tar_and_upload_dir_not_s3(sagemaker_session):
bucket = "mybucket"
s3_key_prefix = "something/source"
script = os.path.basename(__file__)
directory = os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe())))
result = fw_utils.tar_and_upload_dir(sagemaker_session, bucket,
s3_key_prefix, script, directory)
assert result == fw_utils.UploadedCode(
"s3://{}/{}/sourcedir.tar.gz".format(bucket, s3_key_prefix), script)
def fit(self, inputs, wait=True, logs=True, job_name=None):
"""Train a model using the input training dataset.
The API calls the Amazon SageMaker CreateTrainingJob API to start model training.
The API uses configuration you provided to create the estimator and the
specified input training data to send the CreatingTrainingJob request to Amazon SageMaker.
This is a synchronous operation. After the model training successfully completes,
you can call the ``deploy()`` method to host the model using the Amazon SageMaker hosting services.
Args:
inputs (str or dict or sagemaker.session.s3_input): Information about the training data.
This can be one of three types:
(str) - the S3 location where training data is saved.
(dict[str, str] or dict[str, sagemaker.session.s3_input]) - If using multiple channels for
training data, you can specify a dict mapping channel names
to strings or :func:`~sagemaker.session.s3_input` objects.
(sagemaker.session.s3_input) - channel configuration for S3 data sources that can provide
additional information about the training dataset. See :func:`sagemaker.session.s3_input`
for full details.
wait (bool): Whether the call shouldl wait until the job completes (default: True).
logs (bool): Whether to show the logs produced by the job.
Only meaningful when wait is True (default: True).
job_name (str): Training job name. If not specified, the estimator generates a default job name,
based on the training image name and current timestamp.
"""
# always determine new job name _here_ because it is used before base is called
if job_name is not None:
self._current_job_name = job_name
else:
# honor supplied base_job_name or generate it
base_name = self.base_job_name or base_name_from_image(self.train_image())
self._current_job_name = name_from_base(base_name)
if self.code_location is None:
code_bucket = self.sagemaker_session.default_bucket()
code_s3_prefix = '{}/source'.format(self._current_job_name)
else:
code_bucket, key_prefix = parse_s3_url(self.code_location)
code_s3_prefix = '{}/{}/source'.format(key_prefix, self._current_job_name)
self.uploaded_code = tar_and_upload_dir(session=self.sagemaker_session.boto_session,
bucket=code_bucket,
s3_key_prefix=code_s3_prefix,
script=self.entry_point,
directory=self.source_dir)
# Modify hyperparameters in-place to add the URLs to the uploaded code.
self._hyperparameters[DIR_PARAM_NAME] = self.uploaded_code.s3_prefix
self._hyperparameters[SCRIPT_PARAM_NAME] = self.uploaded_code.script_name
self._hyperparameters[CLOUDWATCH_METRICS_PARAM_NAME] = self.enable_cloudwatch_metrics
self._hyperparameters[CONTAINER_LOG_LEVEL_PARAM_NAME] = self.container_log_level
self._hyperparameters[JOB_NAME_PARAM_NAME] = self._current_job_name
self._hyperparameters[SAGEMAKER_REGION_PARAM_NAME] = self.sagemaker_session.boto_session.region_name
super(Framework, self).fit(inputs, wait, logs, self._current_job_name)
def test_tar_and_upload_dir_with_directory_and_files(sagemaker_session, tmpdir):
file_tree(tmpdir, ['src-dir/train.py', 'src-dir/laucher', 'src-dir/module/__init__.py'])
source_dir = os.path.join(str(tmpdir), 'src-dir')
with patch('shutil.rmtree'):
result = fw_utils.tar_and_upload_dir(sagemaker_session, 'bucket', 'prefix', 'train.py', source_dir)
assert result == fw_utils.UploadedCode(s3_prefix='s3://bucket/prefix/sourcedir.tar.gz',
script_name='train.py')
assert {'/laucher', '/module/__init__.py', '/train.py'} == list_source_dir_files(sagemaker_session, tmpdir)
def create_training(data_dir,
customer_script,
optml,
image,
additional_volumes,
additional_env_vars,
additional_hps=None,
cluster_size=1,
source_dir=None,
entrypoint=None):
additional_hps = additional_hps or None
session = boto3.Session()
tmpdir = os.path.abspath(optml)
hosts = create_host_names(cluster_size)
print('creating hosts: {}'.format(hosts))
config = create_input_data_config(data_dir)
hyperparameters = read_hyperparameters(additional_hps)
if customer_script:
s3_script_path = fw_utils.tar_and_upload_dir(
session=session,
bucket=default_bucket(session),
s3_key_prefix='test',
script=customer_script,
directory=source_dir)[0]
hyperparameters.update({
'sagemaker_submit_directory':
s3_script_path,
'sagemaker_program':
os.path.basename(customer_script)
})
for host in hosts:
for d in ['input', 'input/config', 'output', 'model']:
os.makedirs(os.path.join(tmpdir, host, d))
write_hyperparameters(tmpdir, host, hyperparameters)
write_resource_config(tmpdir, hosts, host)
write_inputdataconfig(tmpdir, host, config)
shutil.copytree(data_dir, os.path.join(tmpdir, host, 'input', 'data'))
write_docker_file('train', tmpdir, hosts, image, additional_volumes,
additional_env_vars, customer_script, source_dir,
entrypoint)
print("training dir: \n{}".format(
str(subprocess.check_output(['ls', '-l', tmpdir]).decode('utf-8'))))
return tmpdir
def test_tar_and_upload_dir_no_directory_only_entrypoint(sagemaker_session, tmpdir):
source_dir = file_tree(tmpdir, ['train.py', 'not_me.py'])
entrypoint = os.path.join(source_dir, 'train.py')
with patch('shutil.rmtree'):
result = fw_utils.tar_and_upload_dir(sagemaker_session, 'bucket', 'prefix', entrypoint, None)
assert result == fw_utils.UploadedCode(s3_prefix='s3://bucket/prefix/sourcedir.tar.gz',
script_name='train.py')
assert {'/train.py'} == list_source_dir_files(sagemaker_session, tmpdir)
def test_tar_and_upload_dir_with_directories_and_files(sagemaker_session, tmpdir):
file_tree(tmpdir, ['src-dir/a/b', 'src-dir/a/b2', 'src-dir/x/y', 'src-dir/x/y2', 'src-dir/z'])
source_dir = os.path.join(str(tmpdir), 'src-dir')
with patch('shutil.rmtree'):
result = fw_utils.tar_and_upload_dir(sagemaker_session, 'bucket', 'prefix', 'a/b', source_dir)
assert result == fw_utils.UploadedCode(s3_prefix='s3://bucket/prefix/sourcedir.tar.gz',
script_name='a/b')
assert {'/a/b', '/a/b2', '/x/y', '/x/y2', '/z'} == list_source_dir_files(sagemaker_session, tmpdir)
def test_test_tar_and_upload_dir_with_subfolders(sagemaker_session, tmpdir):
file_tree(tmpdir, ['a/b/c', 'a/b/c2'])
root = file_tree(tmpdir, ['x/y/z', 'x/y/z2'])
with patch('shutil.rmtree'):
result = fw_utils.tar_and_upload_dir(sagemaker_session, 'bucket', 'prefix', 'b/c',
os.path.join(root, 'a'), [os.path.join(root, 'x')])
assert result == fw_utils.UploadedCode(s3_prefix='s3://bucket/prefix/sourcedir.tar.gz',
script_name='b/c')
assert {'/b/c', '/b/c2', '/x/y/z', '/x/y/z2'} == list_source_dir_files(sagemaker_session, tmpdir)
def _upload_code(self, key_prefix):
local_code = get_config_value('local.local_code',
self.sagemaker_session.config)
if self.sagemaker_session.local_mode and local_code:
self.uploaded_code = None
else:
self.uploaded_code = tar_and_upload_dir(
session=self.sagemaker_session.boto_session,
bucket=self.bucket or self.sagemaker_session.default_bucket(),
s3_key_prefix=key_prefix,
script=self.entry_point,
directory=self.source_dir)
def test_tar_and_upload_dir_with_directory(sagemaker_session, tmpdir):
file_tree(tmpdir, ["src-dir/train.py"])
source_dir = os.path.join(str(tmpdir), "src-dir")
with patch("shutil.rmtree"):
result = fw_utils.tar_and_upload_dir(sagemaker_session, "bucket",
"prefix", "train.py", source_dir)
assert result == fw_utils.UploadedCode(
s3_prefix="s3://bucket/prefix/sourcedir.tar.gz",
script_name="train.py")
assert {"/train.py"} == list_source_dir_files(sagemaker_session, tmpdir)
def test_tar_and_upload_dir_s3_kms_enabled_by_default(utils, sagemaker_session):
bucket = "mybucket"
s3_key_prefix = "something/source"
script = "inference.py"
result = fw_utils.tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script)
assert result == fw_utils.UploadedCode(
"s3://{}/{}/sourcedir.tar.gz".format(bucket, s3_key_prefix), script
)
extra_args = {"ServerSideEncryption": "aws:kms"}
obj = sagemaker_session.resource("s3").Object("", "")
obj.upload_file.assert_called_with(utils.create_tar_file(), ExtraArgs=extra_args)
def test_tar_and_upload_dir_with_many_folders(sagemaker_session, tmpdir):
file_tree(tmpdir, ['src-dir/a/b', 'src-dir/a/b2', 'common/x/y', 'common/x/y2', 't/y/z'])
source_dir = os.path.join(str(tmpdir), 'src-dir')
dependencies = [os.path.join(str(tmpdir), 'common'), os.path.join(str(tmpdir), 't', 'y', 'z')]
with patch('shutil.rmtree'):
result = fw_utils.tar_and_upload_dir(sagemaker_session, 'bucket', 'prefix',
'pipeline.py', source_dir, dependencies)
assert result == fw_utils.UploadedCode(s3_prefix='s3://bucket/prefix/sourcedir.tar.gz',
script_name='pipeline.py')
assert {'/a/b', '/a/b2', '/common/x/y', '/common/x/y2', '/z'} == list_source_dir_files(sagemaker_session, tmpdir)
def test_tar_and_upload_dir_s3_with_kms(utils, sagemaker_session):
result = fw_utils.tar_and_upload_dir(sagemaker_session,
'mybucker',
'something/source',
'mnist.py',
kms_key='kms-key')
assert result == fw_utils.UploadedCode('s3://mybucker/something/source/sourcedir.tar.gz',
'mnist.py')
extra_args = {'ServerSideEncryption': 'aws:kms', 'SSEKMSKeyId': 'kms-key'}
obj = sagemaker_session.resource('s3').Object('', '')
obj.upload_file.assert_called_with(utils.create_tar_file(), ExtraArgs=extra_args)
def test_tar_and_upload_dir_no_directory_only_entrypoint(
sagemaker_session, tmpdir):
source_dir = file_tree(tmpdir, ["train.py", "not_me.py"])
entrypoint = os.path.join(source_dir, "train.py")
with patch("shutil.rmtree"):
result = fw_utils.tar_and_upload_dir(sagemaker_session, "bucket",
"prefix", entrypoint, None)
assert result == fw_utils.UploadedCode(
s3_prefix="s3://bucket/prefix/sourcedir.tar.gz",
script_name="train.py")
assert {"/train.py"} == list_source_dir_files(sagemaker_session, tmpdir)
def test_tar_and_upload_dir_s3_without_kms_with_overridden_settings(utils, sagemaker_session):
bucket = "mybucket"
s3_key_prefix = "something/source"
script = "inference.py"
settings = SessionSettings(encrypt_repacked_artifacts=False)
result = fw_utils.tar_and_upload_dir(
sagemaker_session, bucket, s3_key_prefix, script, settings=settings
)
assert result == fw_utils.UploadedCode(
"s3://{}/{}/sourcedir.tar.gz".format(bucket, s3_key_prefix), script
)
obj = sagemaker_session.resource("s3").Object("", "")
obj.upload_file.assert_called_with(utils.create_tar_file(), ExtraArgs=None)
def create_docker_services(command,
tmpdir,
hosts,
image,
additional_volumes,
additional_env_vars,
customer_script,
source_dir,
entrypoint,
use_gpu=False):
environment = []
session = boto3.Session()
optml_dirs = set()
if command == 'train':
optml_dirs = {'output', 'input'}
elif command == 'serve':
environment.extend(DEFAULT_HOSTING_ENV)
if customer_script:
timestamp = utils.sagemaker_timestamp()
s3_script_path = fw_utils.tar_and_upload_dir(
session=session,
bucket=default_bucket(session),
s3_key_prefix='test-{}'.format(timestamp),
script=customer_script,
directory=source_dir)[0]
environment.extend([
'SAGEMAKER_PROGRAM={}'.format(
os.path.basename(customer_script)),
'SAGEMAKER_SUBMIT_DIRECTORY={}'.format(s3_script_path)
])
else:
raise ValueError('Unexpected command: {}'.format(command))
environment.extend(credentials_to_env(session))
environment.extend(additional_env_vars)
return {
h: create_docker_host(tmpdir, h, image, environment, optml_dirs,
command, additional_volumes, entrypoint, use_gpu)
for h in hosts
}
def test_tar_and_upload_dir_with_many_folders(sagemaker_session, tmpdir):
file_tree(tmpdir, ["src-dir/a/b", "src-dir/a/b2", "common/x/y", "common/x/y2", "t/y/z"])
source_dir = os.path.join(str(tmpdir), "src-dir")
dependencies = [os.path.join(str(tmpdir), "common"), os.path.join(str(tmpdir), "t", "y", "z")]
with patch("shutil.rmtree"):
result = fw_utils.tar_and_upload_dir(
sagemaker_session, "bucket", "prefix", "pipeline.py", source_dir, dependencies
)
assert result == fw_utils.UploadedCode(
s3_prefix="s3://bucket/prefix/sourcedir.tar.gz", script_name="pipeline.py"
)
assert {"/a/b", "/a/b2", "/common/x/y", "/common/x/y2", "/z"} == list_source_dir_files(
sagemaker_session, tmpdir
)
def _stage_user_code_in_s3(self):
"""Upload the user training script to s3 and return the location.
Returns: s3 uri
"""
if self.code_location is None:
code_bucket = self.sagemaker_session.default_bucket()
code_s3_prefix = '{}/source'.format(self._current_job_name)
else:
code_bucket, key_prefix = parse_s3_url(self.code_location)
code_s3_prefix = '/'.join(filter(None, [key_prefix, self._current_job_name, 'source']))
return tar_and_upload_dir(session=self.sagemaker_session.boto_session,
bucket=code_bucket,
s3_key_prefix=code_s3_prefix,
script=self.entry_point,
directory=self.source_dir)
def _stage_user_code_in_s3(self):
""" Upload the user training script to s3 and return the location.
Returns: s3 uri
"""
if self.code_location is None:
code_bucket = self.sagemaker_session.default_bucket()
code_s3_prefix = '{}/source'.format(self._current_job_name)
else:
code_bucket, key_prefix = parse_s3_url(self.code_location)
code_s3_prefix = '/'.join(filter(None, [key_prefix, self._current_job_name, 'source']))
return tar_and_upload_dir(session=self.sagemaker_session.boto_session,
bucket=code_bucket,
s3_key_prefix=code_s3_prefix,
script=self.entry_point,
directory=self.source_dir)
def test_test_tar_and_upload_dir_with_subfolders(sagemaker_session, tmpdir):
file_tree(tmpdir, ["a/b/c", "a/b/c2"])
root = file_tree(tmpdir, ["x/y/z", "x/y/z2"])
with patch("shutil.rmtree"):
result = fw_utils.tar_and_upload_dir(
sagemaker_session,
"bucket",
"prefix",
"b/c",
os.path.join(root, "a"),
[os.path.join(root, "x")],
)
assert result == fw_utils.UploadedCode(
s3_prefix="s3://bucket/prefix/sourcedir.tar.gz", script_name="b/c")
assert {"/b/c", "/b/c2", "/x/y/z",
"/x/y/z2"} == list_source_dir_files(sagemaker_session, tmpdir)
def test_linear_regression(docker_image, sagemaker_session, opt_ml, processor):
resource_path = 'test/resources/linear_regression'
# create training data
train_data = np.random.uniform(0, 1, [1000, 2])
train_label = np.array(
[train_data[i][0] + 2 * train_data[i][1] for i in range(1000)])
# eval data... repeat so there's enough to cover multicpu/gpu contexts
eval_data = np.array([[7, 2], [6, 10], [12, 2]]).repeat(32, 0)
eval_label = np.array([11, 26, 16]).repeat(32, 0)
# save training data
for path in ['training', 'evaluation']:
os.makedirs(os.path.join(opt_ml, 'input', 'data', path))
np.savetxt(os.path.join(opt_ml, 'input/data/training/train_data.txt'),
train_data)
np.savetxt(os.path.join(opt_ml, 'input/data/training/train_label.txt'),
train_label)
np.savetxt(os.path.join(opt_ml, 'input/data/evaluation/eval_data.txt'),
eval_data)
np.savetxt(os.path.join(opt_ml, 'input/data/evaluation/eval_label.txt'),
eval_label)
s3_source_archive = fw_utils.tar_and_upload_dir(
session=sagemaker_session.boto_session,
bucket=sagemaker_session.default_bucket(),
s3_key_prefix=sagemaker_timestamp(),
script='linear_regression.py',
directory=resource_path)
utils.create_config_files('linear_regression.py',
s3_source_archive.s3_prefix, opt_ml)
os.makedirs(os.path.join(opt_ml, 'model'))
docker_utils.train(docker_image, opt_ml, processor)
for f in [
'output/success', 'model/model-symbol.json',
'model/model-0000.params', 'model/model-shapes.json'
]:
assert os.path.exists(os.path.join(
opt_ml, f)), 'expected file not found: {}'.format(f)
def test_estimator_classification(docker_image, sagemaker_session, opt_ml,
processor):
resource_path = os.path.join(SCRIPT_PATH, '../resources/iris')
copy_resource(resource_path, opt_ml, 'code')
copy_resource(resource_path, opt_ml, 'data', 'input/data')
s3_source_archive = fw_utils.tar_and_upload_dir(
session=sagemaker_session.boto_session,
bucket=sagemaker_session.default_bucket(),
s3_key_prefix='test_job',
script='iris.py',
directory=os.path.join(resource_path, 'code'))
additional_hyperparameters = dict(
training_steps=1,
evaluation_steps=1,
sagemaker_requirements='requirements.txt')
create_config_files('iris.py', s3_source_archive.s3_prefix, opt_ml,
additional_hyperparameters)
os.makedirs(os.path.join(opt_ml, 'model'))
train(docker_image, opt_ml, processor)
assert file_exists(opt_ml, 'model/export/Servo'), 'model was not exported'
assert file_exists(opt_ml,
'model/checkpoint'), 'checkpoint was not created'
assert file_exists(opt_ml,
'output/success'), 'Success file was not created'
assert not file_exists(opt_ml, 'output/failure'), 'Failure happened'
with HostingContainer(opt_ml=opt_ml,
image=docker_image,
script_name='iris.py',
processor=processor,
requirements_file='requirements.txt') as c:
c.execute_pytest(
'test/integ/container_tests/estimator_classification.py')
modules = c.execute_command(['pip', 'freeze'])
assert 'beautifulsoup4==4.6.0' in modules
def _upload_code(self, key_prefix, repack=False):
"""
Args:
key_prefix:
repack:
"""
local_code = utils.get_config_value("local.local_code",
self.sagemaker_session.config)
if self.sagemaker_session.local_mode and local_code:
self.uploaded_code = None
elif not repack:
bucket = self.bucket or self.sagemaker_session.default_bucket()
self.uploaded_code = fw_utils.tar_and_upload_dir(
session=self.sagemaker_session.boto_session,
bucket=bucket,
s3_key_prefix=key_prefix,
script=self.entry_point,
directory=self.source_dir,
dependencies=self.dependencies,
)
if repack:
bucket = self.bucket or self.sagemaker_session.default_bucket()
repacked_model_data = "s3://" + os.path.join(
bucket, key_prefix, "model.tar.gz")
utils.repack_model(
inference_script=self.entry_point,
source_directory=self.source_dir,
dependencies=self.dependencies,
model_uri=self.model_data,
repacked_model_uri=repacked_model_data,
sagemaker_session=self.sagemaker_session,
kms_key=self.model_kms_key,
)
self.repacked_model_data = repacked_model_data
self.uploaded_code = UploadedCode(
s3_prefix=self.repacked_model_data,
script_name=os.path.basename(self.entry_point))
def test_no_serving_input_fn(docker_image, sagemaker_session, opt_ml, processor):
resource_path = os.path.join(SCRIPT_PATH, '../resources/iris')
copy_resource(resource_path, opt_ml, 'code')
copy_resource(resource_path, opt_ml, 'data', 'input/data')
s3_source_archive = fw_utils.tar_and_upload_dir(session=sagemaker_session.boto_session,
bucket=sagemaker_session.default_bucket(),
s3_key_prefix='test_job',
script='iris_no_serving_input.py',
directory=os.path.join(resource_path, 'code'))
additional_hyperparameters = dict(training_steps=1, evaluation_steps=1)
create_config_files('iris_no_serving_input.py', s3_source_archive.s3_prefix, opt_ml, additional_hyperparameters)
os.makedirs(os.path.join(opt_ml, 'model'))
train(docker_image, opt_ml, processor)
# model is not saved so do not expect it there
assert not file_exists(opt_ml, 'model/export/Servo'), 'model was not exported'
assert file_exists(opt_ml, 'model/checkpoint'), 'checkpoint was not created'
assert file_exists(opt_ml, 'output/success'), 'Success file was not created'
assert not file_exists(opt_ml, 'output/failure'), 'Failure happened'
def test_s3_checkpoint_save_timeout(docker_image, opt_ml, sagemaker_session, processor):
resource_path = os.path.join(SCRIPT_PATH, '../resources/python_sdk')
default_bucket = sagemaker_session.default_bucket()
s3_source_archive = fw_utils.tar_and_upload_dir(session=sagemaker_session.boto_session,
bucket=default_bucket,
s3_key_prefix='test_job',
script='rand_model_emb.py',
directory=resource_path)
checkpoint_s3_path = 's3://{}/integ-s3-timeout/checkpoints-{}'.format(default_bucket,
uuid.uuid4())
hyperparameters = dict(
training_steps=3,
evaluation_steps=3,
checkpoint_path=checkpoint_s3_path
)
create_config_files('rand_model_emb.py', s3_source_archive.s3_prefix, opt_ml, hyperparameters)
train(docker_image, opt_ml, processor)
assert file_exists(opt_ml, 'output/success'), 'Success file was not created'
assert not file_exists(opt_ml, 'output/failure'), 'Failure happened'
