def run_benchmark(instance_count,
subnet,
security_group,
aws_account,
base_image,
region='us-west-2',
role="SageMakerRole",
tag='tensorflow-hvd:latest',
build_image=False,
wait=True):
if build_image:
build(base_image=base_image,
entrypoint='launcher.sh',
source_dir='benchmarks',
tag=tag)
ecr_image_name = push(tag)
output_path = 's3://sagemaker-{}-{}/hvd-1-single/{}node-{}'.format(
region, aws_account, instance_count, time.time_ns())
estimator = Estimator(ecr_image_name,
role=role,
base_job_name='hvd-bench',
hyperparameters={},
train_instance_count=instance_count,
train_instance_type='ml.p3.16xlarge',
output_path=output_path,
subnets=[subnet],
security_group_ids=[security_group])
estimator.fit('s3://sagemaker-sample-data-%s/spark/mnist/train/' % region,
wait=wait)
def test_estimator_transformer_creation_with_optional_params(sagemaker_session):
base_name = 'foo'
estimator = Estimator(image_name=IMAGE_NAME, role=ROLE, train_instance_count=INSTANCE_COUNT,
train_instance_type=INSTANCE_TYPE, sagemaker_session=sagemaker_session,
base_job_name=base_name)
estimator.latest_training_job = _TrainingJob(sagemaker_session, JOB_NAME)
sagemaker_session.create_model_from_job.return_value = JOB_NAME
strategy = 'MultiRecord'
assemble_with = 'Line'
kms_key = 'key'
accept = 'text/csv'
max_concurrent_transforms = 1
max_payload = 6
env = {'FOO': 'BAR'}
transformer = estimator.transformer(INSTANCE_COUNT, INSTANCE_TYPE, strategy=strategy, assemble_with=assemble_with,
output_path=OUTPUT_PATH, output_kms_key=kms_key, accept=accept, tags=TAGS,
max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload,
env=env, role=ROLE)
sagemaker_session.create_model_from_job.assert_called_with(JOB_NAME, role=ROLE)
assert transformer.strategy == strategy
assert transformer.assemble_with == assemble_with
assert transformer.output_path == OUTPUT_PATH
assert transformer.output_kms_key == kms_key
assert transformer.accept == accept
assert transformer.max_concurrent_transforms == max_concurrent_transforms
assert transformer.max_payload == max_payload
assert transformer.env == env
assert transformer.base_transform_job_name == base_name
assert transformer.tags == TAGS
def main():
download_training_and_eval_data()
print('Starting model training.')
print(
'Note: if launching for the first time in local mode, container image download might take a few minutes to complete.'
)
image = 'sagemaker-hdbscan-local'
local_estimator = Estimator(image,
DUMMY_IAM_ROLE,
instance_count=1,
instance_type="local",
hyperparameters={
"min_cluster_size": 50,
})
train_location = 'file://' + local_train
local_estimator.fit({'train': train_location})
print('Completed model training')
model_data = local_estimator.model_data
print(model_data)
def test_xgb_train_container_cpu(sagemaker_session, instance_type):
training_data_path = os.path.join(test_dir, 'resources/data/')
estimator = Estimator(role=ROLE,
sagemaker_session=sagemaker_session,
train_instance_count=1,
train_instance_type=instance_type,
image_name=XGB_IMAGE_NAME,
output_path=MODEL_SAVE_PATH,
hyperparameters={
"train-file": "penguins.csv",
"max-depth": 3,
"categorical-columns": 'island,sex'
})
inputs = estimator.sagemaker_session.upload_data(path=os.path.join(
training_data_path, 'penguins.csv'),
bucket=BUCKET_NAME,
key_prefix='penguins/tmp')
estimator.fit(
inputs, job_name=unique_name_from_base('test-sagemaker-xgb-training'))
# Clean up the models folder and re-create it
if os.path.exists(os.path.join(test_dir, 'resources/models_tar')):
shutil.rmtree(os.path.join(test_dir, 'resources/models_tar'))
os.mkdir(os.path.join(test_dir, 'resources/models_tar'))
# Download the model files
obj_name = os.path.relpath(estimator.model_data, 's3://' + BUCKET_NAME)
s3.Bucket(BUCKET_NAME).download_file(
obj_name, os.path.join(test_dir, 'resources/models_tar/model.tar.gz'))
_assert_s3_file_exists(sagemaker_session.boto_region_name,
estimator.model_data)
def main():
download_training_and_eval_data()
image = 'sagemaker-tensorflow2-local'
print('Starting model training.')
california_housing_estimator = Estimator(
image,
DUMMY_IAM_ROLE,
hyperparameters={'epochs': 10,
'batch_size': 64,
'learning_rate': 0.1},
instance_count=1,
instance_type="local")
inputs = {'train': 'file://./data/train', 'test': 'file://./data/test'}
california_housing_estimator.fit(inputs, logs=True)
print('Completed model training')
print('Deploying endpoint in local mode')
predictor = california_housing_estimator.deploy(initial_instance_count=1, instance_type='local')
do_inference_on_local_endpoint(predictor)
print('About to delete the endpoint to stop paying (if in cloud mode).')
predictor.delete_endpoint(predictor.endpoint_name)
def main():
args = get_args()
sess = sagemaker.Session()
role = get_execution_role()
client = boto3.client('sts')
account = client.get_caller_identity()['Account']
my_session = boto3.session.Session()
region = my_session.region_name
container_name = args.container_name
ecr_image = '{}.dkr.ecr.{}.amazonaws.com/{}:latest'.format(
account, region, container_name)
inputs = sess.upload_data(path=args.data, key_prefix=DATASET_PREFIX)
hyperparameters = {'train-steps': 1000}
instance_type = 'ml.m4.xlarge'
estimator = Estimator(role=role,
hyperparameters=hyperparameters,
instance_count=1,
instance_type=instance_type,
image_uri=ecr_image)
estimator.fit(inputs)
def test_prepare_for_training_with_name_based_on_image(sagemaker_session):
estimator = Estimator(image_name='some-image',
role='some_image',
train_instance_count=1,
train_instance_type='ml.m4.xlarge',
sagemaker_session=sagemaker_session)
estimator._prepare_for_training()
assert 'some-image' in estimator._current_job_name
class SagemakerTFEstimator(object):
def __init__(self, container_image_uri: str, sm_session: sm.Session,
sm_role: str, project_tag: List[Dict[str, str]],
tn_instance_type: str, tn_instance_count: int,
tn_volumesize: int, tn_job_name: str, max_run: int,
shared_hyperparameters: Dict[str, str], **kwargs) -> None:
self.estimator = Estimator(
image_uri=container_image_uri,
instance_type=tn_instance_type,
instance_count=tn_instance_count,
volume_size=tn_volumesize,
role=sm_role,
sagemaker_session=sm_session,
tags=project_tag,
max_run=max_run,
hyperparameters=shared_hyperparameters,
**kwargs,
)
self._training_job_name = tn_job_name
self._project_tag = project_tag
def model_fit(
self,
inputs: Dict[str, str],
hparam: Dict[str, Any] = None,
) -> None:
if hparam is not None:
tuner = HyperparameterTuner(
estimator=self.estimator,
objective_metric_name=hparam.get('objective_metric_name'),
metric_definitions=hparam.get('metric_definitions'),
hyperparameter_ranges=hparam.get('hyperparameter_ranges'),
objective_type=hparam.get('objective_type'),
max_jobs=hparam.get('max_jobs'),
max_parallel_jobs=hparam.get('max_parallel_jobs'),
tags=self._project_tag,
base_tuning_job_name=self._training_job_name,
)
tuner.fit(
inputs=inputs,
job_name=self._training_job_name,
wait=False,
logs='All',
)
else:
self.estimator.fit(
inputs=inputs,
job_name=self._training_job_name,
wait=False,
logs='All',
)
def estimator(self, batch_n):
ll_estimator = Estimator(self.container,
role=self.role,
instance_count=1,
instance_type='ml.m5.large',
output_path='s3://{}/{}/output'.format(
self.bucket, self.prefix))
ll_estimator.set_hyperparameters(predictor_type='regressor',
mini_batch_size=batch_n)
return ll_estimator
def main(
gpu: bool = typer.Option(
False,
"--gpu",
help=
"Should a GPU based docker image be used? If this flag is set, and you are running a SageMaker job, you must specify an instance with a GPU (e.g. ml.p2/3...).",
),
instance_type: str = typer.Option(
"local",
help=
"SageMaker instance used to run the model, e.g. ml.p2.xlarge or ml.c5.xlarge. Setting this to local will run the container locally.",
),
):
image_name = f"{REPO_URL}:{VERSION}"
if gpu:
image_name = image_name + "-gpu"
input_channels = {
"train": train,
"test": test,
"word_embedding": word_embedding,
"indices": indices,
# Setting these to file:// will upload the data from the local drive
# "train": "file://data/processed/train.jsonl",
# "test": "file://data/processed/test.jsonl",
# "word_embedding": "file://data/raw/glove.6B.50d.txt",
}
estimator = Estimator(
image_name=image_name,
role=ROLE_ARN,
train_instance_count=1,
train_instance_type=instance_type,
hyperparameters={
"test-path": "/opt/ml/input/data/test/" + test_file,
"train-path": "/opt/ml/input/data/train/" + train_file,
"indices-path": "/opt/ml/input/data/indices/" + indices_file,
"output-path": "/opt/ml/model/",
"model-output-path": "/opt/ml/model/",
"embedding-path":
"/opt/ml/input/data/word_embedding/" + word_embedding_file,
"embedding-dim": 50,
"batch-size": 1024,
"epochs": 2,
"learning-rate": 0.01,
"seq-length": 1000,
"checkpoint": True,
"checkpoint-path": "/opt/ml/model/",
},
)
estimator.fit(inputs=input_channels)
def test_async_byo_estimator(sagemaker_session, region):
image_name = registry(region) + "/factorization-machines:1"
endpoint_name = unique_name_from_base('byo')
training_data_path = os.path.join(DATA_DIR, 'dummy_tensor')
training_job_name = ""
with timeout(minutes=5):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {
'encoding': 'latin1'
}
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
prefix = 'test_byo_estimator'
key = 'recordio-pb-data'
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(
prefix, 'train',
key))
estimator = Estimator(image_name=image_name,
role='SageMakerRole',
train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session,
base_job_name='test-byo')
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type='binary_classifier')
# training labels must be 'float32'
estimator.fit({'train': s3_train_data}, wait=False)
training_job_name = estimator.latest_training_job.name
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
estimator = Estimator.attach(training_job_name=training_job_name,
sagemaker_session=sagemaker_session)
model = estimator.create_model()
predictor = model.deploy(1,
'ml.m4.xlarge',
endpoint_name=endpoint_name)
predictor.serializer = fm_serializer
predictor.content_type = 'application/json'
predictor.deserializer = sagemaker.predictor.json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result['predictions']) == 10
for prediction in result['predictions']:
assert prediction['score'] is not None
assert estimator.train_image() == image_name
def test_async_byo_estimator(sagemaker_session, region):
image_name = registry(region) + "/factorization-machines:1"
endpoint_name = unique_name_from_base("byo")
training_data_path = os.path.join(DATA_DIR, "dummy_tensor")
job_name = unique_name_from_base("byo")
with timeout(minutes=5):
data_path = os.path.join(DATA_DIR, "one_p_mnist", "mnist.pkl.gz")
pickle_args = {} if sys.version_info.major == 2 else {
"encoding": "latin1"
}
with gzip.open(data_path, "rb") as f:
train_set, _, _ = pickle.load(f, **pickle_args)
prefix = "test_byo_estimator"
key = "recordio-pb-data"
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(
prefix, "train",
key))
estimator = Estimator(
image_name=image_name,
role="SageMakerRole",
train_instance_count=1,
train_instance_type="ml.c4.xlarge",
sagemaker_session=sagemaker_session,
)
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type="binary_classifier")
# training labels must be 'float32'
estimator.fit({"train": s3_train_data}, wait=False, job_name=job_name)
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
estimator = Estimator.attach(training_job_name=job_name,
sagemaker_session=sagemaker_session)
model = estimator.create_model()
predictor = model.deploy(1,
"ml.m4.xlarge",
endpoint_name=endpoint_name)
predictor.serializer = fm_serializer
predictor.content_type = "application/json"
predictor.deserializer = sagemaker.predictor.json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result["predictions"]) == 10
for prediction in result["predictions"]:
assert prediction["score"] is not None
assert estimator.train_image() == image_name
def test_async_byo_estimator(sagemaker_session, region):
image_name = registry(region) + "/factorization-machines:1"
endpoint_name = name_from_base('byo')
training_job_name = ""
with timeout(minutes=5):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
# take 100 examples for faster execution
vectors = np.array([t.tolist() for t in train_set[0][:100]]).astype('float32')
labels = np.where(np.array([t.tolist() for t in train_set[1][:100]]) == 0, 1.0, 0.0).astype('float32')
buf = io.BytesIO()
write_numpy_to_dense_tensor(buf, vectors, labels)
buf.seek(0)
bucket = sagemaker_session.default_bucket()
prefix = 'test_byo_estimator'
key = 'recordio-pb-data'
boto3.resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'train', key)).upload_fileobj(buf)
s3_train_data = 's3://{}/{}/train/{}'.format(bucket, prefix, key)
estimator = Estimator(image_name=image_name,
role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session, base_job_name='test-byo')
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type='binary_classifier')
# training labels must be 'float32'
estimator.fit({'train': s3_train_data}, wait=False)
training_job_name = estimator.latest_training_job.name
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
estimator = Estimator.attach(training_job_name=training_job_name, sagemaker_session=sagemaker_session)
model = estimator.create_model()
predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name)
predictor.serializer = fm_serializer
predictor.content_type = 'application/json'
predictor.deserializer = sagemaker.predictor.json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result['predictions']) == 10
for prediction in result['predictions']:
assert prediction['score'] is not None
assert estimator.train_image() == image_name
def test_generic_to_fit_no_hps(sagemaker_session):
e = Estimator(IMAGE_NAME, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, output_path=OUTPUT_PATH,
sagemaker_session=sagemaker_session)
e.fit({'train': 's3://bucket/training-prefix'})
sagemaker_session.train.assert_called_once()
assert len(sagemaker_session.train.call_args[0]) == 0
args = sagemaker_session.train.call_args[1]
assert args['job_name'].startswith(IMAGE_NAME)
args.pop('job_name')
args.pop('role')
assert args == BASE_TRAIN_CALL
def test_install_requirements(capsys):
estimator = Estimator(
image_name="sagemaker-training-toolkit-test:dummy",
role="SageMakerRole",
train_instance_count=1,
train_instance_type="local",
)
estimator.fit()
stdout = capsys.readouterr().out
assert "Installing collected packages: pyfiglet, train.py" in stdout
assert "Successfully installed pyfiglet-0.8.post1 train.py-1.0.0" in stdout
assert "Reporting training SUCCESS" in stdout
def test_estimator_transformer_creation(sagemaker_session):
estimator = Estimator(image_name=IMAGE_NAME, role=ROLE, train_instance_count=INSTANCE_COUNT,
train_instance_type=INSTANCE_TYPE, sagemaker_session=sagemaker_session)
estimator.latest_training_job = _TrainingJob(sagemaker_session, JOB_NAME)
sagemaker_session.create_model_from_job.return_value = JOB_NAME
transformer = estimator.transformer(INSTANCE_COUNT, INSTANCE_TYPE)
sagemaker_session.create_model_from_job.assert_called_with(JOB_NAME, role=None)
assert isinstance(transformer, Transformer)
assert transformer.sagemaker_session == sagemaker_session
assert transformer.instance_count == INSTANCE_COUNT
assert transformer.instance_type == INSTANCE_TYPE
assert transformer.model_name == JOB_NAME
assert transformer.tags is None
def create_estimator(params, sagemaker_role):
train_repository_uri = params['train-image-uri']
instance_type = 'ml.p3.2xlarge'
metric_definitions = [{
'Name': 'val:mAP',
'Regex': 'Average Precision \(AP\) \@\[ IoU=0.50:0.95 \| area= all \| maxDets=100 \] = ([0-9\\.]+)'
}]
estimator = Estimator(
image_uri=train_repository_uri,
role=sagemaker_role,
metric_definitions=metric_definitions,
instance_count=1,
instance_type=instance_type,
hyperparameters={
'batch-size': params['hyperparameters']['batch-size'],
'test-batch-size': 4,
'lr': 0.01,
'epochs': params['hyperparameters']['epoch'],
'experiment-name': params['experiment-name'],
'mlflow-server': params['mlflow-server-uri']
},
output_path=params['train-output-path'])
return estimator
def test_transform_byo_estimator(sagemaker_session, cpu_instance_type):
data_path = os.path.join(DATA_DIR, "one_p_mnist")
pickle_args = {} if sys.version_info.major == 2 else {"encoding": "latin1"}
tags = [{"Key": "some-tag", "Value": "value-for-tag"}]
# Load the data into memory as numpy arrays
train_set_path = os.path.join(data_path, "mnist.pkl.gz")
with gzip.open(train_set_path, "rb") as f:
train_set, _, _ = pickle.load(f, **pickle_args)
kmeans = KMeans(
role="SageMakerRole",
train_instance_count=1,
train_instance_type=cpu_instance_type,
k=10,
sagemaker_session=sagemaker_session,
output_path="s3://{}/".format(sagemaker_session.default_bucket()),
)
# set kmeans specific hp
kmeans.init_method = "random"
kmeans.max_iterators = 1
kmeans.tol = 1
kmeans.num_trials = 1
kmeans.local_init_method = "kmeans++"
kmeans.half_life_time_size = 1
kmeans.epochs = 1
records = kmeans.record_set(train_set[0][:100])
job_name = unique_name_from_base("test-kmeans-attach")
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
kmeans.fit(records, job_name=job_name)
estimator = Estimator.attach(training_job_name=job_name, sagemaker_session=sagemaker_session)
estimator._enable_network_isolation = True
transform_input_path = os.path.join(data_path, "transform_input.csv")
transform_input_key_prefix = "integ-test-data/one_p_mnist/transform"
transform_input = kmeans.sagemaker_session.upload_data(
path=transform_input_path, key_prefix=transform_input_key_prefix
)
transformer = estimator.transformer(1, cpu_instance_type, tags=tags)
transformer.transform(transform_input, content_type="text/csv")
with timeout_and_delete_model_with_transformer(
transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES
):
transformer.wait()
model_desc = sagemaker_session.sagemaker_client.describe_model(
ModelName=transformer.model_name
)
assert model_desc["EnableNetworkIsolation"]
model_tags = sagemaker_session.sagemaker_client.list_tags(
ResourceArn=model_desc["ModelArn"]
)["Tags"]
assert tags == model_tags
def estimator(sagemaker_session):
return Estimator(IMAGE_NAME,
ROLE,
TRAIN_INSTANCE_COUNT,
TRAIN_INSTANCE_TYPE,
output_path='s3://bucket/prefix',
sagemaker_session=sagemaker_session)
def create_estimator(self, role, output_path, hyperparameters,
sagemaker_session, **kwargs):
estimator = Estimator(
self.algo_image_uri,
role=role,
instance_count=self.training_resource_config["instance_count"],
instance_type=self.training_resource_config["instance_type"],
output_path=output_path,
sagemaker_session=sagemaker_session,
**kwargs,
)
hyperparameters.update(self.candidate_specific_static_hps)
estimator.set_hyperparameters(**hyperparameters)
return estimator
def test_training_step(sagemaker_session):
estimator = Estimator(
image_uri=IMAGE_URI,
role=ROLE,
instance_count=1,
instance_type="c4.4xlarge",
profiler_config=ProfilerConfig(system_monitor_interval_millis=500),
rules=[],
sagemaker_session=sagemaker_session,
)
inputs = TrainingInput(f"s3://{BUCKET}/train_manifest")
cache_config = CacheConfig(enable_caching=True, expire_after="PT1H")
step = TrainingStep(name="MyTrainingStep",
estimator=estimator,
inputs=inputs,
cache_config=cache_config)
assert step.to_request() == {
"Name": "MyTrainingStep",
"Type": "Training",
"Arguments": {
"AlgorithmSpecification": {
"TrainingImage": IMAGE_URI,
"TrainingInputMode": "File"
},
"InputDataConfig": [{
"ChannelName": "training",
"DataSource": {
"S3DataSource": {
"S3DataDistributionType": "FullyReplicated",
"S3DataType": "S3Prefix",
"S3Uri": f"s3://{BUCKET}/train_manifest",
}
},
}],
"OutputDataConfig": {
"S3OutputPath": f"s3://{BUCKET}/"
},
"ResourceConfig": {
"InstanceCount": 1,
"InstanceType": "c4.4xlarge",
"VolumeSizeInGB": 30,
},
"RoleArn":
ROLE,
"StoppingCondition": {
"MaxRuntimeInSeconds": 86400
},
"ProfilerConfig": {
"ProfilingIntervalInMilliseconds": 500,
"S3OutputPath": f"s3://{BUCKET}/",
},
},
"CacheConfig": {
"Enabled": True,
"ExpireAfter": "PT1H"
},
}
assert step.properties.TrainingJobName.expr == {
"Get": "Steps.MyTrainingStep.TrainingJobName"
}
def test_byo_airflow_config_uploads_data_source_to_s3_when_inputs_provided(
sagemaker_session, cpu_instance_type
):
with timeout(seconds=AIRFLOW_CONFIG_TIMEOUT_IN_SECONDS):
training_data_path = os.path.join(DATA_DIR, "dummy_tensor")
data_source_location = "test-airflow-config-{}".format(sagemaker_timestamp())
inputs = sagemaker_session.upload_data(
path=training_data_path, key_prefix=os.path.join(data_source_location, "train")
)
estimator = Estimator(
image_name=get_image_uri(
sagemaker_session.boto_session.region_name, "factorization-machines"
),
role=ROLE,
train_instance_count=SINGLE_INSTANCE_COUNT,
train_instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
)
training_config = _build_airflow_workflow(
estimator=estimator, instance_type=cpu_instance_type, inputs=inputs
)
_assert_that_s3_url_contains_data(
sagemaker_session,
training_config["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"],
)
def test_distributed_gpu_local_mode(LocalSession):
with pytest.raises(RuntimeError):
Estimator(IMAGE_NAME,
ROLE,
3,
'local_gpu',
output_path='s3://bucket/prefix')
def test_async_byo_estimator(sagemaker_session, region, cpu_instance_type,
training_set):
image_uri = image_uris.retrieve("factorization-machines", region)
endpoint_name = unique_name_from_base("byo")
training_data_path = os.path.join(DATA_DIR, "dummy_tensor")
job_name = unique_name_from_base("byo")
with timeout(minutes=5):
prefix = "test_byo_estimator"
key = "recordio-pb-data"
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(
prefix, "train",
key))
estimator = Estimator(
image_uri=image_uri,
role="SageMakerRole",
instance_count=1,
instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
)
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type="binary_classifier")
# training labels must be 'float32'
estimator.fit({"train": s3_train_data}, wait=False, job_name=job_name)
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
estimator = Estimator.attach(training_job_name=job_name,
sagemaker_session=sagemaker_session)
model = estimator.create_model()
predictor = model.deploy(
1,
cpu_instance_type,
endpoint_name=endpoint_name,
serializer=_FactorizationMachineSerializer(),
deserializer=sagemaker.deserializers.JSONDeserializer(),
)
result = predictor.predict(training_set[0][:10])
assert len(result["predictions"]) == 10
for prediction in result["predictions"]:
assert prediction["score"] is not None
assert estimator.training_image_uri() == image_uri
def estimator(sagemaker_session):
return Estimator(
image_uri=IMAGE_URI,
role=ROLE,
instance_count=1,
instance_type="ml.c4.4xlarge",
sagemaker_session=sagemaker_session,
)
def estimator_knn(sagemaker_session, cpu_instance_type):
knn_image = image_uris.retrieve("knn", sagemaker_session.boto_region_name)
estimator = Estimator(
image_uri=knn_image,
role=EXECUTION_ROLE,
instance_count=1,
instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
)
estimator.set_hyperparameters(k=10,
sample_size=500,
feature_dim=784,
mini_batch_size=100,
predictor_type="regressor")
return estimator
def test_async_byo_estimator(sagemaker_session, region):
image_name = registry(region) + "/factorization-machines:1"
endpoint_name = name_from_base('byo')
training_job_name = ""
with timeout(minutes=5):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
# take 100 examples for faster execution
vectors = np.array([t.tolist() for t in train_set[0][:100]]).astype('float32')
labels = np.where(np.array([t.tolist() for t in train_set[1][:100]]) == 0, 1.0, 0.0).astype('float32')
buf = io.BytesIO()
write_numpy_to_dense_tensor(buf, vectors, labels)
buf.seek(0)
bucket = sagemaker_session.default_bucket()
prefix = 'test_byo_estimator'
key = 'recordio-pb-data'
boto3.resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'train', key)).upload_fileobj(buf)
s3_train_data = 's3://{}/{}/train/{}'.format(bucket, prefix, key)
estimator = Estimator(image_name=image_name,
role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session, base_job_name='test-byo')
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type='binary_classifier')
# training labels must be 'float32'
estimator.fit({'train': s3_train_data}, wait=False)
training_job_name = estimator.latest_training_job.name
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
estimator = Estimator.attach(training_job_name=training_job_name, sagemaker_session=sagemaker_session)
model = estimator.create_model()
predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name)
predictor.serializer = fm_serializer
predictor.content_type = 'application/json'
predictor.deserializer = sagemaker.predictor.json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result['predictions']) == 10
for prediction in result['predictions']:
assert prediction['score'] is not None
assert estimator.train_image() == image_name
def test_generic_to_deploy(sagemaker_session):
e = Estimator(IMAGE_NAME,
ROLE,
INSTANCE_COUNT,
INSTANCE_TYPE,
output_path=OUTPUT_PATH,
sagemaker_session=sagemaker_session)
e.set_hyperparameters(**HYPERPARAMS)
e.fit({'train': 's3://bucket/training-prefix'})
predictor = e.deploy(INSTANCE_COUNT, INSTANCE_TYPE)
sagemaker_session.train.assert_called_once()
assert len(sagemaker_session.train.call_args[0]) == 0
args = sagemaker_session.train.call_args[1]
assert args['job_name'].startswith(IMAGE_NAME)
args.pop('job_name')
args.pop('role')
assert args == HP_TRAIN_CALL
sagemaker_session.create_model.assert_called_once()
args = sagemaker_session.create_model.call_args[0]
assert args[0].startswith(IMAGE_NAME)
assert args[1] == ROLE
assert args[2]['Image'] == IMAGE_NAME
assert args[2]['ModelDataUrl'] == MODEL_DATA
assert isinstance(predictor, RealTimePredictor)
assert predictor.endpoint.startswith(IMAGE_NAME)
assert predictor.sagemaker_session == sagemaker_session
def test_byo_estimator(sagemaker_session, region):
"""Use Factorization Machines algorithm as an example here.
First we need to prepare data for training. We take standard data set, convert it to the
format that the algorithm can process and upload it to S3.
Then we create the Estimator and set hyperparamets as required by the algorithm.
Next, we can call fit() with path to the S3.
Later the trained model is deployed and prediction is called against the endpoint.
Default predictor is updated with json serializer and deserializer.
"""
image_name = registry(region) + "/factorization-machines:1"
training_data_path = os.path.join(DATA_DIR, "dummy_tensor")
job_name = unique_name_from_base("byo")
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, "one_p_mnist", "mnist.pkl.gz")
pickle_args = {} if sys.version_info.major == 2 else {
"encoding": "latin1"
}
with gzip.open(data_path, "rb") as f:
train_set, _, _ = pickle.load(f, **pickle_args)
prefix = "test_byo_estimator"
key = "recordio-pb-data"
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(
prefix, "train",
key))
estimator = Estimator(
image_name=image_name,
role="SageMakerRole",
train_instance_count=1,
train_instance_type="ml.c4.xlarge",
sagemaker_session=sagemaker_session,
)
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type="binary_classifier")
# training labels must be 'float32'
estimator.fit({"train": s3_train_data}, job_name=job_name)
with timeout_and_delete_endpoint_by_name(job_name, sagemaker_session):
model = estimator.create_model()
predictor = model.deploy(1, "ml.m4.xlarge", endpoint_name=job_name)
predictor.serializer = fm_serializer
predictor.content_type = "application/json"
predictor.deserializer = sagemaker.predictor.json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result["predictions"]) == 10
for prediction in result["predictions"]:
assert prediction["score"] is not None
def estimator_fm(sagemaker_session, cpu_instance_type):
fm_image = image_uris.retrieve("factorization-machines",
sagemaker_session.boto_region_name)
estimator = Estimator(
image_uri=fm_image,
role=EXECUTION_ROLE,
instance_count=1,
instance_type=cpu_instance_type,
sagemaker_session=sagemaker_session,
)
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type="regressor")
return estimator
def test_start_new_not_local_mode_error(sagemaker_session):
training_job = _TrainingJob(sagemaker_session, JOB_NAME)
inputs = 'file://mybucket/train'
estimator = Estimator(IMAGE_NAME, ROLE, INSTANCE_COUNT, INSTANCE_TYPE,
output_path=OUTPUT_PATH, sagemaker_session=sagemaker_session)
with pytest.raises(ValueError) as error:
training_job.start_new(estimator, inputs)
assert 'File URIs are supported in local mode only. Please use a S3 URI instead.' == str(error)
def test_generic_to_deploy(sagemaker_session):
e = Estimator(IMAGE_NAME, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, output_path=OUTPUT_PATH,
sagemaker_session=sagemaker_session)
e.set_hyperparameters(**HYPERPARAMS)
e.fit({'train': 's3://bucket/training-prefix'})
predictor = e.deploy(INSTANCE_COUNT, INSTANCE_TYPE)
sagemaker_session.train.assert_called_once()
assert len(sagemaker_session.train.call_args[0]) == 0
args = sagemaker_session.train.call_args[1]
assert args['job_name'].startswith(IMAGE_NAME)
args.pop('job_name')
args.pop('role')
assert args == HP_TRAIN_CALL
sagemaker_session.create_model.assert_called_once()
args = sagemaker_session.create_model.call_args[0]
assert args[0].startswith(IMAGE_NAME)
assert args[1] == ROLE
assert args[2]['Image'] == IMAGE_NAME
assert args[2]['ModelDataUrl'] == MODEL_DATA
assert isinstance(predictor, RealTimePredictor)
assert predictor.endpoint.startswith(IMAGE_NAME)
assert predictor.sagemaker_session == sagemaker_session
def test_generic_training_job_analytics(sagemaker_session):
sagemaker_session.sagemaker_client.describe_training_job = Mock(name='describe_training_job', return_value={
'TuningJobArn': 'arn:aws:sagemaker:us-west-2:968277160000:hyper-parameter-tuning-job/mock-tuner',
'TrainingStartTime': 1530562991.299,
})
sagemaker_session.sagemaker_client.describe_hyper_parameter_tuning_job = Mock(
name='describe_hyper_parameter_tuning_job',
return_value={
'TrainingJobDefinition': {
"AlgorithmSpecification": {
"TrainingImage": "some-image-url",
"TrainingInputMode": "File",
"MetricDefinitions": [
{
"Name": "train:loss",
"Regex": "train_loss=([0-9]+\\.[0-9]+)"
},
{
"Name": "validation:loss",
"Regex": "valid_loss=([0-9]+\\.[0-9]+)"
}
]
}
}
}
)
e = Estimator(IMAGE_NAME, ROLE, INSTANCE_COUNT, INSTANCE_TYPE, output_path=OUTPUT_PATH,
sagemaker_session=sagemaker_session)
with pytest.raises(ValueError) as err: # noqa: F841
# No training job yet
a = e.training_job_analytics
assert a is not None # This line is never reached
e.set_hyperparameters(**HYPERPARAMS)
e.fit({'train': 's3://bucket/training-prefix'})
a = e.training_job_analytics
assert a is not None
def test_async_byo_estimator(sagemaker_session, region):
image_name = registry(region) + "/factorization-machines:1"
endpoint_name = name_from_base('byo')
training_data_path = os.path.join(DATA_DIR, 'dummy_tensor')
training_job_name = ""
with timeout(minutes=5):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
prefix = 'test_byo_estimator'
key = 'recordio-pb-data'
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(prefix, 'train', key))
estimator = Estimator(image_name=image_name,
role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session, base_job_name='test-byo')
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type='binary_classifier')
# training labels must be 'float32'
estimator.fit({'train': s3_train_data}, wait=False)
training_job_name = estimator.latest_training_job.name
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
estimator = Estimator.attach(training_job_name=training_job_name, sagemaker_session=sagemaker_session)
model = estimator.create_model()
predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name)
predictor.serializer = fm_serializer
predictor.content_type = 'application/json'
predictor.deserializer = sagemaker.predictor.json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result['predictions']) == 10
for prediction in result['predictions']:
assert prediction['score'] is not None
assert estimator.train_image() == image_name
def test_byo_estimator(sagemaker_session, region):
"""Use Factorization Machines algorithm as an example here.
First we need to prepare data for training. We take standard data set, convert it to the
format that the algorithm can process and upload it to S3.
Then we create the Estimator and set hyperparamets as required by the algorithm.
Next, we can call fit() with path to the S3.
Later the trained model is deployed and prediction is called against the endpoint.
Default predictor is updated with json serializer and deserializer.
"""
image_name = registry(region) + "/factorization-machines:1"
training_data_path = os.path.join(DATA_DIR, 'dummy_tensor')
with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
prefix = 'test_byo_estimator'
key = 'recordio-pb-data'
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(prefix, 'train', key))
estimator = Estimator(image_name=image_name,
role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session, base_job_name='test-byo')
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type='binary_classifier')
# training labels must be 'float32'
estimator.fit({'train': s3_train_data})
endpoint_name = name_from_base('byo')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model = estimator.create_model()
predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name)
predictor.serializer = fm_serializer
predictor.content_type = 'application/json'
predictor.deserializer = sagemaker.predictor.json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result['predictions']) == 10
for prediction in result['predictions']:
assert prediction['score'] is not None
return {'train': train_data_location, 'test': test_data_location}
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--ecr-repository', help='ECR repo where images will be pushed',
default='add-ecr-repo-here', required=True)
parser.add_argument('--tf-version', default='latest')
parser.add_argument('--instance-type', default='local', choices=['local', 'ml.c5.xlarge', 'ml.p2.xlarge'])
args = parser.parse_args()
tensorflow_version_tag = get_tensorflow_version_tag(args.tf_version, args.instance_type)
image_name = get_image_name(args.ecr_repository, args.tensorflow_version_tag)
build_image(image_name, tensorflow_version_tag)
if not args.instance_type.startswith('local'):
push_image(image_name)
hyperparameters = dict(batch_size=32, data_augmentation=True, learning_rate=.0001,
width_shift_range=.1, height_shift_range=.1)
estimator = Estimator(image_name, role='SageMakerRole', train_instance_count=1,
train_instance_type=args.instance_type, hyperparameters=hyperparameters)
channels = upload_training_data()
estimator.fit(channels)
def test_byo_estimator(sagemaker_session, region):
"""Use Factorization Machines algorithm as an example here.
First we need to prepare data for training. We take standard data set, convert it to the
format that the algorithm can process and upload it to S3.
Then we create the Estimator and set hyperparamets as required by the algorithm.
Next, we can call fit() with path to the S3.
Later the trained model is deployed and prediction is called against the endpoint.
Default predictor is updated with json serializer and deserializer.
"""
image_name = registry(region) + "/factorization-machines:1"
with timeout(minutes=15):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
# take 100 examples for faster execution
vectors = np.array([t.tolist() for t in train_set[0][:100]]).astype('float32')
labels = np.where(np.array([t.tolist() for t in train_set[1][:100]]) == 0, 1.0, 0.0).astype('float32')
buf = io.BytesIO()
write_numpy_to_dense_tensor(buf, vectors, labels)
buf.seek(0)
bucket = sagemaker_session.default_bucket()
prefix = 'test_byo_estimator'
key = 'recordio-pb-data'
boto3.resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'train', key)).upload_fileobj(buf)
s3_train_data = 's3://{}/{}/train/{}'.format(bucket, prefix, key)
estimator = Estimator(image_name=image_name,
role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session, base_job_name='test-byo')
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type='binary_classifier')
# training labels must be 'float32'
estimator.fit({'train': s3_train_data})
endpoint_name = name_from_base('byo')
with timeout_and_delete_endpoint_by_name(endpoint_name, sagemaker_session):
model = estimator.create_model()
predictor = model.deploy(1, 'ml.m4.xlarge', endpoint_name=endpoint_name)
predictor.serializer = fm_serializer
predictor.content_type = 'application/json'
predictor.deserializer = sagemaker.predictor.json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result['predictions']) == 10
for prediction in result['predictions']:
assert prediction['score'] is not None
def test_tuning_byo_estimator(sagemaker_session):
"""Use Factorization Machines algorithm as an example here.
First we need to prepare data for training. We take standard data set, convert it to the
format that the algorithm can process and upload it to S3.
Then we create the Estimator and set hyperparamets as required by the algorithm.
Next, we can call fit() with path to the S3.
Later the trained model is deployed and prediction is called against the endpoint.
Default predictor is updated with json serializer and deserializer.
"""
image_name = registry(sagemaker_session.boto_session.region_name) + '/factorization-machines:1'
training_data_path = os.path.join(DATA_DIR, 'dummy_tensor')
with timeout(minutes=TUNING_DEFAULT_TIMEOUT_MINUTES):
data_path = os.path.join(DATA_DIR, 'one_p_mnist', 'mnist.pkl.gz')
pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'}
with gzip.open(data_path, 'rb') as f:
train_set, _, _ = pickle.load(f, **pickle_args)
prefix = 'test_byo_estimator'
key = 'recordio-pb-data'
s3_train_data = sagemaker_session.upload_data(path=training_data_path,
key_prefix=os.path.join(prefix, 'train', key))
estimator = Estimator(image_name=image_name,
role='SageMakerRole', train_instance_count=1,
train_instance_type='ml.c4.xlarge',
sagemaker_session=sagemaker_session, base_job_name='test-byo')
estimator.set_hyperparameters(num_factors=10,
feature_dim=784,
mini_batch_size=100,
predictor_type='binary_classifier')
hyperparameter_ranges = {'mini_batch_size': IntegerParameter(100, 200)}
tuner = HyperparameterTuner(estimator=estimator, base_tuning_job_name='byo',
objective_metric_name='test:binary_classification_accuracy',
hyperparameter_ranges=hyperparameter_ranges,
max_jobs=2, max_parallel_jobs=2)
tuner.fit({'train': s3_train_data, 'test': s3_train_data}, include_cls_metadata=False)
print('Started hyperparameter tuning job with name:' + tuner.latest_tuning_job.name)
time.sleep(15)
tuner.wait()
best_training_job = tuner.best_training_job()
with timeout_and_delete_endpoint_by_name(best_training_job, sagemaker_session):
predictor = tuner.deploy(1, 'ml.m4.xlarge', endpoint_name=best_training_job)
predictor.serializer = _fm_serializer
predictor.content_type = 'application/json'
predictor.deserializer = json_deserializer
result = predictor.predict(train_set[0][:10])
assert len(result['predictions']) == 10
for prediction in result['predictions']:
assert prediction['score'] is not None