def _format_model_uri_input(model_uri, validate_uri=True):
"""Placeholder docstring"""
if isinstance(model_uri, string_types
) and validate_uri and model_uri.startswith("s3://"):
return TrainingInput(
model_uri,
input_mode="File",
distribution="FullyReplicated",
content_type="application/x-sagemaker-model",
)
if isinstance(model_uri, string_types
) and validate_uri and model_uri.startswith("file://"):
return file_input(model_uri)
if isinstance(model_uri, string_types) and validate_uri:
raise ValueError(
'Model URI must be a valid S3 or FILE URI: must start with "s3://" or '
'"file://')
if isinstance(model_uri, string_types):
return TrainingInput(
model_uri,
input_mode="File",
distribution="FullyReplicated",
content_type="application/x-sagemaker-model",
)
raise ValueError(
"Cannot format model URI {}. Expecting str".format(model_uri))
def get_pipeline(region, role, image_uri, model_path):
session = get_session(region)
if role is None:
role = sagemaker.session.get_execution_role(session)
train_data_param = ParameterString(name='train-data')
validation_data_param = ParameterString(name='validation-data')
image_uri_param = ParameterString(name='image-uri')
model_path_param = ParameterString(name='model-path')
estimator = Estimator(image_uri=image_uri,
instance_type='ml.m5.xlarge',
instance_count=1,
output_path=model_path,
sagemaker_session=session,
role=role)
### Your Pipeline definition goes here ....
###########################################
step_train = TrainingStep(name="iris-model-train",
estimator=estimator,
inputs={
"train":
TrainingInput(s3_data=train_data_param,
content_type='text/csv'),
"validation":
TrainingInput(s3_data=validation_data_param,
content_type='text/csv')
})
step_register = RegisterModel(
name='iris-model-register',
model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,
estimator=estimator,
content_types=["text/csv"],
response_types=["text/csv"],
inference_instances=["ml.t2.medium", "ml.m5.large"],
transform_instances=["ml.m5.large"],
model_package_group_name='iris-model')
pipeline = Pipeline(name='iris-model-pipeline',
steps=[step_train, step_register],
parameters=[train_data_param, validation_data_param],
sagemaker_session=session)
### end of Pipeline definition
###########################################
return pipeline
def _format_string_uri_input(
uri_input,
validate_uri=True,
content_type=None,
input_mode=None,
compression=None,
target_attribute_name=None,
):
"""
Args:
uri_input:
validate_uri:
content_type:
input_mode:
compression:
target_attribute_name:
"""
if isinstance(uri_input,
str) and validate_uri and uri_input.startswith("s3://"):
s3_input_result = TrainingInput(
uri_input,
content_type=content_type,
input_mode=input_mode,
compression=compression,
target_attribute_name=target_attribute_name,
)
return s3_input_result
if isinstance(
uri_input,
str) and validate_uri and uri_input.startswith("file://"):
return file_input(uri_input)
if isinstance(uri_input, str) and validate_uri:
raise ValueError(
'URI input {} must be a valid S3 or FILE URI: must start with "s3://" or '
'"file://"'.format(uri_input))
if isinstance(uri_input, str):
s3_input_result = TrainingInput(
uri_input,
content_type=content_type,
input_mode=input_mode,
compression=compression,
target_attribute_name=target_attribute_name,
)
return s3_input_result
if isinstance(uri_input, (TrainingInput, file_input, FileSystemInput)):
return uri_input
raise ValueError(
"Cannot format input {}. Expecting one of str, TrainingInput, file_input or "
"FileSystemInput".format(uri_input))
def parseInputsAndAllowAccess(args, sm_project):
input_data_path = None
distribution = "FullyReplicated"
if args.input_path:
input_data_path, distribution, subdir = args.input_path[0]
if input_data_path.lower().startswith("s3://"):
input_data_path = sagemaker.s3.s3_path_join(
input_data_path, subdir)
else:
input_data_path = os.path.join(input_data_path, subdir)
inputs = dict()
if args.input_task:
for (input_name, task_name, ttype, distribution,
subdir) in args.input_task:
inputs[input_name] = sm_project.getInputConfig(
task_name, ttype, distribution=distribution, subdir=subdir)
if args.input_s3:
for (input_name, s3_uri, distribution, subdir) in args.input_s3:
s3_uri = sagemaker.s3.s3_path_join(s3_uri, subdir)
bucket, _ = sagemaker.s3.parse_s3_url(s3_uri)
sm_project.allowAccessToS3Bucket(bucket)
inputs[input_name] = TrainingInput(s3_uri,
distribution=distribution)
return input_data_path, distribution, inputs
def __init__(
self,
name: str,
sagemaker_session,
role,
model_data: str,
entry_point: str,
source_dir: str = None,
dependencies: List = None,
depends_on: List[str] = None,
**kwargs,
):
"""Constructs a TrainingStep, given an `EstimatorBase` instance.
In addition to the estimator instance, the other arguments are those that are supplied to
the `fit` method of the `sagemaker.estimator.Estimator`.
Args:
name (str): The name of the training step.
estimator (EstimatorBase): A `sagemaker.estimator.EstimatorBase` instance.
inputs (TrainingInput): A `sagemaker.inputs.TrainingInput` instance. Defaults to `None`.
"""
# yeah, go ahead and save the originals for now
self._model_data = model_data
self.sagemaker_session = sagemaker_session
self.role = role
if isinstance(model_data, Properties):
self._model_prefix = model_data
self._model_archive = "model.tar.gz"
else:
self._model_prefix = "/".join(self._model_data.split("/")[:-1])
self._model_archive = self._model_data.split("/")[-1]
self._entry_point = entry_point
self._entry_point_basename = os.path.basename(self._entry_point)
self._source_dir = source_dir
self._dependencies = dependencies
# the real estimator and inputs
repacker = SKLearn(
framework_version=FRAMEWORK_VERSION,
instance_type=INSTANCE_TYPE,
entry_point=REPACK_SCRIPT,
source_dir=self._source_dir,
dependencies=self._dependencies,
sagemaker_session=self.sagemaker_session,
role=self.role,
hyperparameters={
"inference_script": self._entry_point_basename,
"model_archive": self._model_archive,
},
**kwargs,
)
repacker.disable_profiler = True
inputs = TrainingInput(self._model_prefix)
# super!
super(_RepackModelStep, self).__init__(name=name,
depends_on=depends_on,
estimator=repacker,
inputs=inputs)
def test_training_step(sagemaker_session):
estimator = Estimator(
image_uri=IMAGE_URI,
role=ROLE,
instance_count=1,
instance_type="c4.4xlarge",
profiler_config=ProfilerConfig(system_monitor_interval_millis=500),
rules=[],
sagemaker_session=sagemaker_session,
)
inputs = TrainingInput(f"s3://{BUCKET}/train_manifest")
cache_config = CacheConfig(enable_caching=True, expire_after="PT1H")
step = TrainingStep(name="MyTrainingStep",
estimator=estimator,
inputs=inputs,
cache_config=cache_config)
assert step.to_request() == {
"Name": "MyTrainingStep",
"Type": "Training",
"Arguments": {
"AlgorithmSpecification": {
"TrainingImage": IMAGE_URI,
"TrainingInputMode": "File"
},
"InputDataConfig": [{
"ChannelName": "training",
"DataSource": {
"S3DataSource": {
"S3DataDistributionType": "FullyReplicated",
"S3DataType": "S3Prefix",
"S3Uri": f"s3://{BUCKET}/train_manifest",
}
},
}],
"OutputDataConfig": {
"S3OutputPath": f"s3://{BUCKET}/"
},
"ResourceConfig": {
"InstanceCount": 1,
"InstanceType": "c4.4xlarge",
"VolumeSizeInGB": 30,
},
"RoleArn":
ROLE,
"StoppingCondition": {
"MaxRuntimeInSeconds": 86400
},
"ProfilerConfig": {
"ProfilingIntervalInMilliseconds": 500,
"S3OutputPath": f"s3://{BUCKET}/",
},
},
"CacheConfig": {
"Enabled": True,
"ExpireAfter": "PT1H"
},
}
assert step.properties.TrainingJobName.expr == {
"Get": "Steps.MyTrainingStep.TrainingJobName"
}
def test_tuning_step_with_single_algo_tuner(pipeline_session, entry_point):
inputs = TrainingInput(
s3_data=f"s3://{pipeline_session.default_bucket()}/training-data")
pytorch_estimator = PyTorch(
entry_point=entry_point,
role=sagemaker.get_execution_role(),
framework_version="1.5.0",
py_version="py3",
instance_count=1,
instance_type="ml.m5.xlarge",
sagemaker_session=pipeline_session,
enable_sagemaker_metrics=True,
max_retry_attempts=3,
)
hyperparameter_ranges = {
"batch-size": IntegerParameter(64, 128),
}
tuner = HyperparameterTuner(
estimator=pytorch_estimator,
objective_metric_name="test:acc",
objective_type="Maximize",
hyperparameter_ranges=hyperparameter_ranges,
metric_definitions=[{
"Name": "test:acc",
"Regex": "Overall test accuracy: (.*?);"
}],
max_jobs=2,
max_parallel_jobs=2,
)
with warnings.catch_warnings(record=True) as w:
step_args = tuner.fit(inputs=inputs)
assert len(w) == 1
assert issubclass(w[-1].category, UserWarning)
assert "Running within a PipelineSession" in str(w[-1].message)
with warnings.catch_warnings(record=True) as w:
step = TuningStep(
name="MyTuningStep",
step_args=step_args,
)
assert len(w) == 0
pipeline = Pipeline(
name="MyPipeline",
steps=[step],
sagemaker_session=pipeline_session,
)
assert json.loads(pipeline.definition())["Steps"][0] == {
"Name": "MyTuningStep",
"Type": "Tuning",
"Arguments": step_args,
}
def getInputConfig(self,
output_type,
distribution="FullyReplicated",
subdir="",
return_s3uri=False):
uri = self.getOutputTargetUri(**{output_type: True})
if subdir:
uri = sagemaker.s3.s3_path_join(uri, subdir)
if return_s3uri:
return uri
return TrainingInput(uri, distribution=distribution)
def test_training_step(sagemaker_session):
estimator = Estimator(
image_uri=IMAGE_URI,
role=ROLE,
instance_count=1,
instance_type="c4.4xlarge",
sagemaker_session=sagemaker_session,
)
inputs = TrainingInput(f"s3://{BUCKET}/train_manifest")
step = TrainingStep(
name="MyTrainingStep",
estimator=estimator,
inputs=inputs,
)
assert step.to_request() == {
"Name": "MyTrainingStep",
"Type": "Training",
"Arguments": {
"AlgorithmSpecification": {
"TrainingImage": IMAGE_URI,
"TrainingInputMode": "File"
},
"InputDataConfig": [{
"ChannelName": "training",
"DataSource": {
"S3DataSource": {
"S3DataDistributionType": "FullyReplicated",
"S3DataType": "S3Prefix",
"S3Uri": f"s3://{BUCKET}/train_manifest",
}
},
}],
"OutputDataConfig": {
"S3OutputPath": f"s3://{BUCKET}/"
},
"ResourceConfig": {
"InstanceCount": 1,
"InstanceType": "c4.4xlarge",
"VolumeSizeInGB": 30,
},
"RoleArn":
ROLE,
"StoppingCondition": {
"MaxRuntimeInSeconds": 86400
},
},
}
assert step.properties.TrainingJobName.expr == {
"Get": "Steps.MyTrainingStep.TrainingJobName"
}
def create_training_step(params, estimator, execution_input):
prepro_output_data = params['prep-output-path']
training_input = TrainingInput(s3_data=prepro_output_data,
input_mode='FastFile')
training_step = TrainingStep(
"SageMaker Training Step",
estimator=estimator,
data={"training": training_input},
job_name=execution_input["TrainingJobName"],
wait_for_completion=True,
)
return training_step
def build_training_input(channel, i):
return TrainingInput(
s3_data=channel.local,
# distribution=None,
# compression=None,
# content_type=None,
record_wrapping=get_record_wrapping(channel.mode),
# content_type='application/x-recordio' if get_mode(
# mode) not in ['File'] else None,
s3_data_type=get_s3_data_type(channel.mode),
input_mode=get_mode(channel.mode),
attribute_names=channel.attributes,
shuffle_config=ShuffleConfig(123 + i) if channel.shuffle else None
# target_attribute_name=None,
# shuffle_config=None,
)
def test_training_step_no_profiler_warning(sagemaker_session):
estimator = TensorFlow(
entry_point=DUMMY_SCRIPT_PATH,
role=ROLE,
model_dir=False,
image_uri=IMAGE_URI,
source_dir="s3://mybucket/source",
framework_version="2.4.1",
py_version="py37",
disable_profiler=True,
instance_count=1,
instance_type="ml.p3.16xlarge",
sagemaker_session=sagemaker_session,
hyperparameters={
"batch-size": 500,
"epochs": 5,
},
debugger_hook_config=False,
distribution={"smdistributed": {
"dataparallel": {
"enabled": True
}
}},
)
inputs = TrainingInput(s3_data=f"s3://{BUCKET}/train_manifest")
cache_config = CacheConfig(enable_caching=True, expire_after="PT1H")
with warnings.catch_warnings(record=True) as w:
# profiler disabled, cache config not None
TrainingStep(name="MyTrainingStep",
estimator=estimator,
inputs=inputs,
cache_config=cache_config)
assert len(w) == 0
with warnings.catch_warnings(record=True) as w:
# profiler enabled, cache config is None
estimator.disable_profiler = False
TrainingStep(name="MyTrainingStep",
estimator=estimator,
inputs=inputs,
cache_config=None)
assert len(w) == 0
def get_pipeline(
region,
role=None,
default_bucket=None,
model_package_group_name="AbalonePackageGroup",
pipeline_name="AbalonePipeline",
base_job_prefix="Abalone",
):
"""Gets a SageMaker ML Pipeline instance working with on abalone data.
Args:
region: AWS region to create and run the pipeline.
role: IAM role to create and run steps and pipeline.
default_bucket: the bucket to use for storing the artifacts
Returns:
an instance of a pipeline
"""
sagemaker_session = get_session(region, default_bucket)
if role is None:
role = sagemaker.session.get_execution_role(sagemaker_session)
# Create cache configuration
cache_config = CacheConfig(enable_caching=True, expire_after="T30m")
# Create SKlean processor object
sklearn_processor = SKLearnProcessor(
framework_version="0.20.0",
role=role,
instance_type=processing_instance_type,
instance_count=processing_instance_count,
base_job_name="credit-processing-job"
)
# Use the sklearn_processor in a Sagemaker pipelines ProcessingStep
step_preprocess_data = ProcessingStep(
name="PreprocessCreditData",
processor=sklearn_processor,
cache_config=cache_config,
inputs=[
ProcessingInput(source=input_data, destination="/opt/ml/processing/input"),
],
outputs=[
ProcessingOutput(output_name="train", source="/opt/ml/processing/output/train"),
ProcessingOutput(output_name="validation", source="/opt/ml/processing/output/validation"),
ProcessingOutput(output_name="test", source="/opt/ml/processing/output/test"),
ProcessingOutput(output_name="baseline_with_headers", source="/opt/ml/processing/output/baseline")
],
code=os.path.join(BASE_DIR, "preprocessing.py"),
)
# Where to store the trained model
model_path = f"s3://{default_bucket}/CreditTrain"
# Fetch container to use for training
image_uri = sagemaker.image_uris.retrieve(
framework="xgboost",
region=region,
version="1.2-2",
py_version="py3",
instance_type=training_instance_type,
)
# Create XGBoost estimator object
xgb_estimator = Estimator(
image_uri=image_uri,
instance_type=training_instance_type,
instance_count=1,
output_path=model_path,
role=role,
disable_profiler=True,
)
# Specify hyperparameters
xgb_estimator.set_hyperparameters(max_depth=5,
eta=0.2,
gamma=4,
min_child_weight=6,
subsample=0.8,
objective='binary:logistic',
num_round=25)
# Use the xgb_estimator in a Sagemaker pipelines ProcessingStep.
# NOTE how the input to the training job directly references the output of the previous step.
step_train_model = TrainingStep(
name="TrainCreditModel",
estimator=xgb_estimator,
cache_config=cache_config,
inputs={
"train": TrainingInput(
s3_data=step_preprocess_data.properties.ProcessingOutputConfig.Outputs[
"train"
].S3Output.S3Uri,
content_type="text/csv"
),
"validation": TrainingInput(
s3_data=step_preprocess_data.properties.ProcessingOutputConfig.Outputs[
"validation"
].S3Output.S3Uri,
content_type="text/csv"
)
},
)
# Create ScriptProcessor object.
evaluate_model_processor = ScriptProcessor(
image_uri=image_uri,
command=["python3"],
instance_type=processing_instance_type,
instance_count=1,
base_job_name="script-credit-eval",
role=role,
)
# Create a PropertyFile
# We use a PropertyFile to be able to reference outputs from a processing step, for instance to use in a condition step, which we'll see later on.
# For more information, visit https://docs.aws.amazon.com/sagemaker/latest/dg/build-and-manage-propertyfile.html
evaluation_report = PropertyFile(
name="EvaluationReport",
output_name="evaluation",
path="evaluation.json"
)
# Use the evaluate_model_processor in a Sagemaker pipelines ProcessingStep.
step_evaluate_model = ProcessingStep(
name="EvaluateCreditModel",
processor=evaluate_model_processor,
cache_config=cache_config,
inputs=[
ProcessingInput(
source=step_train_model.properties.ModelArtifacts.S3ModelArtifacts,
destination="/opt/ml/processing/model"
),
ProcessingInput(
source=step_preprocess_data.properties.ProcessingOutputConfig.Outputs[
"test"
].S3Output.S3Uri,
destination="/opt/ml/processing/test"
)
],
outputs=[
ProcessingOutput(output_name="evaluation", source="/opt/ml/processing/evaluation"),
],
code=os.path.join(BASE_DIR, "evaluation.py"),
property_files=[evaluation_report],
)
model_metrics = ModelMetrics(
model_statistics=MetricsSource(
s3_uri="{}/evaluation.json".format(
step_evaluate_model.arguments["ProcessingOutputConfig"]["Outputs"][0]["S3Output"]["S3Uri"]
),
content_type="application/json"
)
)
# Crete a RegisterModel step, which registers your model with Sagemaker Model Registry.
step_register_model = RegisterModel(
name="RegisterCreditModel",
estimator=xgb_estimator,
model_data=step_train_model.properties.ModelArtifacts.S3ModelArtifacts,
content_types=["text/csv"],
response_types=["text/csv"],
inference_instances=["ml.t2.medium", "ml.m5.xlarge", "ml.m5.large"],
transform_instances=["ml.m5.xlarge"],
model_package_group_name=model_package_group_name,
approval_status=model_approval_status,
model_metrics=model_metrics
)
# Create Processor object using the model monitor image
baseline_processor = sagemaker.processing.Processor(
base_job_name="credit-risk-baseline-processor",
image_uri=sagemaker.image_uris.retrieve(framework='model-monitor', region='eu-west-1'),
role=role,
instance_count=1,
instance_type=processing_instance_type,
env = {
"dataset_format": "{\"csv\": {\"header\": true} }",
"dataset_source": "/opt/ml/processing/sm_input",
"output_path": "/opt/ml/processing/sm_output",
"publish_cloudwatch_metrics": "Disabled"
}
)
# Create a Sagemaker Pipeline step, using the baseline_processor.
step_create_data_baseline = ProcessingStep(
name="CreateModelQualityBaseline",
processor=baseline_processor,
cache_config=cache_config,
inputs=[
ProcessingInput(
source=step_preprocess_data.properties.ProcessingOutputConfig.Outputs[
"baseline_with_headers"
].S3Output.S3Uri,
destination="/opt/ml/processing/sm_input",
)
],
outputs=[
ProcessingOutput(
source="/opt/ml/processing/sm_output",
destination="s3://{}/{}/baseline".format(default_bucket, base_job_prefix),
output_name="baseline_result",
)
],
)
# Create Condition
cond_gte = ConditionGreaterThanOrEqualTo(
left=JsonGet(
step=step_evaluate_model,
property_file=evaluation_report,
json_path="binary_classification_metrics.accuracy.value"
),
right=0.7
)
# Create a Sagemaker Pipelines ConditionStep, using the condition we just created.
step_cond = ConditionStep(
name="AccuracyCondition",
conditions=[cond_gte],
if_steps=[step_register_model],
else_steps=[],
)
from sagemaker.workflow.pipeline import Pipeline
# Create a Sagemaker Pipeline
pipeline = Pipeline(
name=pipeline_name,
parameters=[
processing_instance_type,
processing_instance_count,
training_instance_type,
model_approval_status,
input_data,
],
steps=[step_preprocess_data, step_train_model, step_evaluate_model, step_create_data_baseline, step_cond],
)
return pipeline
def test_training_job_with_debugger_and_profiler(
sagemaker_session,
pipeline_name,
role,
pytorch_training_latest_version,
pytorch_training_latest_py_version,
):
instance_count = ParameterInteger(name="InstanceCount", default_value=1)
instance_type = ParameterString(name="InstanceType",
default_value="ml.m5.xlarge")
rules = [
Rule.sagemaker(rule_configs.vanishing_gradient()),
Rule.sagemaker(base_config=rule_configs.all_zero(),
rule_parameters={"tensor_regex": ".*"}),
Rule.sagemaker(rule_configs.loss_not_decreasing()),
]
debugger_hook_config = DebuggerHookConfig(
s3_output_path=
f"s3://{sagemaker_session.default_bucket()}/{uuid.uuid4()}/tensors")
base_dir = os.path.join(DATA_DIR, "pytorch_mnist")
script_path = os.path.join(base_dir, "mnist.py")
input_path = sagemaker_session.upload_data(
path=os.path.join(base_dir, "training"),
key_prefix="integ-test-data/pytorch_mnist/training",
)
inputs = TrainingInput(s3_data=input_path)
pytorch_estimator = PyTorch(
entry_point=script_path,
role="SageMakerRole",
framework_version=pytorch_training_latest_version,
py_version=pytorch_training_latest_py_version,
instance_count=instance_count,
instance_type=instance_type,
sagemaker_session=sagemaker_session,
rules=rules,
debugger_hook_config=debugger_hook_config,
)
step_train = TrainingStep(
name="pytorch-train",
estimator=pytorch_estimator,
inputs=inputs,
)
pipeline = Pipeline(
name=pipeline_name,
parameters=[instance_count, instance_type],
steps=[step_train],
sagemaker_session=sagemaker_session,
)
try:
response = pipeline.create(role)
create_arn = response["PipelineArn"]
execution = pipeline.start()
response = execution.describe()
assert response["PipelineArn"] == create_arn
try:
execution.wait(delay=10, max_attempts=60)
except WaiterError:
pass
execution_steps = execution.list_steps()
assert len(execution_steps) == 1
assert execution_steps[0].get("FailureReason", "") == ""
assert execution_steps[0]["StepName"] == "pytorch-train"
assert execution_steps[0]["StepStatus"] == "Succeeded"
training_job_arn = execution_steps[0]["Metadata"]["TrainingJob"]["Arn"]
job_description = sagemaker_session.sagemaker_client.describe_training_job(
TrainingJobName=training_job_arn.split("/")[1])
for index, rule in enumerate(rules):
config = job_description["DebugRuleConfigurations"][index]
assert config["RuleConfigurationName"] == rule.name
assert config["RuleEvaluatorImage"] == rule.image_uri
assert config["VolumeSizeInGB"] == 0
assert (config["RuleParameters"]["rule_to_invoke"] ==
rule.rule_parameters["rule_to_invoke"])
assert job_description[
"DebugHookConfig"] == debugger_hook_config._to_request_dict()
assert job_description["ProfilingStatus"] == "Enabled"
assert job_description["ProfilerConfig"][
"ProfilingIntervalInMilliseconds"] == 500
finally:
try:
pipeline.delete()
except Exception:
pass
def test_model_registration_with_model_repack(
sagemaker_session,
role,
pipeline_name,
region_name,
):
base_dir = os.path.join(DATA_DIR, "pytorch_mnist")
entry_point = os.path.join(base_dir, "mnist.py")
input_path = sagemaker_session.upload_data(
path=os.path.join(base_dir, "training"),
key_prefix="integ-test-data/pytorch_mnist/training",
)
inputs = TrainingInput(s3_data=input_path)
instance_count = ParameterInteger(name="InstanceCount", default_value=1)
instance_type = ParameterString(name="InstanceType",
default_value="ml.m5.xlarge")
good_enough_input = ParameterInteger(name="GoodEnoughInput",
default_value=1)
pytorch_estimator = PyTorch(
entry_point=entry_point,
role=role,
framework_version="1.5.0",
py_version="py3",
instance_count=instance_count,
instance_type=instance_type,
sagemaker_session=sagemaker_session,
)
step_train = TrainingStep(
name="pytorch-train",
estimator=pytorch_estimator,
inputs=inputs,
)
step_register = RegisterModel(
name="pytorch-register-model",
estimator=pytorch_estimator,
model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,
content_types=["*"],
response_types=["*"],
inference_instances=["*"],
transform_instances=["*"],
description="test-description",
entry_point=entry_point,
)
model = Model(
image_uri=pytorch_estimator.training_image_uri(),
model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,
sagemaker_session=sagemaker_session,
role=role,
)
model_inputs = CreateModelInput(
instance_type="ml.m5.large",
accelerator_type="ml.eia1.medium",
)
step_model = CreateModelStep(
name="pytorch-model",
model=model,
inputs=model_inputs,
)
step_cond = ConditionStep(
name="cond-good-enough",
conditions=[
ConditionGreaterThanOrEqualTo(left=good_enough_input, right=1)
],
if_steps=[step_train, step_register],
else_steps=[step_model],
)
pipeline = Pipeline(
name=pipeline_name,
parameters=[good_enough_input, instance_count, instance_type],
steps=[step_cond],
sagemaker_session=sagemaker_session,
)
try:
response = pipeline.create(role)
create_arn = response["PipelineArn"]
assert re.match(
fr"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}",
create_arn)
execution = pipeline.start(parameters={})
assert re.match(
fr"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}/execution/",
execution.arn,
)
execution = pipeline.start(parameters={"GoodEnoughInput": 0})
assert re.match(
fr"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}/execution/",
execution.arn,
)
finally:
try:
pipeline.delete()
except Exception:
pass
def test_three_step_definition(
sagemaker_session,
region_name,
role,
script_dir,
pipeline_name,
athena_dataset_definition,
):
framework_version = "0.20.0"
instance_type = ParameterString(name="InstanceType",
default_value="ml.m5.xlarge")
instance_count = ParameterInteger(name="InstanceCount", default_value=1)
output_prefix = ParameterString(name="OutputPrefix",
default_value="output")
input_data = f"s3://sagemaker-sample-data-{region_name}/processing/census/census-income.csv"
sklearn_processor = SKLearnProcessor(
framework_version=framework_version,
instance_type=instance_type,
instance_count=instance_count,
base_job_name="test-sklearn",
sagemaker_session=sagemaker_session,
role=role,
)
step_process = ProcessingStep(
name="my-process",
processor=sklearn_processor,
inputs=[
ProcessingInput(source=input_data,
destination="/opt/ml/processing/input"),
ProcessingInput(dataset_definition=athena_dataset_definition),
],
outputs=[
ProcessingOutput(output_name="train_data",
source="/opt/ml/processing/train"),
ProcessingOutput(
output_name="test_data",
source="/opt/ml/processing/test",
destination=Join(
on="/",
values=[
"s3:/",
sagemaker_session.default_bucket(),
"test-sklearn",
output_prefix,
ExecutionVariables.PIPELINE_EXECUTION_ID,
],
),
),
],
code=os.path.join(script_dir, "preprocessing.py"),
)
sklearn_train = SKLearn(
framework_version=framework_version,
entry_point=os.path.join(script_dir, "train.py"),
instance_type=instance_type,
sagemaker_session=sagemaker_session,
role=role,
)
step_train = TrainingStep(
name="my-train",
estimator=sklearn_train,
inputs=TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.
Outputs["train_data"].S3Output.S3Uri),
)
model = Model(
image_uri=sklearn_train.image_uri,
model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,
sagemaker_session=sagemaker_session,
role=role,
)
model_inputs = CreateModelInput(
instance_type="ml.m5.large",
accelerator_type="ml.eia1.medium",
)
step_model = CreateModelStep(
name="my-model",
model=model,
inputs=model_inputs,
)
pipeline = Pipeline(
name=pipeline_name,
parameters=[instance_type, instance_count, output_prefix],
steps=[step_process, step_train, step_model],
sagemaker_session=sagemaker_session,
)
definition = json.loads(pipeline.definition())
assert definition["Version"] == "2020-12-01"
assert set(tuple(param.items())
for param in definition["Parameters"]) == set([
tuple({
"Name": "InstanceType",
"Type": "String",
"DefaultValue": "ml.m5.xlarge"
}.items()),
tuple({
"Name": "InstanceCount",
"Type": "Integer",
"DefaultValue": 1
}.items()),
tuple({
"Name": "OutputPrefix",
"Type": "String",
"DefaultValue": "output"
}.items()),
])
steps = definition["Steps"]
assert len(steps) == 3
names_and_types = []
processing_args = {}
training_args = {}
for step in steps:
names_and_types.append((step["Name"], step["Type"]))
if step["Type"] == "Processing":
processing_args = step["Arguments"]
if step["Type"] == "Training":
training_args = step["Arguments"]
if step["Type"] == "Model":
model_args = step["Arguments"]
assert set(names_and_types) == set([
("my-process", "Processing"),
("my-train", "Training"),
("my-model", "Model"),
])
assert processing_args["ProcessingResources"]["ClusterConfig"] == {
"InstanceType": {
"Get": "Parameters.InstanceType"
},
"InstanceCount": {
"Get": "Parameters.InstanceCount"
},
"VolumeSizeInGB": 30,
}
assert training_args["ResourceConfig"] == {
"InstanceCount": 1,
"InstanceType": {
"Get": "Parameters.InstanceType"
},
"VolumeSizeInGB": 30,
}
assert training_args["InputDataConfig"][0]["DataSource"]["S3DataSource"][
"S3Uri"] == {
"Get":
"Steps.my-process.ProcessingOutputConfig.Outputs['train_data'].S3Output.S3Uri"
}
assert model_args["PrimaryContainer"]["ModelDataUrl"] == {
"Get": "Steps.my-train.ModelArtifacts.S3ModelArtifacts"
}
try:
response = pipeline.create(role)
create_arn = response["PipelineArn"]
assert re.match(
fr"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}",
create_arn,
)
finally:
try:
pipeline.delete()
except Exception:
pass
def get_pipeline(
region,
sagemaker_project_arn=None,
role=None,
default_bucket=None,
model_package_group_name="restatePackageGroup", # Choose any name
pipeline_name="restate-p-XXXXXXXXX", # You can find your pipeline name in the Studio UI (project -> Pipelines -> name)
base_job_prefix="restate", # Choose any name
):
"""Gets a SageMaker ML Pipeline instance working with on RE data.
Args:
region: AWS region to create and run the pipeline.
role: IAM role to create and run steps and pipeline.
default_bucket: the bucket to use for storing the artifacts
Returns:
an instance of a pipeline
"""
sagemaker_session = get_session(region, default_bucket)
if role is None:
role = sagemaker.session.get_execution_role(sagemaker_session)
# Parameters for pipeline execution
processing_instance_count = ParameterInteger(name="ProcessingInstanceCount", default_value=1)
processing_instance_type = ParameterString(
name="ProcessingInstanceType", default_value="ml.m5.2xlarge"
)
training_instance_type = ParameterString(
name="TrainingInstanceType", default_value="ml.m5.xlarge"
)
model_approval_status = ParameterString(
name="ModelApprovalStatus",
default_value="PendingManualApproval", # ModelApprovalStatus can be set to a default of "Approved" if you don't want manual approval.
)
input_data = ParameterString(
name="InputDataUrl",
default_value=f"", # Change this to point to the s3 location of your raw input data.
)
data_sources = []
# Sagemaker session
sess = sagemaker_session
# You can configure this with your own bucket name, e.g.
# bucket = "my-bucket"
bucket = sess.default_bucket()
data_sources.append(
ProcessingInput(
input_name="restate-california",
dataset_definition=DatasetDefinition(
local_path="/opt/ml/processing/restate-california",
data_distribution_type="FullyReplicated",
# You can override below to point to other database or use different queries
athena_dataset_definition=AthenaDatasetDefinition(
catalog="AwsDataCatalog",
database="restate",
query_string="SELECT * FROM restate.california_10",
output_s3_uri=f"s3://{bucket}/athena/",
output_format="PARQUET",
),
),
)
)
print(f"Data Wrangler export storage bucket: {bucket}")
# unique flow export ID
flow_export_id = f"{time.strftime('%d-%H-%M-%S', time.gmtime())}-{str(uuid.uuid4())[:8]}"
flow_export_name = f"flow-{flow_export_id}"
# Output name is auto-generated from the select node's ID + output name from the flow file.
output_name = "99ae1ec3-dd5f-453c-bfae-721dac423cd7.default"
s3_output_prefix = f"export-{flow_export_name}/output"
s3_output_path = f"s3://{bucket}/{s3_output_prefix}"
print(f"Flow S3 export result path: {s3_output_path}")
processing_job_output = ProcessingOutput(
output_name=output_name,
source="/opt/ml/processing/output",
destination=s3_output_path,
s3_upload_mode="EndOfJob",
)
# name of the flow file which should exist in the current notebook working directory
flow_file_name = "sagemaker-pipeline/restate-athena-california.flow"
# Load .flow file from current notebook working directory
#!echo "Loading flow file from current notebook working directory: $PWD"
with open(flow_file_name) as f:
flow = json.load(f)
# Upload flow to S3
s3_client = boto3.client("s3")
s3_client.upload_file(
flow_file_name,
bucket,
f"data_wrangler_flows/{flow_export_name}.flow",
ExtraArgs={"ServerSideEncryption": "aws:kms"},
)
flow_s3_uri = f"s3://{bucket}/data_wrangler_flows/{flow_export_name}.flow"
print(f"Data Wrangler flow {flow_file_name} uploaded to {flow_s3_uri}")
## Input - Flow: restate-athena-russia.flow
flow_input = ProcessingInput(
source=flow_s3_uri,
destination="/opt/ml/processing/flow",
input_name="flow",
s3_data_type="S3Prefix",
s3_input_mode="File",
s3_data_distribution_type="FullyReplicated",
)
# IAM role for executing the processing job.
iam_role = role
# Unique processing job name. Give a unique name every time you re-execute processing jobs
processing_job_name = f"data-wrangler-flow-processing-{flow_export_id}"
# Data Wrangler Container URL.
container_uri = sagemaker.image_uris.retrieve(
framework="data-wrangler", # we are using the Sagemaker built in xgboost algorithm
region=region,
)
# Processing Job Instance count and instance type.
instance_count = 2
instance_type = "ml.m5.4xlarge"
# Size in GB of the EBS volume to use for storing data during processing
volume_size_in_gb = 30
# Content type for each output. Data Wrangler supports CSV as default and Parquet.
output_content_type = "CSV"
# Network Isolation mode; default is off
enable_network_isolation = False
# List of tags to be passed to the processing job
user_tags = []
# Output configuration used as processing job container arguments
output_config = {output_name: {"content_type": output_content_type}}
# KMS key for per object encryption; default is None
kms_key = None
processor = Processor(
role=iam_role,
image_uri=container_uri,
instance_count=instance_count,
instance_type=instance_type,
volume_size_in_gb=volume_size_in_gb,
network_config=NetworkConfig(enable_network_isolation=enable_network_isolation),
sagemaker_session=sess,
output_kms_key=kms_key,
tags=user_tags,
)
data_wrangler_step = ProcessingStep(
name="DataWranglerProcess",
processor=processor,
inputs=[flow_input] + data_sources,
outputs=[processing_job_output],
job_arguments=[f"--output-config '{json.dumps(output_config)}'"],
)
# Processing step for feature engineering
# this processor does not have awswrangler installed
sklearn_processor = SKLearnProcessor(
framework_version="0.23-1",
instance_type=processing_instance_type,
instance_count=processing_instance_count,
base_job_name=f"{base_job_prefix}/sklearn-restate-preprocess", # choose any name
sagemaker_session=sagemaker_session,
role=role,
)
step_process = ProcessingStep(
name="Preprocess", # choose any name
processor=sklearn_processor,
inputs=[
ProcessingInput(
source=data_wrangler_step.properties.ProcessingOutputConfig.Outputs[
output_name
].S3Output.S3Uri,
destination="/opt/ml/processing/data/raw-data-dir",
)
],
outputs=[
ProcessingOutput(output_name="train", source="/opt/ml/processing/train"),
ProcessingOutput(output_name="validation", source="/opt/ml/processing/validation"),
ProcessingOutput(output_name="test", source="/opt/ml/processing/test"),
],
code=os.path.join(BASE_DIR, "preprocess.py"),
job_arguments=[
"--input-data",
data_wrangler_step.properties.ProcessingOutputConfig.Outputs[
output_name
].S3Output.S3Uri,
],
)
# Training step for generating model artifacts
model_path = f"s3://{sagemaker_session.default_bucket()}/{base_job_prefix}/restateTrain"
model_bucket_key = f"{sagemaker_session.default_bucket()}/{base_job_prefix}/restateTrain"
cache_config = CacheConfig(enable_caching=True, expire_after="30d")
xgb_image_uri = sagemaker.image_uris.retrieve(
framework="xgboost", # we are using the Sagemaker built in xgboost algorithm
region=region,
version="1.0-1",
py_version="py3",
instance_type=training_instance_type,
)
xgb_train = Estimator(
image_uri=xgb_image_uri,
instance_type=training_instance_type,
instance_count=1,
output_path=model_path,
base_job_name=f"{base_job_prefix}/restate-xgb-train",
sagemaker_session=sagemaker_session,
role=role,
)
xgb_train.set_hyperparameters(
# #objective="binary:logistic",
# objective="reg:linear",
num_round=50,
# max_depth=5,
# eta=0.2,
# gamma=4,
# min_child_weight=6,
# subsample=0.7,
# silent=0,
)
xgb_train.set_hyperparameters(grow_policy="lossguide")
xgb_objective_metric_name = "validation:mse"
xgb_hyperparameter_ranges = {
"max_depth": IntegerParameter(2, 10, scaling_type="Linear"),
}
xgb_tuner_log = HyperparameterTuner(
xgb_train,
xgb_objective_metric_name,
xgb_hyperparameter_ranges,
max_jobs=3,
max_parallel_jobs=3,
strategy="Random",
objective_type="Minimize",
)
xgb_step_tuning = TuningStep(
name="XGBHPTune",
tuner=xgb_tuner_log,
inputs={
"train": TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.Outputs[
"train"
].S3Output.S3Uri,
content_type="text/csv",
),
"validation": TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.Outputs[
"validation"
].S3Output.S3Uri,
content_type="text/csv",
),
},
cache_config=cache_config,
)
# dtree_image_uri = '625467769535.dkr.ecr.ap-southeast-1.amazonaws.com/sagemaker-decision-tree:latest'
dtree_image_uri = sagemaker_session.sagemaker_client.describe_image_version(
ImageName="restate-dtree"
)["ContainerImage"]
dtree_train = Estimator(
image_uri=dtree_image_uri,
role=role,
instance_count=1,
instance_type=training_instance_type,
base_job_name=f"{base_job_prefix}/restate-dtree-train",
output_path=model_path,
sagemaker_session=sagemaker_session,
)
dtree_objective_metric_name = "validation:mse"
dtree_metric_definitions = [{"Name": "validation:mse", "Regex": "mse:(\S+)"}]
dtree_hyperparameter_ranges = {
"max_depth": IntegerParameter(10, 50, scaling_type="Linear"),
"max_leaf_nodes": IntegerParameter(2, 12, scaling_type="Linear"),
}
dtree_tuner_log = HyperparameterTuner(
dtree_train,
dtree_objective_metric_name,
dtree_hyperparameter_ranges,
dtree_metric_definitions,
max_jobs=3,
max_parallel_jobs=3,
strategy="Random",
objective_type="Minimize",
)
dtree_step_tuning = TuningStep(
name="DTreeHPTune",
tuner=dtree_tuner_log,
inputs={
"training": TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.Outputs[
"train"
].S3Output.S3Uri,
content_type="text/csv",
),
"validation": TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.Outputs[
"validation"
].S3Output.S3Uri,
content_type="text/csv",
),
},
cache_config=cache_config,
)
dtree_script_eval = ScriptProcessor(
image_uri=dtree_image_uri,
command=["python3"],
instance_type=processing_instance_type,
instance_count=1,
base_job_name=f"{base_job_prefix}/script-dtree-eval",
sagemaker_session=sagemaker_session,
role=role,
)
dtree_evaluation_report = PropertyFile(
name="EvaluationReportDTree",
output_name="dtree_evaluation",
path="dtree_evaluation.json",
)
dtree_step_eval = ProcessingStep(
name="DTreeEval",
processor=dtree_script_eval,
inputs=[
ProcessingInput(
# source=dtree_step_train.properties.ModelArtifacts.S3ModelArtifacts,
source=dtree_step_tuning.get_top_model_s3_uri(top_k=0, s3_bucket=model_bucket_key),
destination="/opt/ml/processing/model",
),
ProcessingInput(
source=step_process.properties.ProcessingOutputConfig.Outputs[
"test"
].S3Output.S3Uri,
destination="/opt/ml/processing/test",
),
],
outputs=[
ProcessingOutput(
output_name="dtree_evaluation", source="/opt/ml/processing/evaluation"
),
],
code=os.path.join(BASE_DIR, "dtree_evaluate.py"),
property_files=[dtree_evaluation_report],
)
xgb_script_eval = ScriptProcessor(
image_uri=xgb_image_uri,
command=["python3"],
instance_type=processing_instance_type,
instance_count=1,
base_job_name=f"{base_job_prefix}/script-xgb-eval",
sagemaker_session=sagemaker_session,
role=role,
)
xgb_evaluation_report = PropertyFile(
name="EvaluationReportXGBoost",
output_name="xgb_evaluation",
path="xgb_evaluation.json",
)
xgb_step_eval = ProcessingStep(
name="XGBEval",
processor=xgb_script_eval,
inputs=[
ProcessingInput(
source=xgb_step_tuning.get_top_model_s3_uri(top_k=0, s3_bucket=model_bucket_key),
destination="/opt/ml/processing/model",
),
ProcessingInput(
source=step_process.properties.ProcessingOutputConfig.Outputs[
"test"
].S3Output.S3Uri,
destination="/opt/ml/processing/test",
),
],
outputs=[
ProcessingOutput(output_name="xgb_evaluation", source="/opt/ml/processing/evaluation"),
],
code=os.path.join(BASE_DIR, "xgb_evaluate.py"),
property_files=[xgb_evaluation_report],
)
xgb_model_metrics = ModelMetrics(
model_statistics=MetricsSource(
s3_uri="{}/xgb_evaluation.json".format(
xgb_step_eval.arguments["ProcessingOutputConfig"]["Outputs"][0]["S3Output"]["S3Uri"]
),
content_type="application/json",
)
)
dtree_model_metrics = ModelMetrics(
model_statistics=MetricsSource(
s3_uri="{}/dtree_evaluation.json".format(
dtree_step_eval.arguments["ProcessingOutputConfig"]["Outputs"][0]["S3Output"][
"S3Uri"
]
),
content_type="application/json",
)
)
xgb_eval_metrics = JsonGet(
step=xgb_step_eval,
property_file=xgb_evaluation_report,
json_path="regression_metrics.r2s.value", # This should follow the structure of your report_dict defined in the evaluate.py file.
)
dtree_eval_metrics = JsonGet(
step=dtree_step_eval,
property_file=dtree_evaluation_report,
json_path="regression_metrics.r2s.value", # This should follow the structure of your report_dict defined in the evaluate.py file.
)
# Register model step that will be conditionally executed
dtree_step_register = RegisterModel(
name="DTreeReg",
estimator=dtree_train,
model_data=dtree_step_tuning.get_top_model_s3_uri(top_k=0, s3_bucket=model_bucket_key),
content_types=["text/csv"],
response_types=["text/csv"],
inference_instances=["ml.t2.medium", "ml.m5.large"],
transform_instances=["ml.m5.large"],
model_package_group_name=model_package_group_name,
approval_status=model_approval_status,
model_metrics=dtree_model_metrics,
)
# Register model step that will be conditionally executed
xgb_step_register = RegisterModel(
name="XGBReg",
estimator=xgb_train,
model_data=xgb_step_tuning.get_top_model_s3_uri(top_k=0, s3_bucket=model_bucket_key),
content_types=["text/csv"],
response_types=["text/csv"],
inference_instances=["ml.t2.medium", "ml.m5.large"],
transform_instances=["ml.m5.large"],
model_package_group_name=model_package_group_name,
approval_status=model_approval_status,
model_metrics=xgb_model_metrics,
)
# Condition step for evaluating model quality and branching execution
cond_lte = ConditionGreaterThanOrEqualTo( # You can change the condition here
left=JsonGet(
step=dtree_step_eval,
property_file=dtree_evaluation_report,
json_path="regression_metrics.r2s.value", # This should follow the structure of your report_dict defined in the evaluate.py file.
),
right=JsonGet(
step=xgb_step_eval,
property_file=xgb_evaluation_report,
json_path="regression_metrics.r2s.value", # This should follow the structure of your report_dict defined in the evaluate.py file.
), # You can change the threshold here
)
step_cond = ConditionStep(
name="AccuracyCond",
conditions=[cond_lte],
if_steps=[dtree_step_register],
else_steps=[xgb_step_register],
)
create_date = time.strftime("%Y-%m-%d-%H-%M-%S")
# Pipeline instance
pipeline = Pipeline(
name=pipeline_name,
parameters=[
processing_instance_type,
processing_instance_count,
training_instance_type,
model_approval_status,
input_data
],
pipeline_experiment_config=PipelineExperimentConfig(
pipeline_name + "-" + create_date, "restate-{}".format(create_date)
),
steps=[
data_wrangler_step,
step_process,
dtree_step_tuning,
xgb_step_tuning,
dtree_step_eval,
xgb_step_eval,
step_cond,
],
sagemaker_session=sagemaker_session,
)
return pipeline
# set an output path where the trained model will be saved
bucket = sagemaker.Session().default_bucket()
prefix = 'DEMO-xgboost-as-a-built-in-algo'
output_path = 's3://{}/{}/{}/output'.format(bucket, prefix,
'stock-xgb-built-in-algo')
# this line automatically looks for the XGBoost image URI and builds an XGBoost container.
# specify the repo_version depending on your preference.
xgboost_container = sagemaker.image_uris.retrieve("xgboost", region, "1.2-1")
# construct a SageMaker estimator that calls the xgboost-container
estimator = sagemaker.estimator.Estimator(
image_uri=xgboost_container,
hyperparameters=hyperparameters,
role=sagemaker.get_execution_role(),
instance_count=1,
instance_type='ml.m5.2xlarge',
volume_size=5, # 5 GB
output_path=output_path)
# define the data type and paths to the training and validation datasets
content_type = "csv"
train_input = TrainingInput("s3://{}/{}/{}/".format(bucket, prefix, 'train'),
content_type=content_type)
validation_input = TrainingInput("s3://{}/{}/{}/".format(
bucket, prefix, 'validation'),
content_type=content_type)
# execute the XGBoost training job
estimator.fit({'train': train_input, 'validation': validation_input})
def test_single_algo_tuning_step(sagemaker_session):
data_source_uri_parameter = ParameterString(
name="DataSourceS3Uri", default_value=f"s3://{BUCKET}/train_manifest")
estimator = Estimator(
image_uri=IMAGE_URI,
role=ROLE,
instance_count=1,
instance_type="ml.c5.4xlarge",
profiler_config=ProfilerConfig(system_monitor_interval_millis=500),
rules=[],
sagemaker_session=sagemaker_session,
)
estimator.set_hyperparameters(
num_layers=18,
image_shape="3,224,224",
num_classes=257,
num_training_samples=15420,
mini_batch_size=128,
epochs=10,
optimizer="sgd",
top_k="2",
precision_dtype="float32",
augmentation_type="crop",
)
hyperparameter_ranges = {
"learning_rate": ContinuousParameter(0.0001, 0.05),
"momentum": ContinuousParameter(0.0, 0.99),
"weight_decay": ContinuousParameter(0.0, 0.99),
}
tuner = HyperparameterTuner(
estimator=estimator,
objective_metric_name="val:accuracy",
hyperparameter_ranges=hyperparameter_ranges,
objective_type="Maximize",
max_jobs=5,
max_parallel_jobs=2,
early_stopping_type="OFF",
strategy="Bayesian",
warm_start_config=WarmStartConfig(
warm_start_type=WarmStartTypes.IDENTICAL_DATA_AND_ALGORITHM,
parents=set(["parent-hpo"]),
),
)
inputs = TrainingInput(s3_data=data_source_uri_parameter)
tuning_step = TuningStep(
name="MyTuningStep",
tuner=tuner,
inputs=inputs,
)
assert tuning_step.to_request() == {
"Name": "MyTuningStep",
"Type": "Tuning",
"Arguments": {
"HyperParameterTuningJobConfig": {
"Strategy": "Bayesian",
"ResourceLimits": {
"MaxNumberOfTrainingJobs": 5,
"MaxParallelTrainingJobs": 2
},
"TrainingJobEarlyStoppingType": "OFF",
"HyperParameterTuningJobObjective": {
"Type": "Maximize",
"MetricName": "val:accuracy",
},
"ParameterRanges": {
"ContinuousParameterRanges": [
{
"Name": "learning_rate",
"MinValue": "0.0001",
"MaxValue": "0.05",
"ScalingType": "Auto",
},
{
"Name": "momentum",
"MinValue": "0.0",
"MaxValue": "0.99",
"ScalingType": "Auto",
},
{
"Name": "weight_decay",
"MinValue": "0.0",
"MaxValue": "0.99",
"ScalingType": "Auto",
},
],
"CategoricalParameterRanges": [],
"IntegerParameterRanges": [],
},
},
"TrainingJobDefinition": {
"StaticHyperParameters": {
"num_layers": "18",
"image_shape": "3,224,224",
"num_classes": "257",
"num_training_samples": "15420",
"mini_batch_size": "128",
"epochs": "10",
"optimizer": "sgd",
"top_k": "2",
"precision_dtype": "float32",
"augmentation_type": "crop",
},
"RoleArn":
"DummyRole",
"OutputDataConfig": {
"S3OutputPath": "s3://my-bucket/"
},
"ResourceConfig": {
"InstanceCount": 1,
"InstanceType": "ml.c5.4xlarge",
"VolumeSizeInGB": 30,
},
"StoppingCondition": {
"MaxRuntimeInSeconds": 86400
},
"AlgorithmSpecification": {
"TrainingInputMode": "File",
"TrainingImage": "fakeimage",
},
"InputDataConfig": [{
"DataSource": {
"S3DataSource": {
"S3DataType": "S3Prefix",
"S3Uri": data_source_uri_parameter,
"S3DataDistributionType": "FullyReplicated",
}
},
"ChannelName": "training",
}],
},
"WarmStartConfig": {
"WarmStartType":
"IdenticalDataAndAlgorithm",
"ParentHyperParameterTuningJobs": [{
"HyperParameterTuningJobName":
"parent-hpo",
}],
},
},
}
assert tuning_step.properties.HyperParameterTuningJobName.expr == {
"Get": "Steps.MyTuningStep.HyperParameterTuningJobName"
}
assert tuning_step.properties.TrainingJobSummaries[
0].TrainingJobName.expr == {
"Get": "Steps.MyTuningStep.TrainingJobSummaries[0].TrainingJobName"
}
assert tuning_step.get_top_model_s3_uri(
0, "my-bucket", "my-prefix"
).expr == {
"Std:Join": {
"On":
"/",
"Values": [
"s3:/",
"my-bucket",
"my-prefix",
{
"Get":
"Steps.MyTuningStep.TrainingJobSummaries[0].TrainingJobName"
},
"output/model.tar.gz",
],
}
}
def __init__(
self,
name: str,
sagemaker_session,
role,
model_data: str,
entry_point: str,
display_name: str = None,
description: str = None,
source_dir: str = None,
dependencies: List = None,
depends_on: Optional[List[Union[str, Step, "StepCollection"]]] = None,
retry_policies: List[RetryPolicy] = None,
subnets=None,
security_group_ids=None,
**kwargs,
):
"""Base class initializer.
Args:
name (str): The name of the training step.
sagemaker_session (sagemaker.session.Session): Session object which manages
interactions with Amazon SageMaker APIs and any other AWS services needed. If
not specified, the estimator creates one using the default
AWS configuration chain.
role (str): An AWS IAM role (either name or full ARN). The Amazon
SageMaker training jobs and APIs that create Amazon SageMaker
endpoints use this role to access training data and model
artifacts. After the endpoint is created, the inference code
might use the IAM role, if it needs to access an AWS resource.
model_data (str): The S3 location of a SageMaker model data `.tar.gz` file.
entry_point (str): Path (absolute or relative) to the local Python
source file which should be executed as the entry point to
inference. If ``source_dir`` is specified, then ``entry_point``
must point to a file located at the root of ``source_dir``.
If 'git_config' is provided, 'entry_point' should be
a relative location to the Python source file in the Git repo.
Example:
With the following GitHub repo directory structure:
>>> |----- README.md
>>> |----- src
>>> |----- train.py
>>> |----- test.py
You can assign entry_point='src/train.py'.
display_name (str): The display name of this `_RepackModelStep` step (default: None).
description (str): The description of this `_RepackModelStep` (default: None).
source_dir (str): A relative location to a directory with other training
or model hosting source code dependencies aside from the entry point
file in the Git repo (default: None). Structure within this
directory are preserved when training on Amazon SageMaker.
dependencies (list[str]): A list of paths to directories (absolute
or relative) with any additional libraries that will be exported
to the container (default: []). The library folders will be
copied to SageMaker in the same folder where the entrypoint is
copied. If 'git_config' is provided, 'dependencies' should be a
list of relative locations to directories with any additional
libraries needed in the Git repo.
.. admonition:: Example
The following call
>>> Estimator(entry_point='train.py',
... dependencies=['my/libs/common', 'virtual-env'])
results in the following inside the container:
>>> $ ls
>>> opt/ml/code
>>> |------ train.py
>>> |------ common
>>> |------ virtual-env
This is not supported with "local code" in Local Mode.
depends_on (List[Union[str, Step, StepCollection]]): The list of `Step`/`StepCollection`
names or `Step` instances or `StepCollection` instances that the current `Step`
depends on (default: None).
retry_policies (List[RetryPolicy]): The list of retry policies for the current step
(default: None).
subnets (list[str]): List of subnet ids. If not specified, the re-packing
job will be created without VPC config (default: None).
security_group_ids (list[str]): List of security group ids. If not
specified, the re-packing job will be created without VPC config (default: None).
**kwargs: additional arguments for the repacking job.
"""
self._model_data = model_data
self.sagemaker_session = sagemaker_session
self.role = role
self._entry_point = entry_point
self._entry_point_basename = os.path.basename(self._entry_point)
self._source_dir = source_dir
self._dependencies = dependencies
# convert dependencies array into space-delimited string
dependencies_hyperparameter = None
if self._dependencies:
dependencies_hyperparameter = " ".join(self._dependencies)
# the real estimator and inputs
repacker = SKLearn(
framework_version=FRAMEWORK_VERSION,
instance_type=INSTANCE_TYPE,
entry_point=REPACK_SCRIPT,
source_dir=self._source_dir,
dependencies=self._dependencies,
sagemaker_session=self.sagemaker_session,
role=self.role,
hyperparameters={
"inference_script": self._entry_point_basename,
"model_archive": self._model_data,
"dependencies": dependencies_hyperparameter,
"source_dir": self._source_dir,
},
subnets=subnets,
security_group_ids=security_group_ids,
**kwargs,
)
repacker.disable_profiler = True
inputs = TrainingInput(self._model_data)
# super!
super(_RepackModelStep, self).__init__(
name=name,
display_name=display_name,
description=description,
depends_on=depends_on,
retry_policies=retry_policies,
estimator=repacker,
inputs=inputs,
)
def test_tuning_step_with_multi_algo_tuner(pipeline_session, entry_point):
pytorch_estimator = PyTorch(
entry_point=entry_point,
role=sagemaker.get_execution_role(),
framework_version="1.5.0",
py_version="py3",
instance_count=1,
instance_type="ml.m5.xlarge",
sagemaker_session=pipeline_session,
enable_sagemaker_metrics=True,
max_retry_attempts=3,
hyperparameters={
"static-hp": "hp1",
"train_size": "1280"
},
)
tuner = HyperparameterTuner.create(
estimator_dict={
"estimator-1": pytorch_estimator,
"estimator-2": pytorch_estimator,
},
objective_metric_name_dict={
"estimator-1": "test:acc",
"estimator-2": "test:acc",
},
hyperparameter_ranges_dict={
"estimator-1": {
"batch-size": IntegerParameter(64, 128)
},
"estimator-2": {
"batch-size": IntegerParameter(256, 512)
},
},
metric_definitions_dict={
"estimator-1": [{
"Name": "test:acc",
"Regex": "Overall test accuracy: (.*?);"
}],
"estimator-2": [{
"Name": "test:acc",
"Regex": "Overall test accuracy: (.*?);"
}],
},
)
input_path = f"s3://{pipeline_session.default_bucket()}/training-data"
inputs = {
"estimator-1": TrainingInput(s3_data=input_path),
"estimator-2": TrainingInput(s3_data=input_path),
}
step_args = tuner.fit(
inputs=inputs,
include_cls_metadata={
"estimator-1": False,
"estimator-2": False,
},
)
step = TuningStep(
name="MyTuningStep",
step_args=step_args,
)
pipeline = Pipeline(
name="MyPipeline",
steps=[step],
sagemaker_session=pipeline_session,
)
assert json.loads(pipeline.definition())["Steps"][0] == {
"Name": "MyTuningStep",
"Type": "Tuning",
"Arguments": step_args,
}
'max-iterations': int(conf.training_max_iterations),
'batch-size': int(conf.training_batch_size)
},
instance_count=conf.training_instance_count,
instance_type=conf.training_instance_type,
metric_definitions=turicreate_metrics,
input_mode='File')
# TODO: change into Pipe - but that would need additional read f-ions in training.py
step_train = TrainingStep(
name="ModelTraining",
estimator=tf_train,
inputs={
"train":
TrainingInput(step_sframe_process.properties.ProcessingOutputConfig.
Outputs["train"].S3Output.S3Uri,
input_mode="File"),
"test":
TrainingInput(step_sframe_process.properties.ProcessingOutputConfig.
Outputs["test"].S3Output.S3Uri,
input_mode="File"),
},
cache_config=conf.cache_config)
# define the evaluation step
# we have trained the model, but we need to validate it too; what happened
# the first time was, due to the EXIF tags half of images were not rotated
# properly relative to the labels... needless to say, that led to the mAP
# score to be close to nill; that's a good example for why the validation
# step is needed - only models that work should be carried on with.
script_eval = ScriptProcessor(image_uri=str(conf.processing_turicreate_uri),
def test_multi_algo_tuning_step(sagemaker_session):
data_source_uri_parameter = ParameterString(
name="DataSourceS3Uri", default_value=f"s3://{BUCKET}/train_manifest")
instance_count = ParameterInteger(name="InstanceCount", default_value=1)
estimator = Estimator(
image_uri=IMAGE_URI,
role=ROLE,
instance_count=instance_count,
instance_type="ml.c5.4xlarge",
profiler_config=ProfilerConfig(system_monitor_interval_millis=500),
rules=[],
sagemaker_session=sagemaker_session,
max_retry_attempts=10,
)
estimator.set_hyperparameters(
num_layers=18,
image_shape="3,224,224",
num_classes=257,
num_training_samples=15420,
mini_batch_size=128,
epochs=10,
optimizer="sgd",
top_k="2",
precision_dtype="float32",
augmentation_type="crop",
)
initial_lr_param = ParameterString(name="InitialLR",
default_value="0.0001")
hyperparameter_ranges = {
"learning_rate": ContinuousParameter(initial_lr_param, 0.05),
"momentum": ContinuousParameter(0.0, 0.99),
"weight_decay": ContinuousParameter(0.0, 0.99),
}
tuner = HyperparameterTuner.create(
estimator_dict={
"estimator-1": estimator,
"estimator-2": estimator,
},
objective_type="Minimize",
objective_metric_name_dict={
"estimator-1": "val:loss",
"estimator-2": "val:loss",
},
hyperparameter_ranges_dict={
"estimator-1": hyperparameter_ranges,
"estimator-2": hyperparameter_ranges,
},
)
inputs = TrainingInput(s3_data=data_source_uri_parameter)
tuning_step = TuningStep(
name="MyTuningStep",
tuner=tuner,
inputs={
"estimator-1": inputs,
"estimator-2": inputs,
},
)
assert tuning_step.to_request() == {
"Name": "MyTuningStep",
"Type": "Tuning",
"Arguments": {
"HyperParameterTuningJobConfig": {
"Strategy": "Bayesian",
"ResourceLimits": {
"MaxNumberOfTrainingJobs": 1,
"MaxParallelTrainingJobs": 1
},
"TrainingJobEarlyStoppingType": "Off",
},
"TrainingJobDefinitions": [
{
"StaticHyperParameters": {
"num_layers": "18",
"image_shape": "3,224,224",
"num_classes": "257",
"num_training_samples": "15420",
"mini_batch_size": "128",
"epochs": "10",
"optimizer": "sgd",
"top_k": "2",
"precision_dtype": "float32",
"augmentation_type": "crop",
},
"RoleArn":
"DummyRole",
"OutputDataConfig": {
"S3OutputPath": "s3://my-bucket/"
},
"ResourceConfig": {
"InstanceCount": 1,
"InstanceType": "ml.c5.4xlarge",
"VolumeSizeInGB": 30,
},
"StoppingCondition": {
"MaxRuntimeInSeconds": 86400
},
"AlgorithmSpecification": {
"TrainingInputMode": "File",
"TrainingImage": "fakeimage",
},
"InputDataConfig": [{
"DataSource": {
"S3DataSource": {
"S3DataType": "S3Prefix",
"S3Uri": data_source_uri_parameter,
"S3DataDistributionType": "FullyReplicated",
}
},
"ChannelName": "training",
}],
"DefinitionName":
"estimator-1",
"TuningObjective": {
"Type": "Minimize",
"MetricName": "val:loss"
},
"HyperParameterRanges": {
"ContinuousParameterRanges": [
{
"Name": "learning_rate",
"MinValue": initial_lr_param,
"MaxValue": "0.05",
"ScalingType": "Auto",
},
{
"Name": "momentum",
"MinValue": "0.0",
"MaxValue": "0.99",
"ScalingType": "Auto",
},
{
"Name": "weight_decay",
"MinValue": "0.0",
"MaxValue": "0.99",
"ScalingType": "Auto",
},
],
"CategoricalParameterRanges": [],
"IntegerParameterRanges": [],
},
"RetryStrategy": {
"MaximumRetryAttempts": 10,
},
},
{
"StaticHyperParameters": {
"num_layers": "18",
"image_shape": "3,224,224",
"num_classes": "257",
"num_training_samples": "15420",
"mini_batch_size": "128",
"epochs": "10",
"optimizer": "sgd",
"top_k": "2",
"precision_dtype": "float32",
"augmentation_type": "crop",
},
"RoleArn":
"DummyRole",
"OutputDataConfig": {
"S3OutputPath": "s3://my-bucket/"
},
"ResourceConfig": {
"InstanceCount": 1,
"InstanceType": "ml.c5.4xlarge",
"VolumeSizeInGB": 30,
},
"StoppingCondition": {
"MaxRuntimeInSeconds": 86400
},
"AlgorithmSpecification": {
"TrainingInputMode": "File",
"TrainingImage": "fakeimage",
},
"InputDataConfig": [{
"DataSource": {
"S3DataSource": {
"S3DataType": "S3Prefix",
"S3Uri": data_source_uri_parameter,
"S3DataDistributionType": "FullyReplicated",
}
},
"ChannelName": "training",
}],
"DefinitionName":
"estimator-2",
"TuningObjective": {
"Type": "Minimize",
"MetricName": "val:loss"
},
"HyperParameterRanges": {
"ContinuousParameterRanges": [
{
"Name": "learning_rate",
"MinValue": initial_lr_param,
"MaxValue": "0.05",
"ScalingType": "Auto",
},
{
"Name": "momentum",
"MinValue": "0.0",
"MaxValue": "0.99",
"ScalingType": "Auto",
},
{
"Name": "weight_decay",
"MinValue": "0.0",
"MaxValue": "0.99",
"ScalingType": "Auto",
},
],
"CategoricalParameterRanges": [],
"IntegerParameterRanges": [],
},
"RetryStrategy": {
"MaximumRetryAttempts": 10,
},
},
],
},
}
def get_pipeline(
region,
role=None,
default_bucket=None,
model_package_group_name="CustomerChurnPackageGroup", # Choose any name
pipeline_name="CustomerChurnDemo-p-ewf8t7lvhivm", # You can find your pipeline name in the Studio UI (project -> Pipelines -> name)
base_job_prefix="CustomerChurn", # Choose any name
):
"""Gets a SageMaker ML Pipeline instance working with on CustomerChurn data.
Args:
region: AWS region to create and run the pipeline.
role: IAM role to create and run steps and pipeline.
default_bucket: the bucket to use for storing the artifacts
Returns:
an instance of a pipeline
"""
sagemaker_session = get_session(region, default_bucket)
if role is None:
role = sagemaker.session.get_execution_role(sagemaker_session)
# Parameters for pipeline execution
processing_instance_count = ParameterInteger(
name="ProcessingInstanceCount", default_value=1)
processing_instance_type = ParameterString(name="ProcessingInstanceType",
default_value="ml.m5.xlarge")
training_instance_type = ParameterString(name="TrainingInstanceType",
default_value="ml.m5.xlarge")
model_approval_status = ParameterString(
name="ModelApprovalStatus",
default_value=
"PendingManualApproval", # ModelApprovalStatus can be set to a default of "Approved" if you don't want manual approval.
)
input_data = ParameterString(
name="InputDataUrl",
default_value=
f"s3://sm-pipelines-demo-data-123456789/churn.txt", # Change this to point to the s3 location of your raw input data.
)
# Processing step for feature engineering
sklearn_processor = SKLearnProcessor(
framework_version="0.23-1",
instance_type=processing_instance_type,
instance_count=processing_instance_count,
base_job_name=
f"{base_job_prefix}/sklearn-CustomerChurn-preprocess", # choose any name
sagemaker_session=sagemaker_session,
role=role,
)
step_process = ProcessingStep(
name="CustomerChurnProcess", # choose any name
processor=sklearn_processor,
outputs=[
ProcessingOutput(output_name="train",
source="/opt/ml/processing/train"),
ProcessingOutput(output_name="validation",
source="/opt/ml/processing/validation"),
ProcessingOutput(output_name="test",
source="/opt/ml/processing/test"),
],
code=os.path.join(BASE_DIR, "preprocess.py"),
job_arguments=["--input-data", input_data],
)
# Training step for generating model artifacts
model_path = f"s3://{sagemaker_session.default_bucket()}/{base_job_prefix}/CustomerChurnTrain"
image_uri = sagemaker.image_uris.retrieve(
framework=
"xgboost", # we are using the Sagemaker built in xgboost algorithm
region=region,
version="1.0-1",
py_version="py3",
instance_type=training_instance_type,
)
xgb_train = Estimator(
image_uri=image_uri,
instance_type=training_instance_type,
instance_count=1,
output_path=model_path,
base_job_name=f"{base_job_prefix}/CustomerChurn-train",
sagemaker_session=sagemaker_session,
role=role,
)
xgb_train.set_hyperparameters(
objective="binary:logistic",
num_round=50,
max_depth=5,
eta=0.2,
gamma=4,
min_child_weight=6,
subsample=0.7,
silent=0,
)
step_train = TrainingStep(
name="CustomerChurnTrain",
estimator=xgb_train,
inputs={
"train":
TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.
Outputs["train"].S3Output.S3Uri,
content_type="text/csv",
),
"validation":
TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.
Outputs["validation"].S3Output.S3Uri,
content_type="text/csv",
),
},
)
# Processing step for evaluation
script_eval = ScriptProcessor(
image_uri=image_uri,
command=["python3"],
instance_type=processing_instance_type,
instance_count=1,
base_job_name=f"{base_job_prefix}/script-CustomerChurn-eval",
sagemaker_session=sagemaker_session,
role=role,
)
evaluation_report = PropertyFile(
name="EvaluationReport",
output_name="evaluation",
path="evaluation.json",
)
step_eval = ProcessingStep(
name="CustomerChurnEval",
processor=script_eval,
inputs=[
ProcessingInput(
source=step_train.properties.ModelArtifacts.S3ModelArtifacts,
destination="/opt/ml/processing/model",
),
ProcessingInput(
source=step_process.properties.ProcessingOutputConfig.
Outputs["test"].S3Output.S3Uri,
destination="/opt/ml/processing/test",
),
],
outputs=[
ProcessingOutput(output_name="evaluation",
source="/opt/ml/processing/evaluation"),
],
code=os.path.join(BASE_DIR, "evaluate.py"),
property_files=[evaluation_report],
)
# Register model step that will be conditionally executed
model_metrics = ModelMetrics(model_statistics=MetricsSource(
s3_uri="{}/evaluation.json".format(
step_eval.arguments["ProcessingOutputConfig"]["Outputs"][0]
["S3Output"]["S3Uri"]),
content_type="application/json",
))
# Register model step that will be conditionally executed
step_register = RegisterModel(
name="CustomerChurnRegisterModel",
estimator=xgb_train,
model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,
content_types=["text/csv"],
response_types=["text/csv"],
inference_instances=["ml.t2.medium", "ml.m5.large"],
transform_instances=["ml.m5.large"],
model_package_group_name=model_package_group_name,
approval_status=model_approval_status,
model_metrics=model_metrics,
)
# Condition step for evaluating model quality and branching execution
cond_lte = ConditionGreaterThanOrEqualTo( # You can change the condition here
left=JsonGet(
step=step_eval,
property_file=evaluation_report,
json_path=
"binary_classification_metrics.accuracy.value", # This should follow the structure of your report_dict defined in the evaluate.py file.
),
right=0.8, # You can change the threshold here
)
step_cond = ConditionStep(
name="CustomerChurnAccuracyCond",
conditions=[cond_lte],
if_steps=[step_register],
else_steps=[],
)
# Pipeline instance
pipeline = Pipeline(
name=pipeline_name,
parameters=[
processing_instance_type,
processing_instance_count,
training_instance_type,
model_approval_status,
input_data,
],
steps=[step_process, step_train, step_eval, step_cond],
sagemaker_session=sagemaker_session,
)
return pipeline
def test_training_step_base_estimator(sagemaker_session):
instance_type_parameter = ParameterString(name="InstanceType",
default_value="c4.4xlarge")
instance_count_parameter = ParameterInteger(name="InstanceCount",
default_value=1)
data_source_uri_parameter = ParameterString(
name="DataSourceS3Uri", default_value=f"s3://{BUCKET}/train_manifest")
training_epochs_parameter = ParameterInteger(name="TrainingEpochs",
default_value=5)
training_batch_size_parameter = ParameterInteger(name="TrainingBatchSize",
default_value=500)
estimator = Estimator(
image_uri=IMAGE_URI,
role=ROLE,
instance_count=instance_count_parameter,
instance_type=instance_type_parameter,
profiler_config=ProfilerConfig(system_monitor_interval_millis=500),
hyperparameters={
"batch-size": training_batch_size_parameter,
"epochs": training_epochs_parameter,
},
rules=[],
sagemaker_session=sagemaker_session,
)
inputs = TrainingInput(s3_data=data_source_uri_parameter)
cache_config = CacheConfig(enable_caching=True, expire_after="PT1H")
step = TrainingStep(
name="MyTrainingStep",
depends_on=["TestStep"],
estimator=estimator,
inputs=inputs,
cache_config=cache_config,
)
step.add_depends_on(["AnotherTestStep"])
assert step.to_request() == {
"Name": "MyTrainingStep",
"Type": "Training",
"DependsOn": ["TestStep", "AnotherTestStep"],
"Arguments": {
"AlgorithmSpecification": {
"TrainingImage": IMAGE_URI,
"TrainingInputMode": "File"
},
"HyperParameters": {
"batch-size": training_batch_size_parameter,
"epochs": training_epochs_parameter,
},
"InputDataConfig": [{
"ChannelName": "training",
"DataSource": {
"S3DataSource": {
"S3DataDistributionType": "FullyReplicated",
"S3DataType": "S3Prefix",
"S3Uri": data_source_uri_parameter,
}
},
}],
"OutputDataConfig": {
"S3OutputPath": f"s3://{BUCKET}/"
},
"ResourceConfig": {
"InstanceCount": instance_count_parameter,
"InstanceType": instance_type_parameter,
"VolumeSizeInGB": 30,
},
"RoleArn":
ROLE,
"StoppingCondition": {
"MaxRuntimeInSeconds": 86400
},
"ProfilerConfig": {
"ProfilingIntervalInMilliseconds": 500,
"S3OutputPath": f"s3://{BUCKET}/",
},
},
"CacheConfig": {
"Enabled": True,
"ExpireAfter": "PT1H"
},
}
assert step.properties.TrainingJobName.expr == {
"Get": "Steps.MyTrainingStep.TrainingJobName"
}
def records_s3_input(self):
"""Return a TrainingInput to represent the training data"""
return TrainingInput(
self.s3_data, distribution="ShardedByS3Key", s3_data_type=self.s3_data_type
)
def test_training_step_tensorflow(sagemaker_session):
instance_type_parameter = ParameterString(name="InstanceType",
default_value="ml.p3.16xlarge")
instance_count_parameter = ParameterInteger(name="InstanceCount",
default_value=1)
data_source_uri_parameter = ParameterString(
name="DataSourceS3Uri", default_value=f"s3://{BUCKET}/train_manifest")
training_epochs_parameter = ParameterInteger(name="TrainingEpochs",
default_value=5)
training_batch_size_parameter = ParameterInteger(name="TrainingBatchSize",
default_value=500)
estimator = TensorFlow(
entry_point=os.path.join(DATA_DIR, SCRIPT_FILE),
role=ROLE,
model_dir=False,
image_uri=IMAGE_URI,
source_dir="s3://mybucket/source",
framework_version="2.4.1",
py_version="py37",
instance_count=instance_count_parameter,
instance_type=instance_type_parameter,
sagemaker_session=sagemaker_session,
# subnets=subnets,
hyperparameters={
"batch-size": training_batch_size_parameter,
"epochs": training_epochs_parameter,
},
# security_group_ids=security_group_ids,
debugger_hook_config=False,
# Training using SMDataParallel Distributed Training Framework
distribution={"smdistributed": {
"dataparallel": {
"enabled": True
}
}},
)
inputs = TrainingInput(s3_data=data_source_uri_parameter)
cache_config = CacheConfig(enable_caching=True, expire_after="PT1H")
step = TrainingStep(name="MyTrainingStep",
estimator=estimator,
inputs=inputs,
cache_config=cache_config)
step_request = step.to_request()
step_request["Arguments"]["HyperParameters"].pop("sagemaker_job_name",
None)
step_request["Arguments"]["HyperParameters"].pop("sagemaker_program", None)
step_request["Arguments"].pop("ProfilerRuleConfigurations", None)
assert step_request == {
"Name": "MyTrainingStep",
"Type": "Training",
"Arguments": {
"AlgorithmSpecification": {
"TrainingInputMode": "File",
"TrainingImage": "fakeimage",
"EnableSageMakerMetricsTimeSeries": True,
},
"OutputDataConfig": {
"S3OutputPath": "s3://my-bucket/"
},
"StoppingCondition": {
"MaxRuntimeInSeconds": 86400
},
"ResourceConfig": {
"InstanceCount": instance_count_parameter,
"InstanceType": instance_type_parameter,
"VolumeSizeInGB": 30,
},
"RoleArn":
"DummyRole",
"InputDataConfig": [{
"DataSource": {
"S3DataSource": {
"S3DataType": "S3Prefix",
"S3Uri": data_source_uri_parameter,
"S3DataDistributionType": "FullyReplicated",
}
},
"ChannelName": "training",
}],
"HyperParameters": {
"batch-size": training_batch_size_parameter,
"epochs": training_epochs_parameter,
"sagemaker_submit_directory": '"s3://mybucket/source"',
"sagemaker_container_log_level": "20",
"sagemaker_region": '"us-west-2"',
"sagemaker_distributed_dataparallel_enabled": "true",
"sagemaker_instance_type": instance_type_parameter,
"sagemaker_distributed_dataparallel_custom_mpi_options": '""',
},
"ProfilerConfig": {
"S3OutputPath": "s3://my-bucket/"
},
},
"CacheConfig": {
"Enabled": True,
"ExpireAfter": "PT1H"
},
}
assert step.properties.TrainingJobName.expr == {
"Get": "Steps.MyTrainingStep.TrainingJobName"
}
objective="binary:logistic",
num_round=50,
max_depth=5,
eta=0.2,
gamma=4,
min_child_weight=6,
subsample=0.7,
silent=0,
)
step_train = TrainingStep(
name="CustomerChurnTrain",
estimator=xgb_train,
inputs={
"train": TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.Outputs[
"train"
].S3Output.S3Uri,
content_type="text/csv",
),
"validation": TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.Outputs[
"validation"
].S3Output.S3Uri,
content_type="text/csv",
),
},
)
return step_train
def get_pipeline(
region,
role=None,
default_bucket=None,
model_package_group_name="TestPackageGroup",
pipeline_name="TestPipeline",
base_job_prefix="Test",
):
"""Gets a SageMaker ML Pipeline instance working with on abalone data.
Args:
region: AWS region to create and run the pipeline.
role: IAM role to create and run steps and pipeline.
default_bucket: the bucket to use for storing the artifacts
Returns:
an instance of a pipeline
"""
sagemaker_session = get_session(region, default_bucket)
if role is None:
role = sagemaker.session.get_execution_role(sagemaker_session)
# parameters for pipeline execution
processing_instance_count = ParameterInteger(
name="ProcessingInstanceCount", default_value=1)
processing_instance_type = ParameterString(name="ProcessingInstanceType",
default_value="ml.m5.xlarge")
training_instance_type = ParameterString(name="TrainingInstanceType",
default_value="ml.m5.xlarge")
model_approval_status = ParameterString(
name="ModelApprovalStatus", default_value="PendingManualApproval")
input_data = ParameterString(
name="InputDataUrl",
default_value=
f"s3://sagemaker-servicecatalog-seedcode-{region}/dataset/abalone-dataset.csv",
)
# processing step for feature engineering
sklearn_processor = SKLearnProcessor(
framework_version="0.23-1",
instance_type=processing_instance_type,
instance_count=processing_instance_count,
base_job_name=f"{base_job_prefix}/sklearn-test-preprocess",
sagemaker_session=sagemaker_session,
role=role,
)
step_process = ProcessingStep(
name="PreprocessTestData",
processor=sklearn_processor,
outputs=[
ProcessingOutput(output_name="train",
source="/opt/ml/processing/train"),
ProcessingOutput(output_name="validation",
source="/opt/ml/processing/validation"),
ProcessingOutput(output_name="test",
source="/opt/ml/processing/test"),
],
code=os.path.join(BASE_DIR, "preprocess.py"),
job_arguments=["--input-data", input_data],
)
# training step for generating model artifacts
model_path = f"s3://{sagemaker_session.default_bucket()}/{base_job_prefix}/TestTrain"
image_uri = sagemaker.image_uris.retrieve(
framework="xgboost",
region=region,
version="1.0-1",
py_version="py3",
instance_type=training_instance_type,
)
xgb_train = Estimator(
image_uri=image_uri,
instance_type=training_instance_type,
instance_count=1,
output_path=model_path,
base_job_name=f"{base_job_prefix}/test-train",
sagemaker_session=sagemaker_session,
role=role,
)
xgb_train.set_hyperparameters(
objective="reg:linear",
num_round=50,
max_depth=5,
eta=0.2,
gamma=4,
min_child_weight=6,
subsample=0.7,
silent=0,
)
step_train = TrainingStep(
name="TrainTestModel",
estimator=xgb_train,
inputs={
"train":
TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.
Outputs["train"].S3Output.S3Uri,
content_type="text/csv",
),
"validation":
TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.
Outputs["validation"].S3Output.S3Uri,
content_type="text/csv",
),
},
)
# processing step for evaluation
script_eval = ScriptProcessor(
image_uri=image_uri,
command=["python3"],
instance_type=processing_instance_type,
instance_count=1,
base_job_name=f"{base_job_prefix}/script-test-eval",
sagemaker_session=sagemaker_session,
role=role,
)
evaluation_report = PropertyFile(
name="TestEvaluationReport",
output_name="evaluation",
path="evaluation.json",
)
step_eval = ProcessingStep(
name="EvaluateTestModel",
processor=script_eval,
inputs=[
ProcessingInput(
source=step_train.properties.ModelArtifacts.S3ModelArtifacts,
destination="/opt/ml/processing/model",
),
ProcessingInput(
source=step_process.properties.ProcessingOutputConfig.
Outputs["test"].S3Output.S3Uri,
destination="/opt/ml/processing/test",
),
],
outputs=[
ProcessingOutput(output_name="evaluation",
source="/opt/ml/processing/evaluation"),
],
code=os.path.join(BASE_DIR, "evaluate.py"),
property_files=[evaluation_report],
)
# register model step that will be conditionally executed
model_metrics = ModelMetrics(
model_statistics=MetricsSource(s3_uri="{}/evaluation.json".format(
step_eval.arguments["ProcessingOutputConfig"]["Outputs"][0]
["S3Output"]["S3Uri"]),
content_type="application/json"))
step_register = RegisterModel(
name="RegisterTestModel",
estimator=xgb_train,
model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,
content_types=["text/csv"],
response_types=["text/csv"],
inference_instances=["ml.t2.medium", "ml.m5.large"],
transform_instances=["ml.m5.large"],
model_package_group_name=model_package_group_name,
approval_status=model_approval_status,
model_metrics=model_metrics,
)
# condition step for evaluating model quality and branching execution
cond_lte = ConditionLessThanOrEqualTo(
left=JsonGet(step=step_eval,
property_file=evaluation_report,
json_path="regression_metrics.mse.value"),
right=6.0,
)
step_cond = ConditionStep(
name="CheckMSETestEvaluation",
conditions=[cond_lte],
if_steps=[step_register],
else_steps=[],
)
# pipeline instance
pipeline = Pipeline(
name=pipeline_name,
parameters=[
processing_instance_type,
processing_instance_count,
training_instance_type,
model_approval_status,
input_data,
],
steps=[step_process, step_train, step_eval, step_cond],
sagemaker_session=sagemaker_session,
)
return pipeline
def get_pipeline(region, role, default_bucket, pipeline_name,
model_package_group_name, base_job_prefix):
"""Gets a SageMaker ML Pipeline instance working with BERT.
Args:
region: AWS region to create and run the pipeline.
role: IAM role to create and run steps and pipeline.
default_bucket: the bucket to use for storing the artifacts
pipeline_name: name of this pipeline
model_package_group_name: model package group
base_job_prefix: prefic of the job name
Returns:
an instance of a pipeline
"""
sm = boto3.Session().client(service_name="sagemaker", region_name=region)
input_data = ParameterString(
name="InputDataUrl",
default_value="s3://{}/amazon-reviews-pds/tsv/".format(bucket),
)
processing_instance_count = ParameterInteger(
name="ProcessingInstanceCount", default_value=1)
processing_instance_type = ParameterString(name="ProcessingInstanceType",
default_value="ml.c5.2xlarge")
max_seq_length = ParameterInteger(
name="MaxSeqLength",
default_value=64,
)
balance_dataset = ParameterString(
name="BalanceDataset",
default_value="True",
)
train_split_percentage = ParameterFloat(
name="TrainSplitPercentage",
default_value=0.90,
)
validation_split_percentage = ParameterFloat(
name="ValidationSplitPercentage",
default_value=0.05,
)
test_split_percentage = ParameterFloat(
name="TestSplitPercentage",
default_value=0.05,
)
feature_store_offline_prefix = ParameterString(
name="FeatureStoreOfflinePrefix",
default_value="reviews-feature-store-" + str(timestamp),
)
feature_group_name = ParameterString(
name="FeatureGroupName",
default_value="reviews-feature-group-" + str(timestamp))
train_instance_type = ParameterString(name="TrainInstanceType",
default_value="ml.c5.9xlarge")
train_instance_count = ParameterInteger(name="TrainInstanceCount",
default_value=1)
#########################
# PROCESSING STEP
#########################
processor = SKLearnProcessor(
framework_version="0.23-1",
role=role,
instance_type=processing_instance_type,
instance_count=processing_instance_count,
env={"AWS_DEFAULT_REGION": region},
max_runtime_in_seconds=7200,
)
processing_inputs = [
ProcessingInput(
input_name="raw-input-data",
source=input_data,
destination="/opt/ml/processing/input/data/",
s3_data_distribution_type="ShardedByS3Key",
)
]
processing_outputs = [
ProcessingOutput(
output_name="bert-train",
s3_upload_mode="EndOfJob",
source="/opt/ml/processing/output/bert/train",
),
ProcessingOutput(
output_name="bert-validation",
s3_upload_mode="EndOfJob",
source="/opt/ml/processing/output/bert/validation",
),
ProcessingOutput(
output_name="bert-test",
s3_upload_mode="EndOfJob",
source="/opt/ml/processing/output/bert/test",
),
]
# TODO: Figure out why the Parameter's are not resolving properly to their native type when user here.
# We shouldn't be using `default_value`
processing_step = ProcessingStep(
name="Processing",
processor=processor,
inputs=processing_inputs,
outputs=processing_outputs,
job_arguments=[
"--train-split-percentage",
str(train_split_percentage.default_value),
"--validation-split-percentage",
str(validation_split_percentage.default_value),
"--test-split-percentage",
str(test_split_percentage.default_value),
"--max-seq-length",
str(max_seq_length.default_value),
"--balance-dataset",
str(balance_dataset.default_value),
"--feature-store-offline-prefix",
str(feature_store_offline_prefix.default_value),
"--feature-group-name",
str(feature_group_name.default_value),
],
code=os.path.join(BASE_DIR,
"preprocess-scikit-text-to-bert-feature-store.py"),
)
#########################
# TRAINING STEP
#########################
epochs = ParameterInteger(name="Epochs", default_value=1)
learning_rate = ParameterFloat(name="LearningRate", default_value=0.00001)
epsilon = ParameterFloat(name="Epsilon", default_value=0.00000001)
train_batch_size = ParameterInteger(name="TrainBatchSize",
default_value=128)
validation_batch_size = ParameterInteger(name="ValidationBatchSize",
default_value=128)
test_batch_size = ParameterInteger(name="TestBatchSize", default_value=128)
train_steps_per_epoch = ParameterInteger(name="TrainStepsPerEpoch",
default_value=50)
validation_steps = ParameterInteger(name="ValidationSteps",
default_value=50)
test_steps = ParameterInteger(name="TestSteps", default_value=50)
train_volume_size = ParameterInteger(name="TrainVolumeSize",
default_value=1024)
use_xla = ParameterString(
name="UseXLA",
default_value="True",
)
use_amp = ParameterString(
name="UseAMP",
default_value="True",
)
freeze_bert_layer = ParameterString(
name="FreezeBERTLayer",
default_value="False",
)
enable_sagemaker_debugger = ParameterString(
name="EnableSageMakerDebugger",
default_value="False",
)
enable_checkpointing = ParameterString(
name="EnableCheckpointing",
default_value="False",
)
enable_tensorboard = ParameterString(
name="EnableTensorboard",
default_value="False",
)
input_mode = ParameterString(
name="InputMode",
default_value="File",
)
run_validation = ParameterString(
name="RunValidation",
default_value="True",
)
run_test = ParameterString(
name="RunTest",
default_value="False",
)
run_sample_predictions = ParameterString(
name="RunSamplePredictions",
default_value="False",
)
metrics_definitions = [
{
"Name": "train:loss",
"Regex": "loss: ([0-9\\.]+)"
},
{
"Name": "train:accuracy",
"Regex": "accuracy: ([0-9\\.]+)"
},
{
"Name": "validation:loss",
"Regex": "val_loss: ([0-9\\.]+)"
},
{
"Name": "validation:accuracy",
"Regex": "val_accuracy: ([0-9\\.]+)"
},
]
train_src = os.path.join(BASE_DIR, "src")
model_path = f"s3://{default_bucket}/{base_job_prefix}/output/model"
estimator = TensorFlow(
entry_point="tf_bert_reviews.py",
source_dir=BASE_DIR,
role=role,
output_path=model_path,
instance_count=train_instance_count,
instance_type=train_instance_type,
volume_size=train_volume_size,
py_version="py37",
framework_version="2.3.1",
hyperparameters={
"epochs": epochs,
"learning_rate": learning_rate,
"epsilon": epsilon,
"train_batch_size": train_batch_size,
"validation_batch_size": validation_batch_size,
"test_batch_size": test_batch_size,
"train_steps_per_epoch": train_steps_per_epoch,
"validation_steps": validation_steps,
"test_steps": test_steps,
"use_xla": use_xla,
"use_amp": use_amp,
"max_seq_length": max_seq_length,
"freeze_bert_layer": freeze_bert_layer,
"enable_sagemaker_debugger": enable_sagemaker_debugger,
"enable_checkpointing": enable_checkpointing,
"enable_tensorboard": enable_tensorboard,
"run_validation": run_validation,
"run_test": run_test,
"run_sample_predictions": run_sample_predictions,
},
input_mode=input_mode,
metric_definitions=metrics_definitions,
# max_run=7200 # max 2 hours * 60 minutes seconds per hour * 60 seconds per minute
)
training_step = TrainingStep(
name="Train",
estimator=estimator,
inputs={
"train":
TrainingInput(
s3_data=processing_step.properties.ProcessingOutputConfig.
Outputs["bert-train"].S3Output.S3Uri,
content_type="text/csv",
),
"validation":
TrainingInput(
s3_data=processing_step.properties.ProcessingOutputConfig.
Outputs["bert-validation"].S3Output.S3Uri,
content_type="text/csv",
),
"test":
TrainingInput(
s3_data=processing_step.properties.ProcessingOutputConfig.
Outputs["bert-test"].S3Output.S3Uri,
content_type="text/csv",
),
},
)
#########################
# EVALUATION STEP
#########################
evaluation_processor = SKLearnProcessor(
framework_version="0.23-1",
role=role,
instance_type=processing_instance_type,
instance_count=processing_instance_count,
env={"AWS_DEFAULT_REGION": region},
max_runtime_in_seconds=7200,
)
evaluation_report = PropertyFile(name="EvaluationReport",
output_name="metrics",
path="evaluation.json")
evaluation_step = ProcessingStep(
name="EvaluateModel",
processor=evaluation_processor,
code=os.path.join(BASE_DIR, "evaluate_model_metrics.py"),
inputs=[
ProcessingInput(
source=training_step.properties.ModelArtifacts.
S3ModelArtifacts,
destination="/opt/ml/processing/input/model",
),
ProcessingInput(
source=processing_step.properties.
ProcessingInputs["raw-input-data"].S3Input.S3Uri,
destination="/opt/ml/processing/input/data",
),
],
outputs=[
ProcessingOutput(output_name="metrics",
s3_upload_mode="EndOfJob",
source="/opt/ml/processing/output/metrics/"),
],
job_arguments=[
"--max-seq-length",
str(max_seq_length.default_value),
],
property_files=[evaluation_report
], # these cause deserialization issues
)
model_metrics = ModelMetrics(model_statistics=MetricsSource(
s3_uri="{}/evaluation.json".format(
evaluation_step.arguments["ProcessingOutputConfig"]["Outputs"][0]
["S3Output"]["S3Uri"]),
content_type="application/json",
))
#########################
## REGISTER TRAINED MODEL STEP
#########################
model_approval_status = ParameterString(
name="ModelApprovalStatus", default_value="PendingManualApproval")
deploy_instance_type = ParameterString(name="DeployInstanceType",
default_value="ml.m5.4xlarge")
deploy_instance_count = ParameterInteger(name="DeployInstanceCount",
default_value=1)
inference_image_uri = sagemaker.image_uris.retrieve(
framework="tensorflow",
region=region,
version="2.3.1",
py_version="py37",
instance_type=deploy_instance_type,
image_scope="inference",
)
print(inference_image_uri)
register_step = RegisterModel(
name="RegisterModel",
estimator=estimator,
image_uri=
inference_image_uri, # we have to specify, by default it's using training image
model_data=training_step.properties.ModelArtifacts.S3ModelArtifacts,
content_types=["text/csv"],
response_types=["text/csv"],
inference_instances=[
deploy_instance_type
], # The JSON spec must be within these instance types or we will see "Instance Type Not Allowed" Exception
transform_instances=[deploy_instance_type],
model_package_group_name=model_package_group_name,
approval_status=model_approval_status,
)
#########################
## CREATE MODEL FOR DEPLOYMENT STEP
#########################
model = Model(
image_uri=inference_image_uri,
model_data=training_step.properties.ModelArtifacts.S3ModelArtifacts,
sagemaker_session=sess,
role=role,
)
create_inputs = CreateModelInput(instance_type=deploy_instance_type, )
create_step = CreateModelStep(
name="CreateModel",
model=model,
inputs=create_inputs,
)
#########################
## CONDITION STEP: EVALUATE THE MODEL
#########################
min_accuracy_value = ParameterFloat(name="MinAccuracyValue",
default_value=0.01)
minimum_accuracy_condition = ConditionGreaterThanOrEqualTo(
left=JsonGet(
step=evaluation_step,
property_file=evaluation_report,
json_path="metrics.accuracy.value",
),
right=min_accuracy_value, # accuracy
)
minimum_accuracy_condition_step = ConditionStep(
name="AccuracyCondition",
conditions=[minimum_accuracy_condition],
if_steps=[register_step,
create_step], # success, continue with model registration
else_steps=[], # fail, end the pipeline
)
#########################
## CREATE PIPELINE
#########################
pipeline = Pipeline(
name=pipeline_name,
parameters=[
input_data,
processing_instance_count,
processing_instance_type,
max_seq_length,
balance_dataset,
train_split_percentage,
validation_split_percentage,
test_split_percentage,
feature_store_offline_prefix,
feature_group_name,
train_instance_type,
train_instance_count,
epochs,
learning_rate,
epsilon,
train_batch_size,
validation_batch_size,
test_batch_size,
train_steps_per_epoch,
validation_steps,
test_steps,
train_volume_size,
use_xla,
use_amp,
freeze_bert_layer,
enable_sagemaker_debugger,
enable_checkpointing,
enable_tensorboard,
input_mode,
run_validation,
run_test,
run_sample_predictions,
min_accuracy_value,
model_approval_status,
deploy_instance_type,
deploy_instance_count,
],
steps=[
processing_step, training_step, evaluation_step,
minimum_accuracy_condition_step
],
sagemaker_session=sess,
)
#########################
## RETURN PIPELINE
#########################
return pipeline
