def test_json_get_expressions():
assert JsonGet(
step_name="my-step",
property_file="my-property-file",
json_path="my-json-path",
).expr == {
"Std:JsonGet": {
"PropertyFile": {
"Get": "Steps.my-step.PropertyFiles.my-property-file"
},
"Path": "my-json-path",
},
}
property_file = PropertyFile(
name="name",
output_name="result",
path="output",
)
assert JsonGet(
step_name="my-step",
property_file=property_file,
json_path="my-json-path",
).expr == {
"Std:JsonGet": {
"PropertyFile": {
"Get": "Steps.my-step.PropertyFiles.name"
},
"Path": "my-json-path",
},
}
def test_string_builtin_funcs_that_return_bool_on_json_get():
func = JsonGet(
step_name="my-step",
property_file="my-property-file",
json_path="my-json-path",
)
# The func will only be parsed in runtime (Pipeline backend) so not able to tell in SDK
assert not func.startswith("s3")
assert not func.endswith("s3")
def test_json_get_expressions_with_invalid_step_name():
with pytest.raises(ValueError) as err:
JsonGet(
step_name="",
property_file="my-property-file",
json_path="my-json-path",
).expr
assert "Please give a valid step name as a string" in str(err.value)
with pytest.raises(ValueError) as err:
JsonGet(
step_name=ParameterString(name="MyString"),
property_file="my-property-file",
json_path="my-json-path",
).expr
assert "Please give a valid step name as a string" in str(err.value)
def test_to_string_on_json_get():
func = JsonGet(
step_name="my-step",
property_file="my-property-file",
json_path="my-json-path",
)
assert func.to_string().expr == {
"Std:Join": {
"On": "",
"Values": [
{
"Std:JsonGet": {
"Path": "my-json-path",
"PropertyFile": {"Get": "Steps.my-step.PropertyFiles.my-property-file"},
}
}
],
},
}
def test_add_func_of_json_get():
json_get_func1 = JsonGet(
step_name="my-step",
property_file="my-property-file",
json_path="my-json-path",
)
property_file = PropertyFile(
name="name",
output_name="result",
path="output",
)
json_get_func2 = JsonGet(
step_name="my-step",
property_file=property_file,
json_path="my-json-path",
)
with pytest.raises(TypeError) as error:
json_get_func1 + json_get_func2
assert str(error.value) == "Pipeline variables do not support concatenation."
def test_implicit_value_on_json_get():
func = JsonGet(
step_name="my-step",
property_file="my-property-file",
json_path="my-json-path",
)
with pytest.raises(TypeError) as error:
str(func)
assert str(error.value) == "Pipeline variables do not support __str__ operation."
with pytest.raises(TypeError) as error:
int(func)
assert str(error.value) == "Pipeline variables do not support __int__ operation."
with pytest.raises(TypeError) as error:
float(func)
assert str(error.value) == "Pipeline variables do not support __float__ operation."
def get_pipeline(
region,
role=None,
default_bucket=None,
model_package_group_name="CustomerChurnPackageGroup", # Choose any name
pipeline_name="CustomerChurnDemo-p-ewf8t7lvhivm", # You can find your pipeline name in the Studio UI (project -> Pipelines -> name)
base_job_prefix="CustomerChurn", # Choose any name
):
"""Gets a SageMaker ML Pipeline instance working with on CustomerChurn data.
Args:
region: AWS region to create and run the pipeline.
role: IAM role to create and run steps and pipeline.
default_bucket: the bucket to use for storing the artifacts
Returns:
an instance of a pipeline
"""
sagemaker_session = get_session(region, default_bucket)
if role is None:
role = sagemaker.session.get_execution_role(sagemaker_session)
# Parameters for pipeline execution
processing_instance_count = ParameterInteger(
name="ProcessingInstanceCount", default_value=1)
processing_instance_type = ParameterString(name="ProcessingInstanceType",
default_value="ml.m5.xlarge")
training_instance_type = ParameterString(name="TrainingInstanceType",
default_value="ml.m5.xlarge")
model_approval_status = ParameterString(
name="ModelApprovalStatus",
default_value=
"PendingManualApproval", # ModelApprovalStatus can be set to a default of "Approved" if you don't want manual approval.
)
input_data = ParameterString(
name="InputDataUrl",
default_value=
f"s3://sm-pipelines-demo-data-123456789/churn.txt", # Change this to point to the s3 location of your raw input data.
)
# Processing step for feature engineering
sklearn_processor = SKLearnProcessor(
framework_version="0.23-1",
instance_type=processing_instance_type,
instance_count=processing_instance_count,
base_job_name=
f"{base_job_prefix}/sklearn-CustomerChurn-preprocess", # choose any name
sagemaker_session=sagemaker_session,
role=role,
)
step_process = ProcessingStep(
name="CustomerChurnProcess", # choose any name
processor=sklearn_processor,
outputs=[
ProcessingOutput(output_name="train",
source="/opt/ml/processing/train"),
ProcessingOutput(output_name="validation",
source="/opt/ml/processing/validation"),
ProcessingOutput(output_name="test",
source="/opt/ml/processing/test"),
],
code=os.path.join(BASE_DIR, "preprocess.py"),
job_arguments=["--input-data", input_data],
)
# Training step for generating model artifacts
model_path = f"s3://{sagemaker_session.default_bucket()}/{base_job_prefix}/CustomerChurnTrain"
image_uri = sagemaker.image_uris.retrieve(
framework=
"xgboost", # we are using the Sagemaker built in xgboost algorithm
region=region,
version="1.0-1",
py_version="py3",
instance_type=training_instance_type,
)
xgb_train = Estimator(
image_uri=image_uri,
instance_type=training_instance_type,
instance_count=1,
output_path=model_path,
base_job_name=f"{base_job_prefix}/CustomerChurn-train",
sagemaker_session=sagemaker_session,
role=role,
)
xgb_train.set_hyperparameters(
objective="binary:logistic",
num_round=50,
max_depth=5,
eta=0.2,
gamma=4,
min_child_weight=6,
subsample=0.7,
silent=0,
)
step_train = TrainingStep(
name="CustomerChurnTrain",
estimator=xgb_train,
inputs={
"train":
TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.
Outputs["train"].S3Output.S3Uri,
content_type="text/csv",
),
"validation":
TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.
Outputs["validation"].S3Output.S3Uri,
content_type="text/csv",
),
},
)
# Processing step for evaluation
script_eval = ScriptProcessor(
image_uri=image_uri,
command=["python3"],
instance_type=processing_instance_type,
instance_count=1,
base_job_name=f"{base_job_prefix}/script-CustomerChurn-eval",
sagemaker_session=sagemaker_session,
role=role,
)
evaluation_report = PropertyFile(
name="EvaluationReport",
output_name="evaluation",
path="evaluation.json",
)
step_eval = ProcessingStep(
name="CustomerChurnEval",
processor=script_eval,
inputs=[
ProcessingInput(
source=step_train.properties.ModelArtifacts.S3ModelArtifacts,
destination="/opt/ml/processing/model",
),
ProcessingInput(
source=step_process.properties.ProcessingOutputConfig.
Outputs["test"].S3Output.S3Uri,
destination="/opt/ml/processing/test",
),
],
outputs=[
ProcessingOutput(output_name="evaluation",
source="/opt/ml/processing/evaluation"),
],
code=os.path.join(BASE_DIR, "evaluate.py"),
property_files=[evaluation_report],
)
# Register model step that will be conditionally executed
model_metrics = ModelMetrics(model_statistics=MetricsSource(
s3_uri="{}/evaluation.json".format(
step_eval.arguments["ProcessingOutputConfig"]["Outputs"][0]
["S3Output"]["S3Uri"]),
content_type="application/json",
))
# Register model step that will be conditionally executed
step_register = RegisterModel(
name="CustomerChurnRegisterModel",
estimator=xgb_train,
model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,
content_types=["text/csv"],
response_types=["text/csv"],
inference_instances=["ml.t2.medium", "ml.m5.large"],
transform_instances=["ml.m5.large"],
model_package_group_name=model_package_group_name,
approval_status=model_approval_status,
model_metrics=model_metrics,
)
# Condition step for evaluating model quality and branching execution
cond_lte = ConditionGreaterThanOrEqualTo( # You can change the condition here
left=JsonGet(
step_name=step_eval.name,
property_file=evaluation_report,
json_path=
"binary_classification_metrics.accuracy.value", # This should follow the structure of your report_dict defined in the evaluate.py file.
),
right=0.8, # You can change the threshold here
)
step_cond = ConditionStep(
name="CustomerChurnAccuracyCond",
conditions=[cond_lte],
if_steps=[step_register],
else_steps=[],
)
# Pipeline instance
pipeline = Pipeline(
name=pipeline_name,
parameters=[
processing_instance_type,
processing_instance_count,
training_instance_type,
model_approval_status,
input_data,
],
steps=[step_process, step_train, step_eval, step_cond],
sagemaker_session=sagemaker_session,
)
return pipeline
def test_end_to_end_pipeline_successful_execution(
sagemaker_session, region_name, role, pipeline_name, wait=False
):
model_package_group_name = f"{pipeline_name}ModelPackageGroup"
data_path = os.path.join(DATA_DIR, "workflow")
default_bucket = sagemaker_session.default_bucket()
# download the input data
local_input_path = os.path.join(data_path, "abalone-dataset.csv")
s3 = sagemaker_session.boto_session.resource("s3")
s3.Bucket(f"sagemaker-servicecatalog-seedcode-{region_name}").download_file(
"dataset/abalone-dataset.csv", local_input_path
)
# # upload the input data to our bucket
base_uri = f"s3://{default_bucket}/{pipeline_name}"
with open(local_input_path) as data:
body = data.read()
input_data_uri = S3Uploader.upload_string_as_file_body(
body=body,
desired_s3_uri=f"{base_uri}/abalone-dataset.csv",
sagemaker_session=sagemaker_session,
)
# download batch transform data
local_batch_path = os.path.join(data_path, "abalone-dataset-batch")
s3.Bucket(f"sagemaker-servicecatalog-seedcode-{region_name}").download_file(
"dataset/abalone-dataset-batch", local_batch_path
)
# upload the batch transform data
with open(local_batch_path) as data:
body = data.read()
batch_data_uri = S3Uploader.upload_string_as_file_body(
body=body,
desired_s3_uri=f"{base_uri}/abalone-dataset-batch",
sagemaker_session=sagemaker_session,
)
# define parameters
processing_instance_count = ParameterInteger(name="ProcessingInstanceCount", default_value=1)
processing_instance_type = ParameterString(
name="ProcessingInstanceType", default_value="ml.m5.xlarge"
)
training_instance_type = ParameterString(
name="TrainingInstanceType", default_value="ml.m5.xlarge"
)
model_approval_status = ParameterString(name="ModelApprovalStatus", default_value="Approved")
input_data = ParameterString(
name="InputData",
default_value=input_data_uri,
)
batch_data = ParameterString(
name="BatchData",
default_value=batch_data_uri,
)
# define processing step
framework_version = "0.23-1"
sklearn_processor = SKLearnProcessor(
framework_version=framework_version,
instance_type=processing_instance_type,
instance_count=processing_instance_count,
base_job_name=f"{pipeline_name}-process",
role=role,
sagemaker_session=sagemaker_session,
)
step_process = ProcessingStep(
name="AbaloneProcess",
processor=sklearn_processor,
inputs=[
ProcessingInput(source=input_data, destination="/opt/ml/processing/input"),
],
outputs=[
ProcessingOutput(output_name="train", source="/opt/ml/processing/train"),
ProcessingOutput(output_name="validation", source="/opt/ml/processing/validation"),
ProcessingOutput(output_name="test", source="/opt/ml/processing/test"),
],
code=os.path.join(data_path, "abalone/preprocessing.py"),
)
# define training step
model_path = f"s3://{default_bucket}/{pipeline_name}Train"
image_uri = image_uris.retrieve(
framework="xgboost",
region=region_name,
version="1.0-1",
py_version="py3",
instance_type=training_instance_type,
)
xgb_train = Estimator(
image_uri=image_uri,
instance_type=training_instance_type,
instance_count=1,
output_path=model_path,
role=role,
sagemaker_session=sagemaker_session,
)
xgb_train.set_hyperparameters(
objective="reg:linear",
num_round=50,
max_depth=5,
eta=0.2,
gamma=4,
min_child_weight=6,
subsample=0.7,
silent=0,
)
step_train = TrainingStep(
name="AbaloneTrain",
estimator=xgb_train,
inputs={
"train": TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.Outputs[
"train"
].S3Output.S3Uri,
content_type="text/csv",
),
"validation": TrainingInput(
s3_data=step_process.properties.ProcessingOutputConfig.Outputs[
"validation"
].S3Output.S3Uri,
content_type="text/csv",
),
},
)
# define evaluation step
script_eval = ScriptProcessor(
image_uri=image_uri,
command=["python3"],
instance_type=processing_instance_type,
instance_count=1,
base_job_name=f"{pipeline_name}-eval",
role=role,
sagemaker_session=sagemaker_session,
)
evaluation_report = PropertyFile(
name="EvaluationReport", output_name="evaluation", path="evaluation.json"
)
step_eval = ProcessingStep(
name="AbaloneEval",
processor=script_eval,
inputs=[
ProcessingInput(
source=step_train.properties.ModelArtifacts.S3ModelArtifacts,
destination="/opt/ml/processing/model",
),
ProcessingInput(
source=step_process.properties.ProcessingOutputConfig.Outputs[
"test"
].S3Output.S3Uri,
destination="/opt/ml/processing/test",
),
],
outputs=[
ProcessingOutput(output_name="evaluation", source="/opt/ml/processing/evaluation"),
],
code=os.path.join(data_path, "abalone/evaluation.py"),
property_files=[evaluation_report],
)
# define create model step
model = Model(
image_uri=image_uri,
model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,
sagemaker_session=sagemaker_session,
role=role,
)
inputs = CreateModelInput(
instance_type="ml.m5.large",
accelerator_type="ml.eia1.medium",
)
step_create_model = CreateModelStep(
name="AbaloneCreateModel",
model=model,
inputs=inputs,
)
# define transform step
transformer = Transformer(
model_name=step_create_model.properties.ModelName,
instance_type="ml.m5.xlarge",
instance_count=1,
output_path=f"s3://{default_bucket}/{pipeline_name}Transform",
sagemaker_session=sagemaker_session,
)
step_transform = TransformStep(
name="AbaloneTransform",
transformer=transformer,
inputs=TransformInput(data=batch_data),
)
# define register model step
model_metrics = ModelMetrics(
model_statistics=MetricsSource(
s3_uri="{}/evaluation.json".format(
step_eval.arguments["ProcessingOutputConfig"]["Outputs"][0]["S3Output"]["S3Uri"]
),
content_type="application/json",
)
)
step_register = RegisterModel(
name="AbaloneRegisterModel",
estimator=xgb_train,
model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,
content_types=["text/csv"],
response_types=["text/csv"],
inference_instances=["ml.t2.medium", "ml.m5.xlarge"],
transform_instances=["ml.m5.xlarge"],
model_package_group_name=model_package_group_name,
approval_status=model_approval_status,
model_metrics=model_metrics,
)
# define condition step
cond_lte = ConditionLessThanOrEqualTo(
left=JsonGet(
step_name=step_eval.name,
property_file=evaluation_report,
json_path="regression_metrics.mse.value",
),
right=20.0,
)
step_cond = ConditionStep(
name="AbaloneMSECond",
conditions=[cond_lte],
if_steps=[step_register, step_create_model, step_transform],
else_steps=[],
)
# define pipeline
pipeline = Pipeline(
name=pipeline_name,
parameters=[
processing_instance_type,
processing_instance_count,
training_instance_type,
model_approval_status,
input_data,
batch_data,
],
steps=[step_process, step_train, step_eval, step_cond],
sagemaker_session=sagemaker_session,
)
pipeline.create(role)
execution = pipeline.start()
execution_arn = execution.arn
if wait:
execution.wait()
return execution_arn
def test_tuning_multi_algos(
sagemaker_session,
role,
cpu_instance_type,
pipeline_name,
region_name,
script_dir,
athena_dataset_definition,
):
base_dir = os.path.join(DATA_DIR, "pytorch_mnist")
entry_point = os.path.join(base_dir, "mnist.py")
input_path = sagemaker_session.upload_data(
path=os.path.join(base_dir, "training"),
key_prefix="integ-test-data/pytorch_mnist/training",
)
instance_count = ParameterInteger(name="InstanceCount", default_value=1)
instance_type = ParameterString(name="InstanceType",
default_value="ml.m5.xlarge")
input_data = f"s3://sagemaker-sample-data-{region_name}/processing/census/census-income.csv"
sklearn_processor = SKLearnProcessor(
framework_version="0.20.0",
instance_type=instance_type,
instance_count=instance_count,
base_job_name="test-sklearn",
sagemaker_session=sagemaker_session,
role=role,
)
property_file = PropertyFile(name="DataAttributes",
output_name="attributes",
path="attributes.json")
step_process = ProcessingStep(
name="my-process",
display_name="ProcessingStep",
description="description for Processing step",
processor=sklearn_processor,
inputs=[
ProcessingInput(source=input_data,
destination="/opt/ml/processing/input"),
ProcessingInput(dataset_definition=athena_dataset_definition),
],
outputs=[
ProcessingOutput(output_name="train_data",
source="/opt/ml/processing/train"),
ProcessingOutput(output_name="attributes",
source="/opt/ml/processing/attributes.json"),
],
property_files=[property_file],
code=os.path.join(script_dir, "preprocessing.py"),
)
static_hp_1 = ParameterString(name="InstanceType",
default_value="ml.m5.xlarge")
json_get_hp = JsonGet(step_name=step_process.name,
property_file=property_file,
json_path="train_size")
pytorch_estimator = PyTorch(
entry_point=entry_point,
role=role,
framework_version="1.5.0",
py_version="py3",
instance_count=instance_count,
instance_type=instance_type,
sagemaker_session=sagemaker_session,
enable_sagemaker_metrics=True,
max_retry_attempts=3,
hyperparameters={
"static-hp": static_hp_1,
"train_size": json_get_hp
},
)
min_batch_size = ParameterString(name="MinBatchSize", default_value="64")
max_batch_size = json_get_hp
tuner = HyperparameterTuner.create(
estimator_dict={
"estimator-1": pytorch_estimator,
"estimator-2": pytorch_estimator,
},
objective_metric_name_dict={
"estimator-1": "test:acc",
"estimator-2": "test:acc",
},
hyperparameter_ranges_dict={
"estimator-1": {
"batch-size": IntegerParameter(min_batch_size, max_batch_size)
},
"estimator-2": {
"batch-size": IntegerParameter(min_batch_size, max_batch_size)
},
},
metric_definitions_dict={
"estimator-1": [{
"Name": "test:acc",
"Regex": "Overall test accuracy: (.*?);"
}],
"estimator-2": [{
"Name": "test:acc",
"Regex": "Overall test accuracy: (.*?);"
}],
},
)
inputs = {
"estimator-1": TrainingInput(s3_data=input_path),
"estimator-2": TrainingInput(s3_data=input_path),
}
step_tune = TuningStep(
name="my-tuning-step",
tuner=tuner,
inputs=inputs,
)
pipeline = Pipeline(
name=pipeline_name,
parameters=[
instance_count, instance_type, min_batch_size, max_batch_size
],
steps=[step_process, step_tune],
sagemaker_session=sagemaker_session,
)
try:
response = pipeline.create(role)
create_arn = response["PipelineArn"]
assert re.match(
rf"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}",
create_arn,
)
execution = pipeline.start(parameters={})
assert re.match(
rf"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}/execution/",
execution.arn,
)
finally:
try:
pipeline.delete()
except Exception:
pass
def test_pipeline_variable_in_pipeline_definition(sagemaker_session):
param_str = ParameterString(name="MyString", default_value="1")
param_int = ParameterInteger(name="MyInteger", default_value=3)
property_file = PropertyFile(
name="name",
output_name="result",
path="output",
)
json_get_func2 = JsonGet(
step_name="my-step",
property_file=property_file,
json_path="my-json-path",
)
prop = Properties("Steps.MyStep", "DescribeProcessingJobResponse")
cond = ConditionGreaterThan(left=param_str, right=param_int.to_string())
step_fail = FailStep(
name="MyFailStep",
error_message=Join(
on=" ",
values=[
"Execution failed due to condition check fails, see:",
json_get_func2.to_string(),
prop.ProcessingOutputConfig.Outputs["MyOutputName"].S3Output.
S3Uri.to_string(),
param_int,
],
),
)
step_cond = ConditionStep(
name="MyCondStep",
conditions=[cond],
if_steps=[],
else_steps=[step_fail],
)
pipeline = Pipeline(
name="MyPipeline",
parameters=[param_str, param_int],
steps=[step_cond],
sagemaker_session=sagemaker_session,
)
dsl = json.loads(pipeline.definition())
assert dsl["Parameters"] == [
{
"Name": "MyString",
"Type": "String",
"DefaultValue": "1"
},
{
"Name": "MyInteger",
"Type": "Integer",
"DefaultValue": 3
},
]
assert len(dsl["Steps"]) == 1
assert dsl["Steps"][0] == {
"Name": "MyCondStep",
"Type": "Condition",
"Arguments": {
"Conditions": [
{
"Type": "GreaterThan",
"LeftValue": {
"Get": "Parameters.MyString"
},
"RightValue": {
"Std:Join": {
"On": "",
"Values": [{
"Get": "Parameters.MyInteger"
}],
},
},
},
],
"IfSteps": [],
"ElseSteps": [{
"Name": "MyFailStep",
"Type": "Fail",
"Arguments": {
"ErrorMessage": {
"Std:Join": {
"On":
" ",
"Values": [
"Execution failed due to condition check fails, see:",
{
"Std:Join": {
"On":
"",
"Values": [
{
"Std:JsonGet": {
"PropertyFile": {
"Get":
"Steps.my-step.PropertyFiles.name"
},
"Path": "my-json-path",
}
},
],
},
},
{
"Std:Join": {
"On":
"",
"Values": [
{
"Get":
"Steps.MyStep.ProcessingOutputConfig."
+
"Outputs['MyOutputName'].S3Output.S3Uri"
},
],
},
},
{
"Get": "Parameters.MyInteger"
},
],
}
}
},
}],
},
}