def test_join_expressions(): assert Join( values=[ "foo", ParameterFloat(name="MyFloat"), ParameterInteger(name="MyInt"), ParameterString(name="MyStr"), Properties(path="Steps.foo.OutputPath.S3Uri"), ExecutionVariables.PIPELINE_EXECUTION_ID, Join(on=",", values=[1, "a", False, 1.1]), ] ).expr == { "Std:Join": { "On": "", "Values": [ "foo", {"Get": "Parameters.MyFloat"}, {"Get": "Parameters.MyInt"}, {"Get": "Parameters.MyStr"}, {"Get": "Steps.foo.OutputPath.S3Uri"}, {"Get": "Execution.PipelineExecutionId"}, {"Std:Join": {"On": ",", "Values": [1, "a", False, 1.1]}}, ], }, }
def test_join_primitives_default_on(): assert Join(values=[1, "a", False, 1.1]).expr == { "Std:Join": { "On": "", "Values": [1, "a", False, 1.1], }, }
def get_top_model_s3_uri(self, top_k: int, s3_bucket: str, prefix: str = ""): """Get the model artifact s3 uri from the top performing training jobs. Args: top_k (int): the index of the top performing training job tuning step stores up to 50 top performing training jobs, hence a valid top_k value is from 0 to 49. The best training job model is at index 0 s3_bucket (str): the s3 bucket to store the training job output artifact prefix (str): the s3 key prefix to store the training job output artifact """ values = ["s3:/", s3_bucket] if prefix != "" and prefix is not None: values.append(prefix) return Join( on="/", values=values + [ self.properties.TrainingJobSummaries[top_k].TrainingJobName, "output/model.tar.gz", ], )
def test_join_primitives(): assert Join(on=",", values=[1, "a", False, 1.1]).expr == { "Std:Join": { "On": ",", "Values": [1, "a", False, 1.1], }, }
def test_fail_step_with_join_fn_in_error_message(): param = ParameterInteger(name="MyInt", default_value=2) cond = ConditionEquals(left=param, right=1) step_cond = ConditionStep( name="CondStep", conditions=[cond], if_steps=[], else_steps=[], ) step_fail = FailStep( name="FailStep", error_message=Join(on=": ", values=[ "Failed due to xxx == yyy returns", step_cond.properties.Outcome ]), ) pipeline = Pipeline( name="MyPipeline", steps=[step_cond, step_fail], parameters=[param], ) _expected_dsl = [ { "Name": "CondStep", "Type": "Condition", "Arguments": { "Conditions": [{ "Type": "Equals", "LeftValue": { "Get": "Parameters.MyInt" }, "RightValue": 1 }], "IfSteps": [], "ElseSteps": [], }, }, { "Name": "FailStep", "Type": "Fail", "Arguments": { "ErrorMessage": { "Std:Join": { "On": ": ", "Values": [ "Failed due to xxx == yyy returns", { "Get": "Steps.CondStep.Outcome" }, ], } } }, }, ] assert json.loads(pipeline.definition())["Steps"] == _expected_dsl
def test_add_func_of_join(): func_join1 = Join(values=[1, "a"]) param = ParameterInteger(name="MyInteger", default_value=3) with pytest.raises(TypeError) as error: func_join1 + param assert str(error.value) == "Pipeline variables do not support concatenation."
def test_implicit_value_on_join(): func = Join(values=[1, "a", False, 1.1]) with pytest.raises(TypeError) as error: str(func) assert str(error.value) == "Pipeline variables do not support __str__ operation." with pytest.raises(TypeError) as error: int(func) assert str(error.value) == "Pipeline variables do not support __int__ operation." with pytest.raises(TypeError) as error: float(func) assert str(error.value) == "Pipeline variables do not support __float__ operation."
def to_string(self): """Prompt the pipeline to convert the pipeline variable to String in runtime""" from sagemaker.workflow.functions import Join return Join(on="", values=[self])
def test_string_builtin_funcs_that_return_bool_on_join(): func = Join(on=",", values=["s3:/", "my-bucket", "a"]) # The func will only be parsed in runtime (Pipeline backend) so not able to tell in SDK assert not func.startswith("s3") assert not func.endswith("s3")
def test_training_step_with_output_path_as_join( sagemaker_session, role, tf_full_version, tf_full_py_version, pipeline_name, region_name ): base_dir = os.path.join(DATA_DIR, "dummy_tensor") input_path = sagemaker_session.upload_data( path=base_dir, key_prefix="integ-test-data/estimator/training" ) inputs = TrainingInput(s3_data=input_path) instance_count = ParameterInteger(name="InstanceCount", default_value=1) instance_type = ParameterString(name="InstanceType", default_value="ml.m5.xlarge") output_path = Join( on="/", values=["s3:/", f"{sagemaker_session.default_bucket()}", f"{pipeline_name}Train"] ) image_uri = image_uris.retrieve("factorization-machines", sagemaker_session.boto_region_name) estimator = Estimator( image_uri=image_uri, role=role, instance_count=instance_count, instance_type=instance_type, sagemaker_session=sagemaker_session, output_path=output_path, ) estimator.set_hyperparameters( num_factors=10, feature_dim=784, mini_batch_size=100, predictor_type="binary_classifier" ) step_train = TrainingStep( name="MyTrain", estimator=estimator, inputs=inputs, ) pipeline = Pipeline( name=pipeline_name, parameters=[instance_count, instance_type], steps=[step_train], sagemaker_session=sagemaker_session, ) try: response = pipeline.create(role) create_arn = response["PipelineArn"] assert re.match( rf"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}", create_arn, ) execution = pipeline.start(parameters={}) assert re.match( rf"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}/execution/", execution.arn, ) try: execution.wait(delay=30, max_attempts=60) except WaiterError: pass execution_steps = execution.list_steps() assert len(execution_steps) == 1 assert execution_steps[0]["StepName"] == "MyTrain" finally: try: pipeline.delete() except Exception: pass
def test_two_step_fail_pipeline_with_join_fn(sagemaker_session, role, pipeline_name): param = ParameterInteger(name="MyInt", default_value=2) cond = ConditionEquals(left=param, right=1) step_cond = ConditionStep( name="CondStep", conditions=[cond], if_steps=[], else_steps=[], ) step_fail = FailStep( name="FailStep", error_message=Join(on=": ", values=[ "Failed due to xxx == yyy returns", step_cond.properties.Outcome ]), ) pipeline = Pipeline( name=pipeline_name, steps=[step_cond, step_fail], sagemaker_session=sagemaker_session, parameters=[param], ) try: response = pipeline.create(role) pipeline_arn = response["PipelineArn"] execution = pipeline.start(parameters={"MyInt": 3}, ) response = execution.describe() assert response["PipelineArn"] == pipeline_arn try: execution.wait(delay=30, max_attempts=60) except WaiterError: pass execution_steps = execution.list_steps() assert len(execution_steps) == 2 for execution_step in execution_steps: if execution_step["StepName"] == "CondStep": assert execution_step["StepStatus"] == "Succeeded" continue assert execution_step["StepName"] == "FailStep" assert execution_step["StepStatus"] == "Failed" assert execution_step[ "FailureReason"] == "Failed due to xxx == yyy returns: false" metadata = execution_steps[0]["Metadata"]["Fail"] assert metadata[ "ErrorMessage"] == "Failed due to xxx == yyy returns: false" # Check FailureReason field in ListPipelineExecutions executions = sagemaker_session.sagemaker_client.list_pipeline_executions( PipelineName=pipeline.name)["PipelineExecutionSummaries"] assert len(executions) == 1 assert executions[0]["PipelineExecutionStatus"] == "Failed" assert ("Step failure: One or multiple steps failed" in executions[0]["PipelineExecutionFailureReason"]) finally: try: pipeline.delete() except Exception: pass
def test_pipeline_execution_with_custom_experiment_config( sagemaker_session, smclient, role, sklearn_latest_version, cpu_instance_type, pipeline_name, athena_dataset_definition, ): instance_count = ParameterInteger(name="InstanceCount", default_value=2) script_path = os.path.join(DATA_DIR, "dummy_script.py") input_file_path = os.path.join(DATA_DIR, "dummy_input.txt") inputs = [ ProcessingInput(source=input_file_path, destination="/opt/ml/processing/inputs/"), ProcessingInput(dataset_definition=athena_dataset_definition), ] sklearn_processor = SKLearnProcessor( framework_version=sklearn_latest_version, role=role, instance_type=cpu_instance_type, instance_count=instance_count, command=["python3"], sagemaker_session=sagemaker_session, base_job_name="test-sklearn", ) step_sklearn = ProcessingStep( name="sklearn-process", processor=sklearn_processor, inputs=inputs, code=script_path, ) experiment_name = f"my-experiment-{int(time.time() * 10**7)}" pipeline = Pipeline( name=pipeline_name, parameters=[instance_count], pipeline_experiment_config=PipelineExperimentConfig( experiment_name=experiment_name, trial_name=Join( on="-", values=["my-trial", ExecutionVariables.PIPELINE_EXECUTION_ID]), ), steps=[step_sklearn], sagemaker_session=sagemaker_session, ) try: pipeline.create(role) execution = pipeline.start(parameters={}) try: execution.wait(delay=30, max_attempts=3) except WaiterError: pass execution_steps = execution.list_steps() assert len(execution_steps) == 1 assert execution_steps[0]["StepName"] == "sklearn-process" execution_id = execution.arn.split("/")[-1] # trial components trial_components = smclient.list_trial_components( TrialName=f"my-trial-{execution_id}") assert len(trial_components["TrialComponentSummaries"]) == 1 # trial details trial = smclient.describe_trial(TrialName=f"my-trial-{execution_id}") assert experiment_name == trial["ExperimentName"] finally: try: pipeline.delete() except Exception: pass
def test_steps_with_map_params_pipeline( sagemaker_session, role, script_dir, pipeline_name, region_name, athena_dataset_definition, ): instance_count = ParameterInteger(name="InstanceCount", default_value=2) framework_version = "0.20.0" instance_type = ParameterString(name="InstanceType", default_value="ml.m5.xlarge") output_prefix = ParameterString(name="OutputPrefix", default_value="output") input_data = f"s3://sagemaker-sample-data-{region_name}/processing/census/census-income.csv" sklearn_processor = SKLearnProcessor( framework_version=framework_version, instance_type=instance_type, instance_count=instance_count, base_job_name="test-sklearn", sagemaker_session=sagemaker_session, role=role, ) step_process = ProcessingStep( name="my-process", display_name="ProcessingStep", description="description for Processing step", processor=sklearn_processor, inputs=[ ProcessingInput(source=input_data, destination="/opt/ml/processing/input"), ProcessingInput(dataset_definition=athena_dataset_definition), ], outputs=[ ProcessingOutput(output_name="train_data", source="/opt/ml/processing/train"), ProcessingOutput( output_name="test_data", source="/opt/ml/processing/test", destination=Join( on="/", values=[ "s3:/", sagemaker_session.default_bucket(), "test-sklearn", output_prefix, ExecutionVariables.PIPELINE_EXECUTION_ID, ], ), ), ], code=os.path.join(script_dir, "preprocessing.py"), ) sklearn_train = SKLearn( framework_version=framework_version, entry_point=os.path.join(script_dir, "train.py"), instance_type=instance_type, sagemaker_session=sagemaker_session, role=role, hyperparameters={ "batch-size": 500, "epochs": 5, }, ) step_train = TrainingStep( name="my-train", display_name="TrainingStep", description="description for Training step", estimator=sklearn_train, inputs=TrainingInput( s3_data=step_process.properties.ProcessingOutputConfig.Outputs[ "train_data" ].S3Output.S3Uri ), ) model = Model( image_uri=sklearn_train.image_uri, model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts, sagemaker_session=sagemaker_session, role=role, ) model_inputs = CreateModelInput( instance_type="ml.m5.large", accelerator_type="ml.eia1.medium", ) step_model = CreateModelStep( name="my-model", display_name="ModelStep", description="description for Model step", model=model, inputs=model_inputs, ) # Condition step for evaluating model quality and branching execution cond_lte = ConditionGreaterThanOrEqualTo( left=step_train.properties.HyperParameters["batch-size"], right=6.0, ) step_cond = ConditionStep( name="CustomerChurnAccuracyCond", conditions=[cond_lte], if_steps=[], else_steps=[step_model], ) pipeline = Pipeline( name=pipeline_name, parameters=[instance_type, instance_count, output_prefix], steps=[step_process, step_train, step_cond], sagemaker_session=sagemaker_session, ) definition = json.loads(pipeline.definition()) assert definition["Version"] == "2020-12-01" steps = definition["Steps"] assert len(steps) == 3 training_args = {} condition_args = {} for step in steps: if step["Type"] == "Training": training_args = step["Arguments"] if step["Type"] == "Condition": condition_args = step["Arguments"] assert training_args["InputDataConfig"][0]["DataSource"]["S3DataSource"]["S3Uri"] == { "Get": "Steps.my-process.ProcessingOutputConfig.Outputs['train_data'].S3Output.S3Uri" } assert condition_args["Conditions"][0]["LeftValue"] == { "Get": "Steps.my-train.HyperParameters['batch-size']" } try: response = pipeline.create(role) create_arn = response["PipelineArn"] assert re.match( rf"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}", create_arn, ) finally: try: pipeline.delete() except Exception: pass
def test_pipeline_variable_in_pipeline_definition(sagemaker_session): param_str = ParameterString(name="MyString", default_value="1") param_int = ParameterInteger(name="MyInteger", default_value=3) property_file = PropertyFile( name="name", output_name="result", path="output", ) json_get_func2 = JsonGet( step_name="my-step", property_file=property_file, json_path="my-json-path", ) prop = Properties("Steps.MyStep", "DescribeProcessingJobResponse") cond = ConditionGreaterThan(left=param_str, right=param_int.to_string()) step_fail = FailStep( name="MyFailStep", error_message=Join( on=" ", values=[ "Execution failed due to condition check fails, see:", json_get_func2.to_string(), prop.ProcessingOutputConfig.Outputs["MyOutputName"].S3Output. S3Uri.to_string(), param_int, ], ), ) step_cond = ConditionStep( name="MyCondStep", conditions=[cond], if_steps=[], else_steps=[step_fail], ) pipeline = Pipeline( name="MyPipeline", parameters=[param_str, param_int], steps=[step_cond], sagemaker_session=sagemaker_session, ) dsl = json.loads(pipeline.definition()) assert dsl["Parameters"] == [ { "Name": "MyString", "Type": "String", "DefaultValue": "1" }, { "Name": "MyInteger", "Type": "Integer", "DefaultValue": 3 }, ] assert len(dsl["Steps"]) == 1 assert dsl["Steps"][0] == { "Name": "MyCondStep", "Type": "Condition", "Arguments": { "Conditions": [ { "Type": "GreaterThan", "LeftValue": { "Get": "Parameters.MyString" }, "RightValue": { "Std:Join": { "On": "", "Values": [{ "Get": "Parameters.MyInteger" }], }, }, }, ], "IfSteps": [], "ElseSteps": [{ "Name": "MyFailStep", "Type": "Fail", "Arguments": { "ErrorMessage": { "Std:Join": { "On": " ", "Values": [ "Execution failed due to condition check fails, see:", { "Std:Join": { "On": "", "Values": [ { "Std:JsonGet": { "PropertyFile": { "Get": "Steps.my-step.PropertyFiles.name" }, "Path": "my-json-path", } }, ], }, }, { "Std:Join": { "On": "", "Values": [ { "Get": "Steps.MyStep.ProcessingOutputConfig." + "Outputs['MyOutputName'].S3Output.S3Uri" }, ], }, }, { "Get": "Parameters.MyInteger" }, ], } } }, }], }, }
def test_to_string_on_join(): func = Join(values=[1, "a", False, 1.1]) assert func.to_string() == func
def test_three_step_definition( sagemaker_session, region_name, role, script_dir, pipeline_name, athena_dataset_definition, ): framework_version = "0.20.0" instance_type = ParameterString(name="InstanceType", default_value="ml.m5.xlarge") instance_count = ParameterInteger(name="InstanceCount", default_value=1) output_prefix = ParameterString(name="OutputPrefix", default_value="output") input_data = f"s3://sagemaker-sample-data-{region_name}/processing/census/census-income.csv" sklearn_processor = SKLearnProcessor( framework_version=framework_version, instance_type=instance_type, instance_count=instance_count, base_job_name="test-sklearn", sagemaker_session=sagemaker_session, role=role, ) step_process = ProcessingStep( name="my-process", processor=sklearn_processor, inputs=[ ProcessingInput(source=input_data, destination="/opt/ml/processing/input"), ProcessingInput(dataset_definition=athena_dataset_definition), ], outputs=[ ProcessingOutput(output_name="train_data", source="/opt/ml/processing/train"), ProcessingOutput( output_name="test_data", source="/opt/ml/processing/test", destination=Join( on="/", values=[ "s3:/", sagemaker_session.default_bucket(), "test-sklearn", output_prefix, ExecutionVariables.PIPELINE_EXECUTION_ID, ], ), ), ], code=os.path.join(script_dir, "preprocessing.py"), ) sklearn_train = SKLearn( framework_version=framework_version, entry_point=os.path.join(script_dir, "train.py"), instance_type=instance_type, sagemaker_session=sagemaker_session, role=role, ) step_train = TrainingStep( name="my-train", estimator=sklearn_train, inputs=TrainingInput( s3_data=step_process.properties.ProcessingOutputConfig. Outputs["train_data"].S3Output.S3Uri), ) model = Model( image_uri=sklearn_train.image_uri, model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts, sagemaker_session=sagemaker_session, role=role, ) model_inputs = CreateModelInput( instance_type="ml.m5.large", accelerator_type="ml.eia1.medium", ) step_model = CreateModelStep( name="my-model", model=model, inputs=model_inputs, ) pipeline = Pipeline( name=pipeline_name, parameters=[instance_type, instance_count, output_prefix], steps=[step_process, step_train, step_model], sagemaker_session=sagemaker_session, ) definition = json.loads(pipeline.definition()) assert definition["Version"] == "2020-12-01" assert set(tuple(param.items()) for param in definition["Parameters"]) == set([ tuple({ "Name": "InstanceType", "Type": "String", "DefaultValue": "ml.m5.xlarge" }.items()), tuple({ "Name": "InstanceCount", "Type": "Integer", "DefaultValue": 1 }.items()), tuple({ "Name": "OutputPrefix", "Type": "String", "DefaultValue": "output" }.items()), ]) steps = definition["Steps"] assert len(steps) == 3 names_and_types = [] processing_args = {} training_args = {} for step in steps: names_and_types.append((step["Name"], step["Type"])) if step["Type"] == "Processing": processing_args = step["Arguments"] if step["Type"] == "Training": training_args = step["Arguments"] if step["Type"] == "Model": model_args = step["Arguments"] assert set(names_and_types) == set([ ("my-process", "Processing"), ("my-train", "Training"), ("my-model", "Model"), ]) assert processing_args["ProcessingResources"]["ClusterConfig"] == { "InstanceType": { "Get": "Parameters.InstanceType" }, "InstanceCount": { "Get": "Parameters.InstanceCount" }, "VolumeSizeInGB": 30, } assert training_args["ResourceConfig"] == { "InstanceCount": 1, "InstanceType": { "Get": "Parameters.InstanceType" }, "VolumeSizeInGB": 30, } assert training_args["InputDataConfig"][0]["DataSource"]["S3DataSource"][ "S3Uri"] == { "Get": "Steps.my-process.ProcessingOutputConfig.Outputs['train_data'].S3Output.S3Uri" } assert model_args["PrimaryContainer"]["ModelDataUrl"] == { "Get": "Steps.my-train.ModelArtifacts.S3ModelArtifacts" } try: response = pipeline.create(role) create_arn = response["PipelineArn"] assert re.match( fr"arn:aws:sagemaker:{region_name}:\d{{12}}:pipeline/{pipeline_name}", create_arn, ) finally: try: pipeline.delete() except Exception: pass
def test_ppl_var_to_string_and_add(sagemaker_session, role, pipeline_name): param_str = ParameterString(name="MyString", default_value="1") param_int = ParameterInteger(name="MyInteger", default_value=3) cond = ConditionGreaterThan(left=param_str, right=param_int.to_string()) step_cond = ConditionStep( name="CondStep", conditions=[cond], if_steps=[], else_steps=[], ) join_fn1 = Join( on=" ", values=[ "condition greater than check return:", step_cond.properties.Outcome.to_string(), "and left side param str is", param_str, "and right side param int is", param_int, ], ) step_fail = FailStep( name="FailStep", error_message=join_fn1, ) pipeline = Pipeline( name=pipeline_name, parameters=[param_str, param_int], steps=[step_cond, step_fail], sagemaker_session=sagemaker_session, ) try: response = pipeline.create(role) pipeline_arn = response["PipelineArn"] execution = pipeline.start() response = execution.describe() assert response["PipelineArn"] == pipeline_arn try: execution.wait(delay=30, max_attempts=60) except WaiterError: pass execution_steps = execution.list_steps() assert len(execution_steps) == 2 for execution_step in execution_steps: if execution_step["StepName"] == "CondStep": assert execution_step["StepStatus"] == "Succeeded" continue assert execution_step["StepName"] == "FailStep" assert execution_step["StepStatus"] == "Failed" assert ( execution_step["FailureReason"] == "condition greater than check return: false " "and left side param str is 1 and right side param int is 3") # Update int param to update cond step outcome execution = pipeline.start(parameters={"MyInteger": 0}) try: execution.wait(delay=30, max_attempts=60) except WaiterError: pass execution_steps = execution.list_steps() assert len(execution_steps) == 2 for execution_step in execution_steps: if execution_step["StepName"] == "CondStep": assert execution_step["StepStatus"] == "Succeeded" continue assert execution_step["StepName"] == "FailStep" assert execution_step["StepStatus"] == "Failed" assert ( execution_step["FailureReason"] == "condition greater than check return: true " "and left side param str is 1 and right side param int is 0") finally: try: pipeline.delete() except Exception: pass