def transformer(self, instance_count, instance_type, strategy=None, assemble_with=None, output_path=None, output_kms_key=None, accept=None, env=None, max_concurrent_transforms=None, max_payload=None, tags=None, volume_kms_key=None): """Return a ``Transformer`` that uses this Model. Args: instance_count (int): Number of EC2 instances to use. instance_type (str): Type of EC2 instance to use, for example, 'ml.c4.xlarge'. strategy (str): The strategy used to decide how to batch records in a single request (default: None). Valid values: 'MULTI_RECORD' and 'SINGLE_RECORD'. assemble_with (str): How the output is assembled (default: None). Valid values: 'Line' or 'None'. output_path (str): S3 location for saving the transform result. If not specified, results are stored to a default bucket. output_kms_key (str): Optional. KMS key ID for encrypting the transform output (default: None). accept (str): The content type accepted by the endpoint deployed during the transform job. env (dict): Environment variables to be set for use during the transform job (default: None). max_concurrent_transforms (int): The maximum number of HTTP requests to be made to each individual transform container at one time. max_payload (int): Maximum size of the payload in a single HTTP request to the container in MB. tags (list[dict]): List of tags for labeling a transform job. If none specified, then the tags used for the training job are used for the transform job. volume_kms_key (str): Optional. KMS key ID for encrypting the volume attached to the ML compute instance (default: None). """ self._create_sagemaker_model(instance_type) if self.enable_network_isolation(): env = None return Transformer(self.name, instance_count, instance_type, strategy=strategy, assemble_with=assemble_with, output_path=output_path, output_kms_key=output_kms_key, accept=accept, max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload, env=env, tags=tags, base_transform_job_name=self.name, volume_kms_key=volume_kms_key, sagemaker_session=self.sagemaker_session)
def pca_transformer(pca_model): return Transformer( model_name='pca-model', instance_count=1, instance_type='ml.c4.xlarge', output_path='s3://sagemaker/transform-output' )
def test_transformer_fails_without_model(): transformer = Transformer( model_name="remote-model", sagemaker_session=test_local_mode.LocalNoS3Session(), instance_type="local", instance_count=1, ) with pytest.raises(ValueError) as error: transformer.transform("empty-data") assert (str( error.value) == "Failed to fetch model information for remote-model. " "Please ensure that the model exists. " "Local instance types require locally created models.")
def transformer(sagemaker_session): return Transformer(MODEL_NAME, INSTANCE_COUNT, INSTANCE_TYPE, output_path=OUTPUT_PATH, sagemaker_session=sagemaker_session, volume_kms_key=KMS_KEY_ID)
def transformer(self, instance_count, instance_type, strategy=None, assemble_with=None, output_path=None, output_kms_key=None, accept=None, env=None, max_concurrent_transforms=None, max_payload=None, tags=None, role=None): """Return a ``Transformer`` that uses a SageMaker Model based on the training job. It reuses the SageMaker Session and base job name used by the Estimator. Args: instance_count (int): Number of EC2 instances to use. instance_type (str): Type of EC2 instance to use, for example, 'ml.c4.xlarge'. strategy (str): The strategy used to decide how to batch records in a single request (default: None). Valid values: 'MULTI_RECORD' and 'SINGLE_RECORD'. assemble_with (str): How the output is assembled (default: None). Valid values: 'Line' or 'None'. output_path (str): S3 location for saving the transform result. If not specified, results are stored to a default bucket. output_kms_key (str): Optional. KMS key ID for encrypting the transform output (default: None). accept (str): The content type accepted by the endpoint deployed during the transform job. env (dict): Environment variables to be set for use during the transform job (default: None). max_concurrent_transforms (int): The maximum number of HTTP requests to be made to each individual transform container at one time. max_payload (int): Maximum size of the payload in a single HTTP request to the container in MB. tags (list[dict]): List of tags for labeling a transform job. If none specified, then the tags used for the training job are used for the transform job. role (str): The ``ExecutionRoleArn`` IAM Role ARN for the ``Model``, which is also used during transform jobs. If not specified, the role from the Estimator will be used. """ self._ensure_latest_training_job() model_name = self.sagemaker_session.create_model_from_job(self.latest_training_job.name, role=role) tags = tags or self.tags return Transformer(model_name, instance_count, instance_type, strategy=strategy, assemble_with=assemble_with, output_path=output_path, output_kms_key=output_kms_key, accept=accept, max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload, env=env, tags=tags, base_transform_job_name=self.base_job_name, sagemaker_session=self.sagemaker_session)
def test_attach(prepare_init_params, transformer, sagemaker_session): sagemaker_session.sagemaker_client.describe_transform_job = Mock(name="describe_transform_job") attached = Transformer.attach(JOB_NAME, sagemaker_session) assert prepare_init_params.called_once assert attached.latest_transform_job.job_name == JOB_NAME assert attached.model_name == MODEL_NAME assert attached.instance_count == INSTANCE_COUNT assert attached.instance_type == INSTANCE_TYPE
def test_attach(prepare_init_params, transformer, sagemaker_session): sagemaker_session.sagemaker_client.describe_transform_job = Mock(name='describe_transform_job') attached = Transformer.attach(JOB_NAME, sagemaker_session) assert prepare_init_params.called_once assert attached.latest_transform_job.job_name == JOB_NAME assert attached.model_name == MODEL_NAME assert attached.instance_count == INSTANCE_COUNT assert attached.instance_type == INSTANCE_TYPE
def test_transformer_init(sagemaker_session): transformer = Transformer( MODEL_NAME, INSTANCE_COUNT, INSTANCE_TYPE, sagemaker_session=sagemaker_session ) assert transformer.model_name == MODEL_NAME assert transformer.instance_count == INSTANCE_COUNT assert transformer.instance_type == INSTANCE_TYPE assert transformer.sagemaker_session == sagemaker_session assert transformer._current_job_name is None assert transformer.latest_transform_job is None assert transformer._reset_output_path is False
def test_transform_step(sagemaker_session): transformer = Transformer( model_name=MODEL_NAME, instance_count=1, instance_type="c4.4xlarge", sagemaker_session=sagemaker_session, ) inputs = TransformInput(data=f"s3://{BUCKET}/transform_manifest") cache_config = CacheConfig(enable_caching=True, expire_after="PT1H") step = TransformStep( name="MyTransformStep", depends_on=["TestStep"], transformer=transformer, display_name="TransformStep", description="TestDescription", inputs=inputs, cache_config=cache_config, ) step.add_depends_on(["SecondTestStep"]) assert step.to_request() == { "Name": "MyTransformStep", "Type": "Transform", "Description": "TestDescription", "DisplayName": "TransformStep", "DependsOn": ["TestStep", "SecondTestStep"], "Arguments": { "ModelName": "gisele", "TransformInput": { "DataSource": { "S3DataSource": { "S3DataType": "S3Prefix", "S3Uri": "s3://my-bucket/transform_manifest", } } }, "TransformOutput": { "S3OutputPath": None }, "TransformResources": { "InstanceCount": 1, "InstanceType": "c4.4xlarge", }, }, "CacheConfig": { "Enabled": True, "ExpireAfter": "PT1H" }, } assert step.properties.TransformJobName.expr == { "Get": "Steps.MyTransformStep.TransformJobName" }
def test_attach_transform_kmeans(sagemaker_session, cpu_instance_type): data_path = os.path.join(DATA_DIR, "one_p_mnist") pickle_args = {} if sys.version_info.major == 2 else {"encoding": "latin1"} # Load the data into memory as numpy arrays train_set_path = os.path.join(data_path, "mnist.pkl.gz") with gzip.open(train_set_path, "rb") as f: train_set, _, _ = pickle.load(f, **pickle_args) kmeans = KMeans( role="SageMakerRole", train_instance_count=1, train_instance_type=cpu_instance_type, k=10, sagemaker_session=sagemaker_session, output_path="s3://{}/".format(sagemaker_session.default_bucket()), ) # set kmeans specific hp kmeans.init_method = "random" kmeans.max_iterators = 1 kmeans.tol = 1 kmeans.num_trials = 1 kmeans.local_init_method = "kmeans++" kmeans.half_life_time_size = 1 kmeans.epochs = 1 records = kmeans.record_set(train_set[0][:100]) job_name = unique_name_from_base("test-kmeans-attach") with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES): kmeans.fit(records, job_name=job_name) transform_input_path = os.path.join(data_path, "transform_input.csv") transform_input_key_prefix = "integ-test-data/one_p_mnist/transform" transform_input = kmeans.sagemaker_session.upload_data( path=transform_input_path, key_prefix=transform_input_key_prefix) transformer = _create_transformer_and_transform_job( kmeans, transform_input, cpu_instance_type) attached_transformer = Transformer.attach( transformer.latest_transform_job.name, sagemaker_session=sagemaker_session) with timeout_and_delete_model_with_transformer( transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES): attached_transformer.wait()
def test_sagemaker_transform_step_successfully(self, m_default_bucket): m_default_bucket.return_value = "sagemaker-bucket-name" with DataJobStack(scope=self.app, id="some-stack", stage="stg") as djs: transformer = Transformer( model_name="some-model", instance_count=1, instance_type="ml.t2.medium", sagemaker_session=self.sagemaker_session, ) transform_step = TransformStep( datajob_stack=djs, name="transform-job", transformer=transformer, data="s3://some-bucket/some-data.csv", ) estimator = SKLearn( entry_point=str( pathlib.Path(current_dir, "resources", "train.py")), train_instance_type="ml.m5.xlarge", role=self.role, framework_version="0.20.0", py_version="py3", sagemaker_session=self.sagemaker_session, ) tuner = HyperparameterTuner( estimator=estimator, hyperparameter_ranges={ "alpha": ContinuousParameter(0.0001, 0.05) }, objective_metric_name="rmse", ) tuner_step = TuningStep( datajob_stack=djs, name="tuning-step", tuner=tuner, data="s3://some-bucket/some-data.csv", ) with StepfunctionsWorkflow(djs, "sequential") as sfn_workflow: transform_step >> tuner_step
def test_attach_transform_kmeans(sagemaker_session): data_path = os.path.join(DATA_DIR, 'one_p_mnist') pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'} # Load the data into memory as numpy arrays train_set_path = os.path.join(data_path, 'mnist.pkl.gz') with gzip.open(train_set_path, 'rb') as f: train_set, _, _ = pickle.load(f, **pickle_args) kmeans = KMeans(role='SageMakerRole', train_instance_count=1, train_instance_type='ml.c4.xlarge', k=10, sagemaker_session=sagemaker_session, output_path='s3://{}/'.format( sagemaker_session.default_bucket())) # set kmeans specific hp kmeans.init_method = 'random' kmeans.max_iterators = 1 kmeans.tol = 1 kmeans.num_trials = 1 kmeans.local_init_method = 'kmeans++' kmeans.half_life_time_size = 1 kmeans.epochs = 1 records = kmeans.record_set(train_set[0][:100]) with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES): kmeans.fit(records) transform_input_path = os.path.join(data_path, 'transform_input.csv') transform_input_key_prefix = 'integ-test-data/one_p_mnist/transform' transform_input = kmeans.sagemaker_session.upload_data( path=transform_input_path, key_prefix=transform_input_key_prefix) transformer = _create_transformer_and_transform_job( kmeans, transform_input) attached_transformer = Transformer.attach( transformer.latest_transform_job.name, sagemaker_session=sagemaker_session) with timeout_and_delete_model_with_transformer( transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES): attached_transformer.wait()
def test_attach_transform_kmeans(sagemaker_session, cpu_instance_type): kmeans = KMeans( role="SageMakerRole", instance_count=1, instance_type=cpu_instance_type, k=10, sagemaker_session=sagemaker_session, output_path="s3://{}/".format(sagemaker_session.default_bucket()), ) # set kmeans specific hp kmeans.init_method = "random" kmeans.max_iterators = 1 kmeans.tol = 1 kmeans.num_trials = 1 kmeans.local_init_method = "kmeans++" kmeans.half_life_time_size = 1 kmeans.epochs = 1 records = kmeans.record_set(datasets.one_p_mnist()[0][:100]) job_name = unique_name_from_base("test-kmeans-attach") with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES): kmeans.fit(records, job_name=job_name) transform_input_path = os.path.join(DATA_DIR, "one_p_mnist", "transform_input.csv") transform_input_key_prefix = "integ-test-data/one_p_mnist/transform" transform_input = kmeans.sagemaker_session.upload_data( path=transform_input_path, key_prefix=transform_input_key_prefix) transformer = _create_transformer_and_transform_job( kmeans, transform_input, cpu_instance_type) attached_transformer = Transformer.attach( transformer.latest_transform_job.name, sagemaker_session=sagemaker_session) with timeout_and_delete_model_with_transformer( transformer, sagemaker_session, minutes=TRANSFORM_DEFAULT_TIMEOUT_MINUTES): attached_transformer.wait()
def test_transformer_init_optional_params(sagemaker_session): strategy = "MultiRecord" assemble_with = "Line" accept = "text/csv" max_concurrent_transforms = 100 max_payload = 100 tags = {"Key": "foo", "Value": "bar"} env = {"FOO": "BAR"} transformer = Transformer( MODEL_NAME, INSTANCE_COUNT, INSTANCE_TYPE, strategy=strategy, assemble_with=assemble_with, output_path=OUTPUT_PATH, output_kms_key=KMS_KEY_ID, accept=accept, max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload, tags=tags, env=env, base_transform_job_name=JOB_NAME, sagemaker_session=sagemaker_session, volume_kms_key=KMS_KEY_ID, ) assert transformer.model_name == MODEL_NAME assert transformer.strategy == strategy assert transformer.env == env assert transformer.output_path == OUTPUT_PATH assert transformer.output_kms_key == KMS_KEY_ID assert transformer.accept == accept assert transformer.assemble_with == assemble_with assert transformer.instance_count == INSTANCE_COUNT assert transformer.instance_type == INSTANCE_TYPE assert transformer.volume_kms_key == KMS_KEY_ID assert transformer.max_concurrent_transforms == max_concurrent_transforms assert transformer.max_payload == max_payload assert transformer.tags == tags assert transformer.base_transform_job_name == JOB_NAME
def test_transform_step(sagemaker_session): transformer = Transformer( model_name=MODEL_NAME, instance_count=1, instance_type="c4.4xlarge", sagemaker_session=sagemaker_session, ) inputs = TransformInput(data=f"s3://{BUCKET}/transform_manifest") step = TransformStep( name="MyTransformStep", transformer=transformer, inputs=inputs, ) assert step.to_request() == { "Name": "MyTransformStep", "Type": "Transform", "Arguments": { "ModelName": "gisele", "TransformInput": { "DataSource": { "S3DataSource": { "S3DataType": "S3Prefix", "S3Uri": "s3://my-bucket/transform_manifest", } } }, "TransformOutput": { "S3OutputPath": None }, "TransformResources": { "InstanceCount": 1, "InstanceType": "c4.4xlarge", }, }, } assert step.properties.TransformJobName.expr == { "Get": "Steps.MyTransformStep.TransformJobName" }
def test_attach_transform_kmeans(sagemaker_session): data_path = os.path.join(DATA_DIR, 'one_p_mnist') pickle_args = {} if sys.version_info.major == 2 else {'encoding': 'latin1'} # Load the data into memory as numpy arrays train_set_path = os.path.join(data_path, 'mnist.pkl.gz') with gzip.open(train_set_path, 'rb') as f: train_set, _, _ = pickle.load(f, **pickle_args) kmeans = KMeans(role='SageMakerRole', train_instance_count=1, train_instance_type='ml.c4.xlarge', k=10, sagemaker_session=sagemaker_session, output_path='s3://{}/'.format(sagemaker_session.default_bucket())) # set kmeans specific hp kmeans.init_method = 'random' kmeans.max_iterators = 1 kmeans.tol = 1 kmeans.num_trials = 1 kmeans.local_init_method = 'kmeans++' kmeans.half_life_time_size = 1 kmeans.epochs = 1 records = kmeans.record_set(train_set[0][:100]) with timeout(minutes=TRAINING_DEFAULT_TIMEOUT_MINUTES): kmeans.fit(records) transform_input_path = os.path.join(data_path, 'transform_input.csv') transform_input_key_prefix = 'integ-test-data/one_p_mnist/transform' transform_input = kmeans.sagemaker_session.upload_data(path=transform_input_path, key_prefix=transform_input_key_prefix) transformer = _create_transformer_and_transform_job(kmeans, transform_input) attached_transformer = Transformer.attach(transformer.latest_transform_job.name, sagemaker_session=sagemaker_session) attached_transformer.wait()
def transformer( self, instance_count, instance_type, strategy=None, assemble_with=None, output_path=None, output_kms_key=None, accept=None, env=None, max_concurrent_transforms=None, max_payload=None, tags=None, role=None, volume_kms_key=None, ): """Return a ``Transformer`` that uses a SageMaker Model based on the training job. It reuses the SageMaker Session and base job name used by the Estimator. Args: instance_count (int): Number of EC2 instances to use. instance_type (str): Type of EC2 instance to use, for example, 'ml.c4.xlarge'. strategy (str): The strategy used to decide how to batch records in a single request (default: None). Valid values: 'MultiRecord' and 'SingleRecord'. assemble_with (str): How the output is assembled (default: None). Valid values: 'Line' or 'None'. output_path (str): S3 location for saving the transform result. If not specified, results are stored to a default bucket. output_kms_key (str): Optional. KMS key ID for encrypting the transform output (default: None). accept (str): The accept header passed by the client to the inference endpoint. If it is supported by the endpoint, it will be the format of the batch transform output. env (dict): Environment variables to be set for use during the transform job (default: None). max_concurrent_transforms (int): The maximum number of HTTP requests to be made to each individual transform container at one time. max_payload (int): Maximum size of the payload in a single HTTP request to the container in MB. tags (list[dict]): List of tags for labeling a transform job. If none specified, then the tags used for the training job are used for the transform job. role (str): The ``ExecutionRoleArn`` IAM Role ARN for the ``Model``, which is also used during transform jobs. If not specified, the role from the Estimator will be used. volume_kms_key (str): Optional. KMS key ID for encrypting the volume attached to the ML compute instance (default: None). """ role = role or self.role if self.latest_training_job is not None: model = self.create_model(role=role) model._create_sagemaker_model() model_name = model.name transform_env = {} if env is not None: transform_env = model.env.copy() transform_env.update(env) if self._is_marketplace(): transform_env = None tags = tags or self.tags else: raise RuntimeError( "No finished training job found associated with this estimator" ) return Transformer( model_name, instance_count, instance_type, strategy=strategy, assemble_with=assemble_with, output_path=output_path, output_kms_key=output_kms_key, accept=accept, max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload, env=transform_env, tags=tags, base_transform_job_name=self.base_job_name, volume_kms_key=volume_kms_key, sagemaker_session=self.sagemaker_session, )
def transformer( self, instance_count, instance_type, strategy=None, assemble_with=None, output_path=None, output_kms_key=None, accept=None, env=None, max_concurrent_transforms=None, max_payload=None, tags=None, role=None, model_server_workers=None, volume_kms_key=None, endpoint_type=None, entry_point=None, vpc_config_override=VPC_CONFIG_DEFAULT, ): """Return a ``Transformer`` that uses a SageMaker Model based on the training job. It reuses the SageMaker Session and base job name used by the Estimator. Args: instance_count (int): Number of EC2 instances to use. instance_type (str): Type of EC2 instance to use, for example, 'ml.c4.xlarge'. strategy (str): The strategy used to decide how to batch records in a single request (default: None). Valid values: 'MULTI_RECORD' and 'SINGLE_RECORD'. assemble_with (str): How the output is assembled (default: None). Valid values: 'Line' or 'None'. output_path (str): S3 location for saving the transform result. If not specified, results are stored to a default bucket. output_kms_key (str): Optional. KMS key ID for encrypting the transform output (default: None). accept (str): The accept header passed by the client to the inference endpoint. If it is supported by the endpoint, it will be the format of the batch transform output. env (dict): Environment variables to be set for use during the transform job (default: None). max_concurrent_transforms (int): The maximum number of HTTP requests to be made to each individual transform container at one time. max_payload (int): Maximum size of the payload in a single HTTP request to the container in MB. tags (list[dict]): List of tags for labeling a transform job. If none specified, then the tags used for the training job are used for the transform job. role (str): The ``ExecutionRoleArn`` IAM Role ARN for the ``Model``, which is also used during transform jobs. If not specified, the role from the Estimator will be used. model_server_workers (int): Optional. The number of worker processes used by the inference server. If None, server will use one worker per vCPU. volume_kms_key (str): Optional. KMS key ID for encrypting the volume attached to the ML compute instance (default: None). endpoint_type (str): Optional. Selects the software stack used by the inference server. If not specified, the model will be configured to use the default SageMaker model server. If 'tensorflow-serving', the model will be configured to use the SageMaker Tensorflow Serving container. entry_point (str): Path (absolute or relative) to the local Python source file which should be executed as the entry point to training. If not specified and ``endpoint_type`` is 'tensorflow-serving', no entry point is used. If ``endpoint_type`` is also ``None``, then the training entry point is used. vpc_config_override (dict[str, list[str]]): Optional override for the VpcConfig set on the model. Default: use subnets and security groups from this Estimator. * 'Subnets' (list[str]): List of subnet ids. * 'SecurityGroupIds' (list[str]): List of security group ids. """ role = role or self.role if self.latest_training_job is None: logging.warning( "No finished training job found associated with this estimator. Please make sure " "this estimator is only used for building workflow config") return Transformer( self._current_job_name, instance_count, instance_type, strategy=strategy, assemble_with=assemble_with, output_path=output_path, output_kms_key=output_kms_key, accept=accept, max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload, env=env or {}, tags=tags, base_transform_job_name=self.base_job_name, volume_kms_key=volume_kms_key, sagemaker_session=self.sagemaker_session, ) model = self.create_model( model_server_workers=model_server_workers, role=role, vpc_config_override=vpc_config_override, endpoint_type=endpoint_type, entry_point=entry_point, ) return model.transformer( instance_count, instance_type, strategy=strategy, assemble_with=assemble_with, output_path=output_path, output_kms_key=output_kms_key, accept=accept, env=env, max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload, tags=tags, volume_kms_key=volume_kms_key, )
def __init__( self, name: str, estimator: EstimatorBase, model_data, model_inputs, instance_count, instance_type, transform_inputs, # model arguments image_uri=None, predictor_cls=None, env=None, # transformer arguments strategy=None, assemble_with=None, output_path=None, output_kms_key=None, accept=None, max_concurrent_transforms=None, max_payload=None, tags=None, volume_kms_key=None, depends_on: List[str] = None, **kwargs, ): """Construct steps required for a Transformer step collection: An estimator-centric step collection. It models what happens in workflows when invoking the `transform()` method on an estimator instance: First, if custom model artifacts are required, a `_RepackModelStep` is included. Second, a `CreateModelStep` with the model data passed in from a training step or other training job output. Finally, a `TransformerStep`. If repacking the model artifacts is not necessary, only the CreateModelStep and TransformerStep are in the step collection. Args: name (str): The name of the Transform Step. estimator: The estimator instance. instance_count (int): The number of EC2 instances to use. instance_type (str): The type of EC2 instance to use. strategy (str): The strategy used to decide how to batch records in a single request (default: None). Valid values: 'MultiRecord' and 'SingleRecord'. assemble_with (str): How the output is assembled (default: None). Valid values: 'Line' or 'None'. output_path (str): The S3 location for saving the transform result. If not specified, results are stored to a default bucket. output_kms_key (str): Optional. A KMS key ID for encrypting the transform output (default: None). accept (str): The accept header passed by the client to the inference endpoint. If it is supported by the endpoint, it will be the format of the batch transform output. env (dict): The Environment variables to be set for use during the transform job (default: None). depends_on (List[str]): The list of step names the first step in the collection depends on """ steps = [] if "entry_point" in kwargs: entry_point = kwargs["entry_point"] source_dir = kwargs.get("source_dir") dependencies = kwargs.get("dependencies") repack_model_step = _RepackModelStep( name=f"{name}RepackModel", depends_on=depends_on, estimator=estimator, model_data=model_data, entry_point=entry_point, source_dir=source_dir, dependencies=dependencies, ) steps.append(repack_model_step) model_data = repack_model_step.properties.ModelArtifacts.S3ModelArtifacts def predict_wrapper(endpoint, session): return Predictor(endpoint, session) predictor_cls = predictor_cls or predict_wrapper model = Model( image_uri=image_uri or estimator.training_image_uri(), model_data=model_data, predictor_cls=predictor_cls, vpc_config=None, sagemaker_session=estimator.sagemaker_session, role=estimator.role, **kwargs, ) model_step = CreateModelStep( name=f"{name}CreateModelStep", model=model, inputs=model_inputs, ) if "entry_point" not in kwargs and depends_on: # if the CreateModelStep is the first step in the collection model_step.add_depends_on(depends_on) steps.append(model_step) transformer = Transformer( model_name=model_step.properties.ModelName, instance_count=instance_count, instance_type=instance_type, strategy=strategy, assemble_with=assemble_with, output_path=output_path, output_kms_key=output_kms_key, accept=accept, max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload, env=env, tags=tags, base_transform_job_name=name, volume_kms_key=volume_kms_key, sagemaker_session=estimator.sagemaker_session, ) transform_step = TransformStep( name=f"{name}TransformStep", transformer=transformer, inputs=transform_inputs, ) steps.append(transform_step) self.steps = steps
def transformer( self, instance_count, instance_type, strategy=None, assemble_with=None, output_path=None, output_kms_key=None, accept=None, env=None, max_concurrent_transforms=None, max_payload=None, tags=None, role=None, volume_kms_key=None, entry_point=None, vpc_config_override=VPC_CONFIG_DEFAULT, enable_network_isolation=None, model_name=None, ): """Return a ``Transformer`` that uses a SageMaker Model based on the training job. It reuses the SageMaker Session and base job name used by the Estimator. Args: instance_count (int): Number of EC2 instances to use. instance_type (str): Type of EC2 instance to use, for example, 'ml.c4.xlarge'. strategy (str): The strategy used to decide how to batch records in a single request (default: None). Valid values: 'MultiRecord' and 'SingleRecord'. assemble_with (str): How the output is assembled (default: None). Valid values: 'Line' or 'None'. output_path (str): S3 location for saving the transform result. If not specified, results are stored to a default bucket. output_kms_key (str): Optional. KMS key ID for encrypting the transform output (default: None). accept (str): The accept header passed by the client to the inference endpoint. If it is supported by the endpoint, it will be the format of the batch transform output. env (dict): Environment variables to be set for use during the transform job (default: None). max_concurrent_transforms (int): The maximum number of HTTP requests to be made to each individual transform container at one time. max_payload (int): Maximum size of the payload in a single HTTP request to the container in MB. tags (list[dict]): List of tags for labeling a transform job. If none specified, then the tags used for the training job are used for the transform job. role (str): The IAM Role ARN for the ``TensorFlowModel``, which is also used during transform jobs. If not specified, the role from the Estimator is used. volume_kms_key (str): Optional. KMS key ID for encrypting the volume attached to the ML compute instance (default: None). entry_point (str): Path (absolute or relative) to the local Python source file which should be executed as the entry point to training. If ``source_dir`` is specified, then ``entry_point`` must point to a file located at the root of ``source_dir``. If not specified and ``endpoint_type`` is 'tensorflow-serving', no entry point is used. If ``endpoint_type`` is also ``None``, then the training entry point is used. vpc_config_override (dict[str, list[str]]): Optional override for the VpcConfig set on the model. Default: use subnets and security groups from this Estimator. * 'Subnets' (list[str]): List of subnet ids. * 'SecurityGroupIds' (list[str]): List of security group ids. enable_network_isolation (bool): Specifies whether container will run in network isolation mode. Network isolation mode restricts the container access to outside networks (such as the internet). The container does not make any inbound or outbound network calls. If True, a channel named "code" will be created for any user entry script for inference. Also known as Internet-free mode. If not specified, this setting is taken from the estimator's current configuration. model_name (str): Name to use for creating an Amazon SageMaker model. If not specified, the estimator generates a default job name based on the training image name and current timestamp. """ role = role or self.role model_name = self._get_or_create_name(model_name) if self.latest_training_job is None: logger.warning( "No finished training job found associated with this estimator. Please make sure " "this estimator is only used for building workflow config") return Transformer( model_name, instance_count, instance_type, strategy=strategy, assemble_with=assemble_with, output_path=output_path, output_kms_key=output_kms_key, accept=accept, max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload, env=env or {}, tags=tags, base_transform_job_name=self.base_job_name, volume_kms_key=volume_kms_key, sagemaker_session=self.sagemaker_session, ) if enable_network_isolation is None: enable_network_isolation = self.enable_network_isolation() model = self.create_model( role=role, vpc_config_override=vpc_config_override, entry_point=entry_point, enable_network_isolation=enable_network_isolation, name=model_name, ) return model.transformer( instance_count, instance_type, strategy=strategy, assemble_with=assemble_with, output_path=output_path, output_kms_key=output_kms_key, accept=accept, env=env, max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload, tags=tags, volume_kms_key=volume_kms_key, )
def test_start_new(prepare_data_processing, load_config, sagemaker_session): input_config = "input" output_config = "output" resource_config = "resource" load_config.return_value = { "input_config": input_config, "output_config": output_config, "resource_config": resource_config, } strategy = "MultiRecord" max_concurrent_transforms = 100 max_payload = 100 tags = {"Key": "foo", "Value": "bar"} env = {"FOO": "BAR"} transformer = Transformer( MODEL_NAME, INSTANCE_COUNT, INSTANCE_TYPE, strategy=strategy, output_path=OUTPUT_PATH, max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload, tags=tags, env=env, sagemaker_session=sagemaker_session, ) transformer._current_job_name = JOB_NAME content_type = "text/csv" compression_type = "Gzip" split_type = "Line" io_filter = "$" join_source = "Input" model_client_config = { "InvocationsTimeoutInSeconds": 60, "InvocationsMaxRetries": 2 } job = _TransformJob.start_new( transformer=transformer, data=DATA, data_type=S3_DATA_TYPE, content_type=content_type, compression_type=compression_type, split_type=split_type, input_filter=io_filter, output_filter=io_filter, join_source=join_source, experiment_config={"ExperimentName": "exp"}, model_client_config=model_client_config, ) assert job.sagemaker_session == sagemaker_session assert job.job_name == JOB_NAME load_config.assert_called_with(DATA, S3_DATA_TYPE, content_type, compression_type, split_type, transformer) prepare_data_processing.assert_called_with(io_filter, io_filter, join_source) sagemaker_session.transform.assert_called_with( job_name=JOB_NAME, model_name=MODEL_NAME, strategy=strategy, max_concurrent_transforms=max_concurrent_transforms, max_payload=max_payload, env=env, input_config=input_config, output_config=output_config, resource_config=resource_config, experiment_config={"ExperimentName": "exp"}, model_client_config=model_client_config, tags=tags, data_processing=prepare_data_processing.return_value, )
xgb_estimator = Estimator(image_name=xgb_container, role=role, sagemaker_session=sagemaker.session.Session(sess), **config["train_model"]["estimator_config"]) # train_config specifies SageMaker training configuration train_data = create_s3_input(config['train_model']['inputs']['train']) validation_data = create_s3_input( config['train_model']['inputs']['validation']) data_channels = {'train': train_data, 'validation': validation_data} train_config = training_config(estimator=xgb_estimator, inputs=data_channels) # Batch inference xgb_transformer = Transformer( model_name=config['batch_transform']['model_name'], sagemaker_session=sagemaker.session.Session(sess), **config['batch_transform']['transformer_config']) transform_config = transform_config( transformer=xgb_transformer, **config['batch_transform']['transform_config']) # ============================================================================= # define airflow DAG and tasks # ============================================================================= # define airflow DAG args = {'owner': 'airflow', 'start_date': airflow.utils.dates.days_ago(2)} dag = DAG( 'sagemaker-ml-pipeline', default_args=args,
def test_end_to_end_pipeline_successful_execution( sagemaker_session, region_name, role, pipeline_name, wait=False ): model_package_group_name = f"{pipeline_name}ModelPackageGroup" data_path = os.path.join(DATA_DIR, "workflow") default_bucket = sagemaker_session.default_bucket() # download the input data local_input_path = os.path.join(data_path, "abalone-dataset.csv") s3 = sagemaker_session.boto_session.resource("s3") s3.Bucket(f"sagemaker-servicecatalog-seedcode-{region_name}").download_file( "dataset/abalone-dataset.csv", local_input_path ) # # upload the input data to our bucket base_uri = f"s3://{default_bucket}/{pipeline_name}" with open(local_input_path) as data: body = data.read() input_data_uri = S3Uploader.upload_string_as_file_body( body=body, desired_s3_uri=f"{base_uri}/abalone-dataset.csv", sagemaker_session=sagemaker_session, ) # download batch transform data local_batch_path = os.path.join(data_path, "abalone-dataset-batch") s3.Bucket(f"sagemaker-servicecatalog-seedcode-{region_name}").download_file( "dataset/abalone-dataset-batch", local_batch_path ) # upload the batch transform data with open(local_batch_path) as data: body = data.read() batch_data_uri = S3Uploader.upload_string_as_file_body( body=body, desired_s3_uri=f"{base_uri}/abalone-dataset-batch", sagemaker_session=sagemaker_session, ) # define parameters processing_instance_count = ParameterInteger(name="ProcessingInstanceCount", default_value=1) processing_instance_type = ParameterString( name="ProcessingInstanceType", default_value="ml.m5.xlarge" ) training_instance_type = ParameterString( name="TrainingInstanceType", default_value="ml.m5.xlarge" ) model_approval_status = ParameterString(name="ModelApprovalStatus", default_value="Approved") input_data = ParameterString( name="InputData", default_value=input_data_uri, ) batch_data = ParameterString( name="BatchData", default_value=batch_data_uri, ) # define processing step framework_version = "0.23-1" sklearn_processor = SKLearnProcessor( framework_version=framework_version, instance_type=processing_instance_type, instance_count=processing_instance_count, base_job_name=f"{pipeline_name}-process", role=role, sagemaker_session=sagemaker_session, ) step_process = ProcessingStep( name="AbaloneProcess", processor=sklearn_processor, inputs=[ ProcessingInput(source=input_data, destination="/opt/ml/processing/input"), ], outputs=[ ProcessingOutput(output_name="train", source="/opt/ml/processing/train"), ProcessingOutput(output_name="validation", source="/opt/ml/processing/validation"), ProcessingOutput(output_name="test", source="/opt/ml/processing/test"), ], code=os.path.join(data_path, "abalone/preprocessing.py"), ) # define training step model_path = f"s3://{default_bucket}/{pipeline_name}Train" image_uri = image_uris.retrieve( framework="xgboost", region=region_name, version="1.0-1", py_version="py3", instance_type=training_instance_type, ) xgb_train = Estimator( image_uri=image_uri, instance_type=training_instance_type, instance_count=1, output_path=model_path, role=role, sagemaker_session=sagemaker_session, ) xgb_train.set_hyperparameters( objective="reg:linear", num_round=50, max_depth=5, eta=0.2, gamma=4, min_child_weight=6, subsample=0.7, silent=0, ) step_train = TrainingStep( name="AbaloneTrain", estimator=xgb_train, inputs={ "train": TrainingInput( s3_data=step_process.properties.ProcessingOutputConfig.Outputs[ "train" ].S3Output.S3Uri, content_type="text/csv", ), "validation": TrainingInput( s3_data=step_process.properties.ProcessingOutputConfig.Outputs[ "validation" ].S3Output.S3Uri, content_type="text/csv", ), }, ) # define evaluation step script_eval = ScriptProcessor( image_uri=image_uri, command=["python3"], instance_type=processing_instance_type, instance_count=1, base_job_name=f"{pipeline_name}-eval", role=role, sagemaker_session=sagemaker_session, ) evaluation_report = PropertyFile( name="EvaluationReport", output_name="evaluation", path="evaluation.json" ) step_eval = ProcessingStep( name="AbaloneEval", processor=script_eval, inputs=[ ProcessingInput( source=step_train.properties.ModelArtifacts.S3ModelArtifacts, destination="/opt/ml/processing/model", ), ProcessingInput( source=step_process.properties.ProcessingOutputConfig.Outputs[ "test" ].S3Output.S3Uri, destination="/opt/ml/processing/test", ), ], outputs=[ ProcessingOutput(output_name="evaluation", source="/opt/ml/processing/evaluation"), ], code=os.path.join(data_path, "abalone/evaluation.py"), property_files=[evaluation_report], ) # define create model step model = Model( image_uri=image_uri, model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts, sagemaker_session=sagemaker_session, role=role, ) inputs = CreateModelInput( instance_type="ml.m5.large", accelerator_type="ml.eia1.medium", ) step_create_model = CreateModelStep( name="AbaloneCreateModel", model=model, inputs=inputs, ) # define transform step transformer = Transformer( model_name=step_create_model.properties.ModelName, instance_type="ml.m5.xlarge", instance_count=1, output_path=f"s3://{default_bucket}/{pipeline_name}Transform", sagemaker_session=sagemaker_session, ) step_transform = TransformStep( name="AbaloneTransform", transformer=transformer, inputs=TransformInput(data=batch_data), ) # define register model step model_metrics = ModelMetrics( model_statistics=MetricsSource( s3_uri="{}/evaluation.json".format( step_eval.arguments["ProcessingOutputConfig"]["Outputs"][0]["S3Output"]["S3Uri"] ), content_type="application/json", ) ) step_register = RegisterModel( name="AbaloneRegisterModel", estimator=xgb_train, model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts, content_types=["text/csv"], response_types=["text/csv"], inference_instances=["ml.t2.medium", "ml.m5.xlarge"], transform_instances=["ml.m5.xlarge"], model_package_group_name=model_package_group_name, approval_status=model_approval_status, model_metrics=model_metrics, ) # define condition step cond_lte = ConditionLessThanOrEqualTo( left=JsonGet( step_name=step_eval.name, property_file=evaluation_report, json_path="regression_metrics.mse.value", ), right=20.0, ) step_cond = ConditionStep( name="AbaloneMSECond", conditions=[cond_lte], if_steps=[step_register, step_create_model, step_transform], else_steps=[], ) # define pipeline pipeline = Pipeline( name=pipeline_name, parameters=[ processing_instance_type, processing_instance_count, training_instance_type, model_approval_status, input_data, batch_data, ], steps=[step_process, step_train, step_eval, step_cond], sagemaker_session=sagemaker_session, ) pipeline.create(role) execution = pipeline.start() execution_arn = execution.arn if wait: execution.wait() return execution_arn
def test_delete_model(sagemaker_session): transformer = Transformer( MODEL_NAME, INSTANCE_COUNT, INSTANCE_TYPE, sagemaker_session=sagemaker_session ) transformer.delete_model() sagemaker_session.delete_model.assert_called_with(MODEL_NAME)
def transformer(sagemaker_session): return Transformer(MODEL_NAME, INSTANCE_COUNT, INSTANCE_TYPE, output_path=OUTPUT_PATH, sagemaker_session=sagemaker_session)