def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url, assume_role_arn=args.assume_role) logging.info('Submitting Batch Transformation request to SageMaker...') batch_job_name = _utils.create_transform_job(client, vars(args)) def signal_term_handler(signalNumber, frame): _utils.stop_transform_job(client, batch_job_name) logging.info( f"Transform job: {batch_job_name} request submitted to Stop") signal.signal(signal.SIGTERM, signal_term_handler) logging.info('Batch Job request submitted. Waiting for completion...') try: _utils.wait_for_transform_job(client, batch_job_name) except: raise finally: cw_client = _utils.get_cloudwatch_client( args.region, assume_role_arn=args.assume_role) _utils.print_logs_for_job(cw_client, '/aws/sagemaker/TransformJobs', batch_job_name) _utils.write_output(args.output_location_output_path, args.output_location) logging.info('Batch Transformation creation completed.')
def test_wait_for_transform_job_creation(self): mock_client = MagicMock() mock_client.describe_transform_job.side_effect = [ {"TransformJobStatus": "InProgress"}, {"TransformJobStatus": "Completed"}, {"TransformJobStatus": "Should not be called"} ] _utils.wait_for_transform_job(mock_client, 'test-batch', 0) self.assertEqual(mock_client.describe_transform_job.call_count, 2)
def test_wait_for_failed_job(self): mock_client = MagicMock() mock_client.describe_transform_job.side_effect = [ {"TransformJobStatus": "InProgress"}, {"TransformJobStatus": "Failed", "FailureReason": "SYSTEM FAILURE"}, {"TransformJobStatus": "Should not be called"} ] with self.assertRaises(Exception): _utils.wait_for_transform_job(mock_client, 'test-batch', 0) self.assertEqual(mock_client.describe_transform_job.call_count, 2)
def main(argv=None): parser = create_parser() args = parser.parse_args() logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting Batch Transformation request to SageMaker...') batch_job_name = _utils.create_transform_job(client, vars(args)) logging.info('Batch Job request submitted. Waiting for completion...') _utils.wait_for_transform_job(client, batch_job_name) Path(args.output_location_file).parent.mkdir(parents=True, exist_ok=True) Path(args.output_location_file).write_text(unicode(args.output_location)) logging.info('Batch Transformation creation completed.')
def main(argv=None): parser = argparse.ArgumentParser( description='SageMaker Batch Transformation Job') parser.add_argument('--region', type=str, help='The region where the cluster launches.') parser.add_argument( '--model_name', type=str, help='The name of the model that you want to use for the transform job.' ) parser.add_argument( '--input_location', type=str, help= 'The S3 location of the data source that is associated with a channel.' ) parser.add_argument( '--output_location', type=str, help= 'The Amazon S3 path where you want Amazon SageMaker to store the results of the transform job.' ) parser.add_argument( '--output_location_file', type=str, help= 'File path where the program will write the Amazon S3 URI of the transform job results.' ) args = parser.parse_args() logging.getLogger().setLevel(logging.INFO) client = _utils.get_client(args.region) logging.info('Submitting Batch Transformation request to SageMaker...') batch_job_name = _utils.create_transform_job(client, args.model_name, args.input_location, args.output_location) logging.info('Batch Job request submitted. Waiting for completion...') _utils.wait_for_transform_job(client, batch_job_name) _utils.print_tranformation_job_result(args.output_location) Path(args.output_location_file).parent.mkdir(parents=True, exist_ok=True) Path(args.output_location_file).write_text(unicode(args.output_location)) logging.info('Batch Transformation creation completed.')
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting Batch Transformation request to SageMaker...') batch_job_name = _utils.create_transform_job(client, vars(args)) logging.info('Batch Job request submitted. Waiting for completion...') try: _utils.wait_for_transform_job(client, batch_job_name) except: raise finally: cw_client = _utils.get_cloudwatch_client(args.region) _utils.print_logs_for_job(cw_client, '/aws/sagemaker/TransformJobs', batch_job_name) _utils.write_output(args.output_location_output_path, args.output_location) logging.info('Batch Transformation creation completed.')
def main(argv=None): parser = argparse.ArgumentParser( description='SageMaker Batch Transformation Job') parser.add_argument('--region', type=str.strip, required=True, help='The region where the cluster launches.') parser.add_argument('--job_name', type=str.strip, required=False, help='The name of the transform job.', default='') parser.add_argument( '--model_name', type=str.strip, required=True, help='The name of the model that you want to use for the transform job.' ) parser.add_argument( '--max_concurrent', type=_utils.str_to_int, required=False, help= 'The maximum number of parallel requests that can be sent to each instance in a transform job.', default='0') parser.add_argument('--max_payload', type=_utils.str_to_int, required=False, help='The maximum allowed size of the payload, in MB.', default='6') parser.add_argument( '--batch_strategy', choices=['MultiRecord', 'SingleRecord', ''], type=str.strip, required=False, help= 'The number of records to include in a mini-batch for an HTTP inference request.', default='') parser.add_argument( '--environment', type=_utils.str_to_json_dict, required=False, help= 'The dictionary of the environment variables to set in the Docker container. Up to 16 key-value entries in the map.', default='{}') parser.add_argument( '--input_location', type=str.strip, required=True, help= 'The S3 location of the data source that is associated with a channel.' ) parser.add_argument( '--data_type', choices=['ManifestFile', 'S3Prefix', 'AugmentedManifestFile', ''], type=str.strip, required=False, help= 'Data type of the input. Can be ManifestFile, S3Prefix, or AugmentedManifestFile.', default='S3Prefix') parser.add_argument( '--content_type', type=str.strip, required=False, help= 'The multipurpose internet mail extension (MIME) type of the data.', default='') parser.add_argument( '--split_type', choices=['None', 'Line', 'RecordIO', 'TFRecord', ''], type=str.strip, required=False, help= 'The method to use to split the transform job data files into smaller batches.', default='None') parser.add_argument( '--compression_type', choices=['None', 'Gzip', ''], type=str.strip, required=False, help= 'If the transform data is compressed, the specification of the compression type.', default='None') parser.add_argument( '--output_location', type=str.strip, required=True, help= 'The Amazon S3 path where you want Amazon SageMaker to store the results of the transform job.' ) parser.add_argument('--accept', type=str.strip, required=False, help='The MIME type used to specify the output data.') parser.add_argument( '--assemble_with', choices=['None', 'Line', ''], type=str.strip, required=False, help= 'Defines how to assemble the results of the transform job as a single S3 object. Either None or Line.' ) parser.add_argument( '--output_encryption_key', type=str.strip, required=False, help= 'The AWS KMS key that Amazon SageMaker uses to encrypt the model artifacts.', default='') parser.add_argument( '--input_filter', type=str.strip, required=False, help= 'A JSONPath expression used to select a portion of the input data to pass to the algorithm.', default='') parser.add_argument( '--output_filter', type=str.strip, required=False, help= 'A JSONPath expression used to select a portion of the joined dataset to save in the output file for a batch transform job.', default='') parser.add_argument( '--join_source', choices=['None', 'Input', ''], type=str.strip, required=False, help= 'Specifies the source of the data to join with the transformed data.', default='None') parser.add_argument( '--instance_type', choices=[ 'ml.m4.xlarge', 'ml.m4.2xlarge', 'ml.m4.4xlarge', 'ml.m4.10xlarge', 'ml.m4.16xlarge', 'ml.m5.large', 'ml.m5.xlarge', 'ml.m5.2xlarge', 'ml.m5.4xlarge', 'ml.m5.12xlarge', 'ml.m5.24xlarge', 'ml.c4.xlarge', 'ml.c4.2xlarge', 'ml.c4.4xlarge', 'ml.c4.8xlarge', 'ml.p2.xlarge', 'ml.p2.8xlarge', 'ml.p2.16xlarge', 'ml.p3.2xlarge', 'ml.p3.8xlarge', 'ml.p3.16xlarge', 'ml.c5.xlarge', 'ml.c5.2xlarge', 'ml.c5.4xlarge', 'ml.c5.9xlarge', 'ml.c5.18xlarge' ], type=str.strip, required=True, help='The ML compute instance type for the transform job.', default='ml.m4.xlarge') parser.add_argument( '--instance_count', type=_utils.str_to_int, required=False, help='The number of ML compute instances to use in the transform job.') parser.add_argument( '--resource_encryption_key', type=str.strip, required=False, help= 'The AWS KMS key that Amazon SageMaker uses to encrypt data on the storage volume attached to the ML compute instance(s).', default='') parser.add_argument( '--tags', type=_utils.str_to_json_dict, required=False, help='An array of key-value pairs, to categorize AWS resources.', default='{}') parser.add_argument( '--output_location_file', type=str.strip, required=True, help= 'File path where the program will write the Amazon S3 URI of the transform job results.' ) args = parser.parse_args() logging.getLogger().setLevel(logging.INFO) client = _utils.get_client(args.region) logging.info('Submitting Batch Transformation request to SageMaker...') batch_job_name = _utils.create_transform_job(client, vars(args)) logging.info('Batch Job request submitted. Waiting for completion...') _utils.wait_for_transform_job(client, batch_job_name) Path(args.output_location_file).parent.mkdir(parents=True, exist_ok=True) Path(args.output_location_file).write_text(unicode(args.output_location)) logging.info('Batch Transformation creation completed.')