def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting Training Job to SageMaker...') job_name = _utils.create_training_job(client, vars(args)) def signal_term_handler(signalNumber, frame): _utils.stop_training_job(client, job_name) logging.info(f"Training Job: {job_name} request submitted to Stop") signal.signal(signal.SIGTERM, signal_term_handler) logging.info('Job request submitted. Waiting for completion...') try: _utils.wait_for_training_job(client, job_name) _utils.wait_for_debug_rules(client, job_name) except: raise finally: cw_client = _utils.get_cloudwatch_client(args.region) _utils.print_logs_for_job(cw_client, '/aws/sagemaker/TrainingJobs', job_name) image = _utils.get_image_from_job(client, job_name) model_artifact_url = _utils.get_model_artifacts_from_job(client, job_name) logging.info('Get model artifacts %s from training job %s.', model_artifact_url, job_name) _utils.write_output(args.model_artifact_url_output_path, model_artifact_url) _utils.write_output(args.job_name_output_path, job_name) _utils.write_output(args.training_image_output_path, image) logging.info('Job completed.')
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region) logging.info( 'Submitting HyperParameter Tuning Job request to SageMaker...') hpo_job_name = _utils.create_hyperparameter_tuning_job(client, vars(args)) logging.info( 'HyperParameter Tuning Job request submitted. Waiting for completion...' ) _utils.wait_for_hyperparameter_training_job(client, hpo_job_name) best_job, best_hyperparameters = _utils.get_best_training_job_and_hyperparameters( client, hpo_job_name) model_artifact_url = _utils.get_model_artifacts_from_job(client, best_job) image = _utils.get_image_from_job(client, best_job) logging.info('HyperParameter Tuning Job completed.') with open('/tmp/hpo_job_name.txt', 'w') as f: f.write(hpo_job_name) with open('/tmp/best_job_name.txt', 'w') as f: f.write(best_job) with open('/tmp/best_hyperparameters.txt', 'w') as f: f.write(json.dumps(best_hyperparameters)) with open('/tmp/model_artifact_url.txt', 'w') as f: f.write(model_artifact_url) with open('/tmp/training_image.txt', 'w') as f: f.write(image)
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting Training Job to SageMaker...') job_name = _utils.create_training_job(client, vars(args)) logging.info('Job request submitted. Waiting for completion...') _utils.wait_for_training_job(client, job_name) image = _utils.get_image_from_job(client, job_name) model_artifact_url = _utils.get_model_artifacts_from_job(client, job_name) logging.info('Get model artifacts %s from training job %s.', model_artifact_url, job_name) with open('/tmp/model_artifact_url.txt', 'w') as f: f.write(model_artifact_url) with open('/tmp/job_name.txt', 'w') as f: f.write(job_name) with open('/tmp/training_image.txt', 'w') as f: f.write(image) logging.info('Job completed.')
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region) logging.info( 'Submitting HyperParameter Tuning Job request to SageMaker...') hpo_job_name = _utils.create_hyperparameter_tuning_job(client, vars(args)) logging.info( 'HyperParameter Tuning Job request submitted. Waiting for completion...' ) _utils.wait_for_hyperparameter_training_job(client, hpo_job_name) best_job, best_hyperparameters = _utils.get_best_training_job_and_hyperparameters( client, hpo_job_name) model_artifact_url = _utils.get_model_artifacts_from_job(client, best_job) image = _utils.get_image_from_job(client, best_job) logging.info('HyperParameter Tuning Job completed.') _utils.write_output(args.hpo_job_name_output_path, hpo_job_name) _utils.write_output(args.model_artifact_url_output_path, model_artifact_url) _utils.write_output(args.best_job_name_output_path, best_job) _utils.write_output(args.best_hyperparameters_output_path, best_hyperparameters, json_encode=True) _utils.write_output(args.training_image_output_path, image)
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url, assume_role_arn=args.assume_role) logging.info('Submitting Batch Transformation request to SageMaker...') batch_job_name = _utils.create_transform_job(client, vars(args)) def signal_term_handler(signalNumber, frame): _utils.stop_transform_job(client, batch_job_name) logging.info( f"Transform job: {batch_job_name} request submitted to Stop") signal.signal(signal.SIGTERM, signal_term_handler) logging.info('Batch Job request submitted. Waiting for completion...') try: _utils.wait_for_transform_job(client, batch_job_name) except: raise finally: cw_client = _utils.get_cloudwatch_client( args.region, assume_role_arn=args.assume_role) _utils.print_logs_for_job(cw_client, '/aws/sagemaker/TransformJobs', batch_job_name) _utils.write_output(args.output_location_output_path, args.output_location) logging.info('Batch Transformation creation completed.')
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting Processing Job to SageMaker...') job_name = _utils.create_processing_job(client, vars(args)) logging.info('Job request submitted. Waiting for completion...') try: _utils.wait_for_processing_job(client, job_name) except: raise finally: cw_client = _utils.get_cloudwatch_client(args.region) _utils.print_logs_for_job(cw_client, '/aws/sagemaker/ProcessingJobs', job_name) outputs = _utils.get_processing_job_outputs(client, job_name) with open('/tmp/job_name.txt', 'w') as f: f.write(job_name) with open('/tmp/output_artifacts.txt', 'w') as f: f.write(json.dumps(outputs)) logging.info('Job completed.')
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url, assume_role_arn=args.assume_role) endpoint_name = None old_endpoint_config_name = None if (args.update_endpoint and _utils.endpoint_name_exists(client, args.endpoint_name)): ## Get the old endpoint config to cleanup later old_endpoint_config_name = _utils.get_endpoint_config(client, args.endpoint_name) logging.info('Submitting Update Endpoint request to SageMaker...') endpoint_name = _utils.update_deployed_model(client, vars(args)) else: logging.info('Submitting Create Endpoint request to SageMaker...') endpoint_name = _utils.deploy_model(client, vars(args)) logging.info('Endpoint creation/update request submitted. Waiting for completion...') _utils.wait_for_endpoint_creation(client, endpoint_name) ## If updating existing endpoint, cleanup old endpoint config if old_endpoint_config_name: logging.info("Deleting old endpoint config: " + old_endpoint_config_name) if _utils.delete_endpoint_config(client, old_endpoint_config_name): logging.info("Deleted old endpoint config: " + old_endpoint_config_name) else: logging.info("Unable to delete old endpoint config: " + old_endpoint_config_name) _utils.write_output(args.endpoint_name_output_path, endpoint_name) logging.info('Endpoint creation/update completed.')
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting a create workteam request to SageMaker...') workteam_arn = _utils.create_workteam(client, vars(args)) logging.info('Workteam created.') _utils.write_output(args.workteam_arn_output_path, workteam_arn)
def main(argv=None): parser = create_parser() args = parser.parse_args() logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting model creation request to SageMaker...') _utils.create_model(client, vars(args)) logging.info('Model creation completed.') with open('/tmp/model_name.txt', 'w') as f: f.write(args.model_name)
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting model creation request to SageMaker...') _utils.create_model(client, vars(args)) logging.info('Model creation completed.') _utils.write_output(args.model_name_output_path, args.model_name)
def test_assumed_sagemaker_client(self): _utils.get_boto3_session = MagicMock() mock_sm_client = MagicMock() # Mock the client("SageMaker", ...) return value _utils.get_boto3_session.return_value.client.return_value = mock_sm_client client = _utils.get_sagemaker_client("us-east-1", assume_role_arn="abc123") assert client == mock_sm_client _utils.get_boto3_session.assert_called_once_with("us-east-1", "abc123") _utils.get_boto3_session.return_value.client.assert_called_once_with("sagemaker", endpoint_url=None, config=ANY)
def main(argv=None): parser = create_parser() args = parser.parse_args() logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting a create workteam request to SageMaker...') workteam_arn = _utils.create_workteam(client, vars(args)) logging.info('Workteam created.') with open('/tmp/workteam_arn.txt', 'w') as f: f.write(workteam_arn)
def main(argv=None): parser = create_parser() args = parser.parse_args() logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting Batch Transformation request to SageMaker...') batch_job_name = _utils.create_transform_job(client, vars(args)) logging.info('Batch Job request submitted. Waiting for completion...') _utils.wait_for_transform_job(client, batch_job_name) Path(args.output_location_file).parent.mkdir(parents=True, exist_ok=True) Path(args.output_location_file).write_text(unicode(args.output_location)) logging.info('Batch Transformation creation completed.')
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting Endpoint request to SageMaker...') endpoint_name = _utils.deploy_model(client, vars(args)) logging.info('Endpoint creation request submitted. Waiting for completion...') _utils.wait_for_endpoint_creation(client, endpoint_name) with open('/tmp/endpoint_name.txt', 'w') as f: f.write(endpoint_name) logging.info('Endpoint creation completed.')
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url, assume_role_arn=args.assume_role) logging.info('Submitting Endpoint request to SageMaker...') endpoint_name = _utils.deploy_model(client, vars(args)) logging.info( 'Endpoint creation request submitted. Waiting for completion...') _utils.wait_for_endpoint_creation(client, endpoint_name) _utils.write_output(args.endpoint_name_output_path, endpoint_name) logging.info('Endpoint creation completed.')
def main(argv=None): parser = create_parser() args = parser.parse_args() logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting Ground Truth Job request to SageMaker...') _utils.create_labeling_job(client, vars(args)) logging.info('Ground Truth labeling job request submitted. Waiting for completion...') _utils.wait_for_labeling_job(client, args.job_name) output_manifest, active_learning_model_arn = _utils.get_labeling_job_outputs(client, args.job_name, args.enable_auto_labeling) logging.info('Ground Truth Labeling Job completed.') with open('/tmp/output_manifest_location.txt', 'w') as f: f.write(output_manifest) with open('/tmp/active_learning_model_arn.txt', 'w') as f: f.write(active_learning_model_arn)
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url, assume_role_arn=args.assume_role) logging.info('Submitting Processing Job to SageMaker...') job_name = _utils.create_processing_job(client, vars(args)) def signal_term_handler(signalNumber, frame): logging.info(f"Stopping Processing Job: {job_name}") _utils.stop_processing_job(client, job_name) logging.info(f"Processing Job: {job_name} request submitted to Stop") signal.signal(signal.SIGTERM, signal_term_handler) logging.info('Job request submitted. Waiting for completion...') try: _utils.wait_for_processing_job(client, job_name) except: raise finally: cw_client = _utils.get_cloudwatch_client( args.region, assume_role_arn=args.assume_role) _utils.print_logs_for_job(cw_client, '/aws/sagemaker/ProcessingJobs', job_name) outputs = _utils.get_processing_job_outputs(client, job_name) _utils.write_output(args.job_name_output_path, job_name) _utils.write_output(args.output_artifacts_output_path, outputs, json_encode=True) logging.info('Job completed.')
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url) logging.info('Submitting Batch Transformation request to SageMaker...') batch_job_name = _utils.create_transform_job(client, vars(args)) logging.info('Batch Job request submitted. Waiting for completion...') try: _utils.wait_for_transform_job(client, batch_job_name) except: raise finally: cw_client = _utils.get_cloudwatch_client(args.region) _utils.print_logs_for_job(cw_client, '/aws/sagemaker/TransformJobs', batch_job_name) _utils.write_output(args.output_location_output_path, args.output_location) logging.info('Batch Transformation creation completed.')
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url, assume_role_arn=args.assume_role) logging.info( 'Submitting HyperParameter Tuning Job request to SageMaker...') hpo_job_name = _utils.create_hyperparameter_tuning_job(client, vars(args)) def signal_term_handler(signalNumber, frame): _utils.stop_hyperparameter_tuning_job(client, hpo_job_name) logging.info( f"HyperParameter Tuning Job: {hpo_job_name} request submitted to Stop" ) signal.signal(signal.SIGTERM, signal_term_handler) logging.info( 'HyperParameter Tuning Job request submitted. Waiting for completion...' ) _utils.wait_for_hyperparameter_training_job(client, hpo_job_name) best_job, best_hyperparameters = _utils.get_best_training_job_and_hyperparameters( client, hpo_job_name) model_artifact_url = _utils.get_model_artifacts_from_job(client, best_job) image = _utils.get_image_from_job(client, best_job) logging.info('HyperParameter Tuning Job completed.') _utils.write_output(args.hpo_job_name_output_path, hpo_job_name) _utils.write_output(args.model_artifact_url_output_path, model_artifact_url) _utils.write_output(args.best_job_name_output_path, best_job) _utils.write_output(args.best_hyperparameters_output_path, best_hyperparameters, json_encode=True) _utils.write_output(args.training_image_output_path, image)
def main(argv=None): parser = create_parser() args = parser.parse_args(argv) logging.getLogger().setLevel(logging.INFO) client = _utils.get_sagemaker_client(args.region, args.endpoint_url, assume_role_arn=args.assume_role) logging.info('Submitting Ground Truth Job request to SageMaker...') _utils.create_labeling_job(client, vars(args)) def signal_term_handler(signalNumber, frame): _utils.stop_labeling_job(client, args.job_name) logging.info(f"Ground Truth labeling job: {args.job_name} request submitted to Stop") signal.signal(signal.SIGTERM, signal_term_handler) logging.info('Ground Truth labeling job request submitted. Waiting for completion...') _utils.wait_for_labeling_job(client, args.job_name) output_manifest, active_learning_model_arn = _utils.get_labeling_job_outputs(client, args.job_name, args.enable_auto_labeling) logging.info('Ground Truth Labeling Job completed.') _utils.write_output(args.output_manifest_location_output_path, output_manifest) _utils.write_output(args.active_learning_model_arn_output_path, active_learning_model_arn)