def processing_describe_cli(name, field): """ Describe training job NAME. * Print full JSON if FIELD is not specified. * Print only FIELD if specified (e.g., ModelArtifacts.S3ModelArtifacts or LastModifiedTime). """ session = boto3.Session(profile_name=current_profile) session = sagemaker.Session(session) description = processing_describe( job_name=cli_argument(name, session=session), field=field, session=session ) print(description)
def download_files(files, session, base): #print(f"download_files: {files} to base {base}") s3 = session.client('s3') ret = {} for k, v in files.items(): dest = os.path.join(base, k) v = cli_argument(v, session=session) if os.path.exists(v): ret[k.upper()] = v elif os.path.exists(dest): ret[k.upper()] = dest elif v.startswith("s3://"): ret[k.upper()] = download_file_or_folder(uri=v, session=session, dest=dest, file_subfolder=True) else: raise ValueError(f"Unhandled file path {k}: {v}") #print(f"downloaded_files: {ret}") return ret
def upload(src, dst, gz, session: sagemaker.Session, root='.'): dst = cli_argument(dst, session=session) if not os.path.exists(src): raise click.UsageError("Source must exist") if not dst.startswith('s3://'): if dst.startswith('/'): dst = dst[1:] bucket = session.default_bucket() dst = 's3://{}/{}'.format(bucket, dst) url = urlparse(dst) assert url.scheme == 's3' bucket = url.netloc key = url.path if key.startswith('/'): key = key[1:] if os.path.isfile(src): if gz: raise click.UsageError( "Option gz is only valid for source directories") s3 = session.boto_session.client('s3') s3.upload_file(src, bucket, key) elif os.path.isdir(src): if gz: if not re.match(".*\\.(tar\\.gz||tgz)$", dst, re.IGNORECASE): raise click.UsageError( "Destination should end in .tar.gz or tgz") s3_dst = os.path.dirname(dst) file_name = os.path.basename(dst) with _tmpdir() as tmp: p = os.path.join(tmp, file_name) with tarfile.open(p, 'w:gz') as arc: arc.add(name=src, arcname=root, recursive=True) s3 = session.boto_session.client('s3') s3.upload_file(p, bucket, key) else: S3Uploader.upload(local_path=src, desired_s3_uri=dst, sagemaker_session=session) else: raise click.UsageError("Source must be file or directory")
def cli_json_read(path, field): session = boto3.Session(profile_name=current_profile) path = cli_argument(path, session=session) data = json_read(path, field, session=session) print(data)
def cli_json_parse(path): session = boto3.Session(profile_name=current_profile) data = cli_argument(path, session=session) print(data)
def batch_run(args, config: BatchConfig): session = boto3.Session(profile_name=args.profile) sts = session.client('sts') #s3control = session.client('s3control') #s3 = session.client('s3') lambda_client = session.client('lambda') cloudformation = session.client('cloudformation') manifest = cli_argument(args.manifest, session=session) report = cli_argument(args.report, session=session) # Create function ensure_lambda_js(path=args.code_dir, stack_name=args.stack_name, session=session, webpack=config.webpack, deploy=args.deploy, development=args.development, extra_files=config.extra_files, package_json=config.package_json) if args.deploy_only: return function_arn, batch_role_arn = get_cloudformation_output( cloudformation=cloudformation, stack_name=args.stack_name, output_key=['LambdaFunctionArn', 'BatchRoleArn']) assert function_arn assert batch_role_arn # Versioned function for this call lambda_env = {"SOFT_TIMEOUT": args.soft_timeout} if config.env_callback: lambda_env.update(config.env_callback(args)) function_arn = update_function(lambda_client=lambda_client, function_name=function_arn, env=lambda_env, timeout=args.timeout, memory=args.memory) #manifest = args.manifest #report = args.report # todo: report prefix / identity = sts.get_caller_identity() account_id = identity['Account'] print("AccountID: {}".format(account_id)) response = create_job( session=session, manifest=manifest, report=report, arn=function_arn, account_id=account_id, description=args.description, role_name=batch_role_arn, # args.role_name, confirmation_required=args.confirmation_required, ignore=args.ignore) if args.output_json: os.makedirs(os.path.dirname(os.path.abspath(args.output_json)), exist_ok=True) with open(args.output_json, 'w') as f: json.dump(response, f)
def parse_channel_arguments(channels, session): return { k: v.copy(local=cli_argument(v.local, session=session)) for k, v in channels.items() }
def sagemaker_processing_run(args, config): script = args.sagemaker_script script = os.path.abspath(script) script = script.replace("\\", "/") session = sagemaker_session(profile_name=args.sagemaker_profile) inputs = { k: PathArgument( local=cli_argument(getattr(args, k), session=session), optional=v.optional, mode=getattr(args, "{}_mode".format(k) or v.mode or 'File') ) for k, v in config.inputs.items() } for k, v in inputs.items(): if (not v.local) and (not v.optional): raise ValueError("Value required for input agument [{}]".format(k)) inputs = { k: v for k, v in inputs.items() if v.local } outputs = { k: PathArgument( local=cli_argument(getattr(args, k), session=session), remote=cli_argument( getattr(args, "{}_s3".format(k)), session=session), optional=v.optional, mode=getattr(args, "{}_mode".format(k) or v.mode or 'EndOfJob') ) for k, v in config.outputs.items() } # for k, v in outputs.items(): # if (not v) and (not config.outputs[k].optional): # raise ValueError( # "Value required for output agument [{}_s3]".format(k)) # outputs = { # k: v for k, v in outputs.items() if v # } # todo: optional arguments dependencies = { k: getattr(args, k) for k in config.dependencies.keys() } tags = git_get_tags(script) process( inputs=inputs, outputs=outputs, dependencies=dependencies, session=session, role=args.sagemaker_role, script=script, image=args.sagemaker_image, image_path=args.sagemaker_image_path, image_accounts=args.sagemaker_image_accounts, instance=args.sagemaker_instance, base_job_name=args.sagemaker_base_job_name, job_name=args.sagemaker_job_name, volume_size=args.sagemaker_volume_size, python=args.sagemaker_python, runtime_seconds=args.sagemaker_runtime_seconds, output_mount=args.sagemaker_output_mount if hasattr( args, 'sagemaker_output_mount') else None, input_mount=args.sagemaker_input_mount if hasattr( args, 'sagemaker_input_mount') else None, module_mount=args.sagemaker_module_mount if hasattr( args, 'sagemaker_module_mount') else None, arguments=make_arguments(args=args, config=config), requirements=args.sagemaker_requirements, configuration_script=args.sagemaker_configuration_script, configuration_command=args.sagemaker_configuration_command, wait=args.sagemaker_wait, tags=tags, output_json=args.sagemaker_output_json, env=config.env )
def batch_report(session, job, output): if isinstance(session, boto3.Session): session = sagemaker.Session(boto_session=session) job = [cli_argument(j, session=session) for j in job] report_manifests = [ batch_describe(session=session, job_id=j, field='Report.s3') for j in job ] for rm in report_manifests: assert rm, "Missing report manifest" report_manifests.reverse() failure_set = set() success_set = set() s3 = session.boto_session.client('s3') os.makedirs(output, exist_ok=True) success_report = os.path.join(output, 'success.csv') failure_report = os.path.join(output, 'failure.csv') info_file = os.path.join(output, 'info.json') success_reports = [] failure_reports = [] for rm in report_manifests: rms = get_file_string(s3=s3, url=rm) rms = json.loads(rms) results = rms['Results'] print("Report keys: {}".format(r['TaskExecutionStatus'] for r in results)) succeeded = next( (res for res in results if res['TaskExecutionStatus'] == 'succeeded'), None) failed = next( (res for res in results if res['TaskExecutionStatus'] == 'failed'), None) if succeeded: bucket = succeeded['Bucket'] key = succeeded['Key'] url = f"s3://{bucket}/{key}" print(f"Succeeded: {url}") success_reports.append(url) if failed: bucket = failed['Bucket'] key = failed['Key'] url = f"s3://{bucket}/{key}" print(f"Failed: {url}") failure_reports.append(url) with open(success_report, 'w', newline='') as fsuccess: wsuccess = csv.writer(fsuccess) for sr in success_reports: with get_file(url=sr, s3=s3) as fi: reader = codecs.getreader('utf-8')(fi) for row in csv.reader(reader): file_url = f"s3://{row[0]}/{row[1]}" if file_url not in success_set: wsuccess.writerow(row) success_set.add(file_url) with open(failure_report, 'w', newline='') as ffailure: wfailure = csv.writer(ffailure) for fr in failure_reports: with get_file(url=fr, s3=s3) as fi: reader = codecs.getreader('utf-8')(fi) for row in csv.reader(reader): file_url = f"s3://{row[0]}/{row[1]}" if file_url not in success_set and file_url not in failure_set: wfailure.writerow(row) failure_set.add(file_url) info = { "success": len(success_set), "failure": len(failure_set), } with open(info_file, 'w') as f: json.dump(info, f) print("Success: {}, failure: {}".format(info['success'], info['failure']))
def model_create(job, model_artifact, name, session: sagemaker.Session, inference_image, inference_image_path, inference_image_accounts, role, force, multimodel=False, accelerator_type=None): job = cli_argument(job, session=session) name = cli_argument(name, session=session) model_artifact = cli_argument(model_artifact, session=session) image_config = Image(tag=inference_image, path=inference_image_path, accounts=inference_image_accounts) image_uri = ecr_ensure_image(image=image_config, session=session.boto_session) if (job and model_artifact) or (not (job or model_artifact)): raise click.UsageError('Specify one of job_name or model_artifact') if model_artifact and not name: raise click.UsageError('name is required if job is not provided') iam = session.boto_session.client('iam') client = session.boto_session.client('sagemaker') role = ensure_inference_role(iam=iam, role_name=role) if job: client = session.boto_session.client('sagemaker') model_artifact = training_describe( job_name=job, field='ModelArtifacts.S3ModelArtifacts', session=session) if not name: name = job print("Creating model [{}] from job [{}] artifact [{}]".format( name, job, model_artifact)) else: if not model_artifact.startswith('s3://'): if model_artifact.startswith('/'): model_artifact = model_artifact[1:] bucket = session.default_bucket() model_artifact = 's3://{}/{}'.format(bucket, model_artifact) print("Creating model [{}] from artifact [{}]".format( name, model_artifact)) if model_exists(name=name, client=client): if force: print("Deleting existing model") model_delete(name=name, client=client) else: raise click.UsageError('Specify force if overwriting model') model = sagemaker.Model( image_uri=image_uri, model_data=model_artifact, role=role, predictor_cls=None, env=None, name=name, # vpc_config=None, sagemaker_session=session, # enable_network_isolation=False, # model_kms_key=None ) container_def = sagemaker.container_def( model.image_uri, model.model_data, model.env, container_mode='MultiModel' if multimodel else 'SingleModel') """ client.create_model( ModelName='string', PrimaryContainer={ 'ContainerHostname': 'string', 'Image': 'string', 'ImageConfig': { 'RepositoryAccessMode': 'Platform'|'Vpc' }, 'Mode': 'SingleModel'|'MultiModel', 'ModelDataUrl': 'string', 'Environment': { 'string': 'string' }, 'ModelPackageName': 'string' }, """ # self._ensure_base_name_if_needed(container_def["Image"]) # self._set_model_name_if_needed() enable_network_isolation = model.enable_network_isolation() # self._init_sagemaker_session_if_does_not_exist(instance_type) session.create_model( model.name, model.role, container_def, vpc_config=model.vpc_config, enable_network_isolation=enable_network_isolation, # tags=tags, )
def transform_create( session: boto3.Session, base_job_name, job_name, model_name, concurrency, timeout, retries, input_s3, output_s3, input_type, output_type, instance_type, instance_count, payload_mb, output_json ): input_s3 = cli_argument(input_s3, session=session) base_job_name = cli_argument(base_job_name, session=session) output_s3 = cli_argument(output_s3, session=session) job_name = cli_argument(job_name, session=session) model_name = cli_argument(model_name, session=session) if(isinstance(session, sagemaker.Session)): session = session.boto_session if not job_name: if not base_job_name: raise UsageError( "Either --job-name or --base-job-name is required") job_name = name_from_base(base_job_name) print(f"Job name: {job_name}") client = session.client('sagemaker') response = client.create_transform_job( TransformJobName=job_name, ModelName=model_name, MaxConcurrentTransforms=concurrency, ModelClientConfig={ 'InvocationsTimeoutInSeconds': timeout, 'InvocationsMaxRetries': retries }, MaxPayloadInMB=payload_mb, BatchStrategy='SingleRecord', # 'MultiRecord'|'SingleRecord', # Environment={ # 'string': 'string' # }, TransformInput={ 'DataSource': { 'S3DataSource': { 'S3DataType': 'S3Prefix', # 'ManifestFile'|'S3Prefix'|'AugmentedManifestFile', 'S3Uri': input_s3 } }, 'ContentType': input_type, 'CompressionType': 'None', # |'Gzip', 'SplitType': 'None' # |'Line'|'RecordIO'|'TFRecord' }, TransformOutput={ 'S3OutputPath': output_s3, # 'string', 'Accept': output_type, # 'string', 'AssembleWith': 'None' # |'Line', # 'KmsKeyId': 'string' }, TransformResources={ 'InstanceType': instance_type, # 'ml.m4.xlarge'|'ml.m4.2xlarge'|'ml.m4.4xlarge'|'ml.m4.10xlarge'|'ml.m4.16xlarge'|'ml.c4.xlarge'|'ml.c4.2xlarge'|'ml.c4.4xlarge'|'ml.c4.8xlarge'|'ml.p2.xlarge'|'ml.p2.8xlarge'|'ml.p2.16xlarge'|'ml.p3.2xlarge'|'ml.p3.8xlarge'|'ml.p3.16xlarge'|'ml.c5.xlarge'|'ml.c5.2xlarge'|'ml.c5.4xlarge'|'ml.c5.9xlarge'|'ml.c5.18xlarge'|'ml.m5.large'|'ml.m5.xlarge'|'ml.m5.2xlarge'|'ml.m5.4xlarge'|'ml.m5.12xlarge'|'ml.m5.24xlarge', 'InstanceCount': instance_count, # 'VolumeKmsKeyId': 'string' }, # DataProcessing={ # 'InputFilter': 'string', # 'OutputFilter': 'string', # 'JoinSource': 'Input'|'None' # }, Tags=[ { 'Key': 'Source', 'Value': 'aws-sagemaker-remote' }, ], # ExperimentConfig={ # 'ExperimentName': 'string', # 'TrialName': 'string', # 'TrialComponentDisplayName': 'string' # } ) print(f"Response: {response}") arn = response.get('TransformJobArn', None) print(f"ARN: {arn}") if output_json: os.makedirs(os.path.dirname(os.path.abspath(output_json)), exist_ok=True) with open(output_json, 'w') as f: json.dump(response, f) print(f"Response saved as [{output_json}]")