def processing_describe_cli(name, field):
"""
Describe training job NAME.
* Print full JSON if FIELD is not specified.
* Print only FIELD if specified (e.g., ModelArtifacts.S3ModelArtifacts or LastModifiedTime).
"""
session = boto3.Session(profile_name=current_profile)
session = sagemaker.Session(session)
description = processing_describe(
job_name=cli_argument(name, session=session),
field=field,
session=session
)
print(description)
def download_files(files, session, base):
#print(f"download_files: {files} to base {base}")
s3 = session.client('s3')
ret = {}
for k, v in files.items():
dest = os.path.join(base, k)
v = cli_argument(v, session=session)
if os.path.exists(v):
ret[k.upper()] = v
elif os.path.exists(dest):
ret[k.upper()] = dest
elif v.startswith("s3://"):
ret[k.upper()] = download_file_or_folder(uri=v,
session=session,
dest=dest,
file_subfolder=True)
else:
raise ValueError(f"Unhandled file path {k}: {v}")
#print(f"downloaded_files: {ret}")
return ret
def upload(src, dst, gz, session: sagemaker.Session, root='.'):
dst = cli_argument(dst, session=session)
if not os.path.exists(src):
raise click.UsageError("Source must exist")
if not dst.startswith('s3://'):
if dst.startswith('/'):
dst = dst[1:]
bucket = session.default_bucket()
dst = 's3://{}/{}'.format(bucket, dst)
url = urlparse(dst)
assert url.scheme == 's3'
bucket = url.netloc
key = url.path
if key.startswith('/'):
key = key[1:]
if os.path.isfile(src):
if gz:
raise click.UsageError(
"Option gz is only valid for source directories")
s3 = session.boto_session.client('s3')
s3.upload_file(src, bucket, key)
elif os.path.isdir(src):
if gz:
if not re.match(".*\\.(tar\\.gz||tgz)$", dst, re.IGNORECASE):
raise click.UsageError(
"Destination should end in .tar.gz or tgz")
s3_dst = os.path.dirname(dst)
file_name = os.path.basename(dst)
with _tmpdir() as tmp:
p = os.path.join(tmp, file_name)
with tarfile.open(p, 'w:gz') as arc:
arc.add(name=src, arcname=root, recursive=True)
s3 = session.boto_session.client('s3')
s3.upload_file(p, bucket, key)
else:
S3Uploader.upload(local_path=src,
desired_s3_uri=dst,
sagemaker_session=session)
else:
raise click.UsageError("Source must be file or directory")
def cli_json_read(path, field):
session = boto3.Session(profile_name=current_profile)
path = cli_argument(path, session=session)
data = json_read(path, field, session=session)
print(data)
def cli_json_parse(path):
session = boto3.Session(profile_name=current_profile)
data = cli_argument(path, session=session)
print(data)
def batch_run(args, config: BatchConfig):
session = boto3.Session(profile_name=args.profile)
sts = session.client('sts')
#s3control = session.client('s3control')
#s3 = session.client('s3')
lambda_client = session.client('lambda')
cloudformation = session.client('cloudformation')
manifest = cli_argument(args.manifest, session=session)
report = cli_argument(args.report, session=session)
# Create function
ensure_lambda_js(path=args.code_dir,
stack_name=args.stack_name,
session=session,
webpack=config.webpack,
deploy=args.deploy,
development=args.development,
extra_files=config.extra_files,
package_json=config.package_json)
if args.deploy_only:
return
function_arn, batch_role_arn = get_cloudformation_output(
cloudformation=cloudformation,
stack_name=args.stack_name,
output_key=['LambdaFunctionArn', 'BatchRoleArn'])
assert function_arn
assert batch_role_arn
# Versioned function for this call
lambda_env = {"SOFT_TIMEOUT": args.soft_timeout}
if config.env_callback:
lambda_env.update(config.env_callback(args))
function_arn = update_function(lambda_client=lambda_client,
function_name=function_arn,
env=lambda_env,
timeout=args.timeout,
memory=args.memory)
#manifest = args.manifest
#report = args.report
# todo: report prefix /
identity = sts.get_caller_identity()
account_id = identity['Account']
print("AccountID: {}".format(account_id))
response = create_job(
session=session,
manifest=manifest,
report=report,
arn=function_arn,
account_id=account_id,
description=args.description,
role_name=batch_role_arn, # args.role_name,
confirmation_required=args.confirmation_required,
ignore=args.ignore)
if args.output_json:
os.makedirs(os.path.dirname(os.path.abspath(args.output_json)),
exist_ok=True)
with open(args.output_json, 'w') as f:
json.dump(response, f)
def parse_channel_arguments(channels, session):
return {
k: v.copy(local=cli_argument(v.local, session=session))
for k, v in channels.items()
}
def sagemaker_processing_run(args, config):
script = args.sagemaker_script
script = os.path.abspath(script)
script = script.replace("\\", "/")
session = sagemaker_session(profile_name=args.sagemaker_profile)
inputs = {
k: PathArgument(
local=cli_argument(getattr(args, k), session=session),
optional=v.optional,
mode=getattr(args, "{}_mode".format(k) or v.mode or 'File')
) for k, v in config.inputs.items()
}
for k, v in inputs.items():
if (not v.local) and (not v.optional):
raise ValueError("Value required for input agument [{}]".format(k))
inputs = {
k: v for k, v in inputs.items() if v.local
}
outputs = {
k: PathArgument(
local=cli_argument(getattr(args, k), session=session),
remote=cli_argument(
getattr(args, "{}_s3".format(k)), session=session),
optional=v.optional,
mode=getattr(args, "{}_mode".format(k) or v.mode or 'EndOfJob')
) for k, v in config.outputs.items()
}
# for k, v in outputs.items():
# if (not v) and (not config.outputs[k].optional):
# raise ValueError(
# "Value required for output agument [{}_s3]".format(k))
# outputs = {
# k: v for k, v in outputs.items() if v
# }
# todo: optional arguments
dependencies = {
k: getattr(args, k) for k in config.dependencies.keys()
}
tags = git_get_tags(script)
process(
inputs=inputs,
outputs=outputs,
dependencies=dependencies,
session=session,
role=args.sagemaker_role,
script=script,
image=args.sagemaker_image,
image_path=args.sagemaker_image_path,
image_accounts=args.sagemaker_image_accounts,
instance=args.sagemaker_instance,
base_job_name=args.sagemaker_base_job_name,
job_name=args.sagemaker_job_name,
volume_size=args.sagemaker_volume_size,
python=args.sagemaker_python,
runtime_seconds=args.sagemaker_runtime_seconds,
output_mount=args.sagemaker_output_mount if hasattr(
args, 'sagemaker_output_mount') else None,
input_mount=args.sagemaker_input_mount if hasattr(
args, 'sagemaker_input_mount') else None,
module_mount=args.sagemaker_module_mount if hasattr(
args, 'sagemaker_module_mount') else None,
arguments=make_arguments(args=args, config=config),
requirements=args.sagemaker_requirements,
configuration_script=args.sagemaker_configuration_script,
configuration_command=args.sagemaker_configuration_command,
wait=args.sagemaker_wait,
tags=tags,
output_json=args.sagemaker_output_json,
env=config.env
)
def batch_report(session, job, output):
if isinstance(session, boto3.Session):
session = sagemaker.Session(boto_session=session)
job = [cli_argument(j, session=session) for j in job]
report_manifests = [
batch_describe(session=session, job_id=j, field='Report.s3')
for j in job
]
for rm in report_manifests:
assert rm, "Missing report manifest"
report_manifests.reverse()
failure_set = set()
success_set = set()
s3 = session.boto_session.client('s3')
os.makedirs(output, exist_ok=True)
success_report = os.path.join(output, 'success.csv')
failure_report = os.path.join(output, 'failure.csv')
info_file = os.path.join(output, 'info.json')
success_reports = []
failure_reports = []
for rm in report_manifests:
rms = get_file_string(s3=s3, url=rm)
rms = json.loads(rms)
results = rms['Results']
print("Report keys: {}".format(r['TaskExecutionStatus']
for r in results))
succeeded = next(
(res
for res in results if res['TaskExecutionStatus'] == 'succeeded'),
None)
failed = next(
(res for res in results if res['TaskExecutionStatus'] == 'failed'),
None)
if succeeded:
bucket = succeeded['Bucket']
key = succeeded['Key']
url = f"s3://{bucket}/{key}"
print(f"Succeeded: {url}")
success_reports.append(url)
if failed:
bucket = failed['Bucket']
key = failed['Key']
url = f"s3://{bucket}/{key}"
print(f"Failed: {url}")
failure_reports.append(url)
with open(success_report, 'w', newline='') as fsuccess:
wsuccess = csv.writer(fsuccess)
for sr in success_reports:
with get_file(url=sr, s3=s3) as fi:
reader = codecs.getreader('utf-8')(fi)
for row in csv.reader(reader):
file_url = f"s3://{row[0]}/{row[1]}"
if file_url not in success_set:
wsuccess.writerow(row)
success_set.add(file_url)
with open(failure_report, 'w', newline='') as ffailure:
wfailure = csv.writer(ffailure)
for fr in failure_reports:
with get_file(url=fr, s3=s3) as fi:
reader = codecs.getreader('utf-8')(fi)
for row in csv.reader(reader):
file_url = f"s3://{row[0]}/{row[1]}"
if file_url not in success_set and file_url not in failure_set:
wfailure.writerow(row)
failure_set.add(file_url)
info = {
"success": len(success_set),
"failure": len(failure_set),
}
with open(info_file, 'w') as f:
json.dump(info, f)
print("Success: {}, failure: {}".format(info['success'], info['failure']))
def model_create(job,
model_artifact,
name,
session: sagemaker.Session,
inference_image,
inference_image_path,
inference_image_accounts,
role,
force,
multimodel=False,
accelerator_type=None):
job = cli_argument(job, session=session)
name = cli_argument(name, session=session)
model_artifact = cli_argument(model_artifact, session=session)
image_config = Image(tag=inference_image,
path=inference_image_path,
accounts=inference_image_accounts)
image_uri = ecr_ensure_image(image=image_config,
session=session.boto_session)
if (job and model_artifact) or (not (job or model_artifact)):
raise click.UsageError('Specify one of job_name or model_artifact')
if model_artifact and not name:
raise click.UsageError('name is required if job is not provided')
iam = session.boto_session.client('iam')
client = session.boto_session.client('sagemaker')
role = ensure_inference_role(iam=iam, role_name=role)
if job:
client = session.boto_session.client('sagemaker')
model_artifact = training_describe(
job_name=job,
field='ModelArtifacts.S3ModelArtifacts',
session=session)
if not name:
name = job
print("Creating model [{}] from job [{}] artifact [{}]".format(
name, job, model_artifact))
else:
if not model_artifact.startswith('s3://'):
if model_artifact.startswith('/'):
model_artifact = model_artifact[1:]
bucket = session.default_bucket()
model_artifact = 's3://{}/{}'.format(bucket, model_artifact)
print("Creating model [{}] from artifact [{}]".format(
name, model_artifact))
if model_exists(name=name, client=client):
if force:
print("Deleting existing model")
model_delete(name=name, client=client)
else:
raise click.UsageError('Specify force if overwriting model')
model = sagemaker.Model(
image_uri=image_uri,
model_data=model_artifact,
role=role,
predictor_cls=None,
env=None,
name=name,
# vpc_config=None,
sagemaker_session=session,
# enable_network_isolation=False,
# model_kms_key=None
)
container_def = sagemaker.container_def(
model.image_uri,
model.model_data,
model.env,
container_mode='MultiModel' if multimodel else 'SingleModel')
"""
client.create_model(
ModelName='string',
PrimaryContainer={
'ContainerHostname': 'string',
'Image': 'string',
'ImageConfig': {
'RepositoryAccessMode': 'Platform'|'Vpc'
},
'Mode': 'SingleModel'|'MultiModel',
'ModelDataUrl': 'string',
'Environment': {
'string': 'string'
},
'ModelPackageName': 'string'
},
"""
# self._ensure_base_name_if_needed(container_def["Image"])
# self._set_model_name_if_needed()
enable_network_isolation = model.enable_network_isolation()
# self._init_sagemaker_session_if_does_not_exist(instance_type)
session.create_model(
model.name,
model.role,
container_def,
vpc_config=model.vpc_config,
enable_network_isolation=enable_network_isolation,
# tags=tags,
)
def transform_create(
session: boto3.Session,
base_job_name,
job_name,
model_name,
concurrency,
timeout,
retries,
input_s3,
output_s3,
input_type,
output_type,
instance_type,
instance_count,
payload_mb,
output_json
):
input_s3 = cli_argument(input_s3, session=session)
base_job_name = cli_argument(base_job_name, session=session)
output_s3 = cli_argument(output_s3, session=session)
job_name = cli_argument(job_name, session=session)
model_name = cli_argument(model_name, session=session)
if(isinstance(session, sagemaker.Session)):
session = session.boto_session
if not job_name:
if not base_job_name:
raise UsageError(
"Either --job-name or --base-job-name is required")
job_name = name_from_base(base_job_name)
print(f"Job name: {job_name}")
client = session.client('sagemaker')
response = client.create_transform_job(
TransformJobName=job_name,
ModelName=model_name,
MaxConcurrentTransforms=concurrency,
ModelClientConfig={
'InvocationsTimeoutInSeconds': timeout,
'InvocationsMaxRetries': retries
},
MaxPayloadInMB=payload_mb,
BatchStrategy='SingleRecord', # 'MultiRecord'|'SingleRecord',
# Environment={
# 'string': 'string'
# },
TransformInput={
'DataSource': {
'S3DataSource': {
'S3DataType': 'S3Prefix', # 'ManifestFile'|'S3Prefix'|'AugmentedManifestFile',
'S3Uri': input_s3
}
},
'ContentType': input_type,
'CompressionType': 'None', # |'Gzip',
'SplitType': 'None' # |'Line'|'RecordIO'|'TFRecord'
},
TransformOutput={
'S3OutputPath': output_s3, # 'string',
'Accept': output_type, # 'string',
'AssembleWith': 'None' # |'Line',
# 'KmsKeyId': 'string'
},
TransformResources={
'InstanceType': instance_type, # 'ml.m4.xlarge'|'ml.m4.2xlarge'|'ml.m4.4xlarge'|'ml.m4.10xlarge'|'ml.m4.16xlarge'|'ml.c4.xlarge'|'ml.c4.2xlarge'|'ml.c4.4xlarge'|'ml.c4.8xlarge'|'ml.p2.xlarge'|'ml.p2.8xlarge'|'ml.p2.16xlarge'|'ml.p3.2xlarge'|'ml.p3.8xlarge'|'ml.p3.16xlarge'|'ml.c5.xlarge'|'ml.c5.2xlarge'|'ml.c5.4xlarge'|'ml.c5.9xlarge'|'ml.c5.18xlarge'|'ml.m5.large'|'ml.m5.xlarge'|'ml.m5.2xlarge'|'ml.m5.4xlarge'|'ml.m5.12xlarge'|'ml.m5.24xlarge',
'InstanceCount': instance_count,
# 'VolumeKmsKeyId': 'string'
},
# DataProcessing={
# 'InputFilter': 'string',
# 'OutputFilter': 'string',
# 'JoinSource': 'Input'|'None'
# },
Tags=[
{
'Key': 'Source',
'Value': 'aws-sagemaker-remote'
},
],
# ExperimentConfig={
# 'ExperimentName': 'string',
# 'TrialName': 'string',
# 'TrialComponentDisplayName': 'string'
# }
)
print(f"Response: {response}")
arn = response.get('TransformJobArn', None)
print(f"ARN: {arn}")
if output_json:
os.makedirs(os.path.dirname(os.path.abspath(output_json)), exist_ok=True)
with open(output_json, 'w') as f:
json.dump(response, f)
print(f"Response saved as [{output_json}]")