def batch_describe(job_id, session, field=None):
if isinstance(session, sagemaker.Session):
session = session.boto_session
client = session.client('s3control')
account_id = get_account(session)
description = batch_describe_get(account_id=account_id,
job_id=job_id,
client=client)
description = get_field(description, field)
return description
def ecr_ensure_image(image,
session,
force=False,
cache=True,
pull=True,
push=True,
wsl=False):
ecr = session.client('ecr') # , region_name='eu-west-1')
user_account = get_account(session)
tag_account, tag_repo, tag_tag = parse_image(image.tag,
account=user_account)
if not force:
img = get_image(ecr, tag_account, tag_repo, tag_tag)
if img is not None:
repo = ecr_ensure_repo(ecr, tag_account, tag_repo, user_account)
repo_uri = repo['repositoryUri']
full_uri = "{}:{}".format(repo_uri, tag_tag)
return full_uri
return ecr_build_image(image=image,
session=session,
pull=pull,
cache=cache,
push=push,
wsl=wsl)
def ecr_build_image(image: Image,
session,
cache=True,
pull=True,
push=True,
wsl=False):
print(f"Building image: {image}")
# todo: add region param
ecr = session.client('ecr') # , region_name='eu-west-1')
region_name = session.region_name
if not region_name:
raise ValueError("No default region name in your AWS profile")
user_account = get_account(session)
ecr_ensure_image_deps(image=image,
ecr=ecr,
user_account=user_account,
session=session,
wsl=wsl)
tag_account, tag_repo, tag_tag = parse_image(image.tag,
account=user_account)
repo = ecr_ensure_repo(ecr, tag_account, tag_repo, user_account)
repo_uri = repo['repositoryUri']
path = os.path.join(image.path, tag_tag)
accounts = list(image.accounts)
if tag_account not in accounts:
accounts.append(tag_account)
client = docker.from_env()
ecr_login(ecr, client, accounts)
image_full = "{}:{}".format(repo_uri, tag_tag)
print("Building image {} (cache: {}, pull:{})".format(
image_full, cache, pull))
buildargs = {"REGION": region_name, "ACCOUNT": tag_account}
base_path = os.path.join(image.path, tag_tag)
#base_path = image.path
df = download_files(
files=image.download_files,
session=session,
base=base_path # todo: optional temp dir if write only
)
if wsl:
tdf = {}
for k, v in df.items():
v = v.replace("\\", "/")
if v[1] == ":":
v = f"/mnt/{v[0]}{v[2:]}"
tdf[k] = v
df = tdf
buildargs.update(df)
print(f"buildargs: {buildargs}")
try:
streamer = client.images.build(
path=path,
tag=image_full,
buildargs=buildargs,
quiet=False,
nocache=not cache,
pull=pull,
# decode=True
)
except BuildError as e:
for l in e.build_log:
if 'stream' in l:
for line in l['stream'].splitlines():
print(line)
raise e
"""
for chunk in streamer.build_log:
if 'stream' in chunk:
for line in chunk['stream'].splitlines():
print(line)
"""
print("Built image {}".format(image_full))
#repo = "{}.dkr.ecr.{}.amazonaws.com/"
if push:
print("Pushing image {}".format(image_full))
client.images.push(repo_uri, tag_tag)
print("Pushed image {}".format(image_full))
return image_full
if __name__ == '__main__':
"""
account = '683880991063'
print(parse_image(
'683880991063.dkr.ecr.us-east-1.amazonaws.com/columbo-sagemaker-training:latest', account=account))
print(parse_image(
'683880991063.dkr.ecr.us-east-1.amazonaws.com/columbo-sagemaker-training', account=account))
print(parse_image('columbo-sagemaker-training:latest', account=account))
print(parse_image('columbo-sagemaker-training', account=account))
from boto3 import Session
session = Session(profile_name='default')
ecr = session.client('ecr')
account = get_account(session)
img = get_image(
ecr, *parse_image('columbo-sagemaker-training', account=account))
print(img)
repo = ecr_describe_repository(ecr, account, 'columbo-sagemaker-training')
print(repo)
"""
from boto3 import Session
session = Session(profile_name='default')
ecr = session.client('ecr')
account = get_account(session)
for image in Images.ALL:
img = ecr_ensure_image(image=image, session=session)
print("Image: {}".format(img))