def batch_describe(job_id, session, field=None): if isinstance(session, sagemaker.Session): session = session.boto_session client = session.client('s3control') account_id = get_account(session) description = batch_describe_get(account_id=account_id, job_id=job_id, client=client) description = get_field(description, field) return description
def ecr_ensure_image(image, session, force=False, cache=True, pull=True, push=True, wsl=False): ecr = session.client('ecr') # , region_name='eu-west-1') user_account = get_account(session) tag_account, tag_repo, tag_tag = parse_image(image.tag, account=user_account) if not force: img = get_image(ecr, tag_account, tag_repo, tag_tag) if img is not None: repo = ecr_ensure_repo(ecr, tag_account, tag_repo, user_account) repo_uri = repo['repositoryUri'] full_uri = "{}:{}".format(repo_uri, tag_tag) return full_uri return ecr_build_image(image=image, session=session, pull=pull, cache=cache, push=push, wsl=wsl)
def ecr_build_image(image: Image, session, cache=True, pull=True, push=True, wsl=False): print(f"Building image: {image}") # todo: add region param ecr = session.client('ecr') # , region_name='eu-west-1') region_name = session.region_name if not region_name: raise ValueError("No default region name in your AWS profile") user_account = get_account(session) ecr_ensure_image_deps(image=image, ecr=ecr, user_account=user_account, session=session, wsl=wsl) tag_account, tag_repo, tag_tag = parse_image(image.tag, account=user_account) repo = ecr_ensure_repo(ecr, tag_account, tag_repo, user_account) repo_uri = repo['repositoryUri'] path = os.path.join(image.path, tag_tag) accounts = list(image.accounts) if tag_account not in accounts: accounts.append(tag_account) client = docker.from_env() ecr_login(ecr, client, accounts) image_full = "{}:{}".format(repo_uri, tag_tag) print("Building image {} (cache: {}, pull:{})".format( image_full, cache, pull)) buildargs = {"REGION": region_name, "ACCOUNT": tag_account} base_path = os.path.join(image.path, tag_tag) #base_path = image.path df = download_files( files=image.download_files, session=session, base=base_path # todo: optional temp dir if write only ) if wsl: tdf = {} for k, v in df.items(): v = v.replace("\\", "/") if v[1] == ":": v = f"/mnt/{v[0]}{v[2:]}" tdf[k] = v df = tdf buildargs.update(df) print(f"buildargs: {buildargs}") try: streamer = client.images.build( path=path, tag=image_full, buildargs=buildargs, quiet=False, nocache=not cache, pull=pull, # decode=True ) except BuildError as e: for l in e.build_log: if 'stream' in l: for line in l['stream'].splitlines(): print(line) raise e """ for chunk in streamer.build_log: if 'stream' in chunk: for line in chunk['stream'].splitlines(): print(line) """ print("Built image {}".format(image_full)) #repo = "{}.dkr.ecr.{}.amazonaws.com/" if push: print("Pushing image {}".format(image_full)) client.images.push(repo_uri, tag_tag) print("Pushed image {}".format(image_full)) return image_full
if __name__ == '__main__': """ account = '683880991063' print(parse_image( '683880991063.dkr.ecr.us-east-1.amazonaws.com/columbo-sagemaker-training:latest', account=account)) print(parse_image( '683880991063.dkr.ecr.us-east-1.amazonaws.com/columbo-sagemaker-training', account=account)) print(parse_image('columbo-sagemaker-training:latest', account=account)) print(parse_image('columbo-sagemaker-training', account=account)) from boto3 import Session session = Session(profile_name='default') ecr = session.client('ecr') account = get_account(session) img = get_image( ecr, *parse_image('columbo-sagemaker-training', account=account)) print(img) repo = ecr_describe_repository(ecr, account, 'columbo-sagemaker-training') print(repo) """ from boto3 import Session session = Session(profile_name='default') ecr = session.client('ecr') account = get_account(session) for image in Images.ALL: img = ecr_ensure_image(image=image, session=session) print("Image: {}".format(img))