def _replicate_image(config: Dict[str, Any], src: str, dest: str) -> str:
logger.info("Replicating Image: %s -> %s", src, dest)
buildspec = yaml.safe_dump(
_generate_buildspec(config["repo_host"], config["repo_prefix"], src,
dest))
logger.debug("BuildSpec:\n%s", buildspec)
client = boto3.client("codebuild")
build_id = client.start_build(
projectName=config["codebuild_project"],
sourceTypeOverride="NO_SOURCE",
buildspecOverride=buildspec,
timeoutInMinutesOverride=config["codebuild_timeout"],
privilegedModeOverride=True,
)["build"]["id"]
logger.info("Started CodeBuild Id: %s", build_id)
while True:
build = client.batch_get_builds(ids=[build_id])["builds"][0]
status: str = build["buildStatus"]
phase: str = build["currentPhase"]
logger.debug("CodeBuild Id: %s, Phase: %s, Status: %s", build_id,
phase, status)
if status == "IN_PROGRESS":
time.sleep(10)
continue
else:
return status
def _inspect_containers(
config: Dict[str, Any],
lock: synchronize.Lock,
replications_queue: Queue, # type: ignore
replication_statuses: Dict[str, str],
containers: List[Any],
) -> Tuple[List[Dict[str, str]], List[str]]:
statuses: List[str] = []
desired_containers: List[Dict[str, str]] = []
for container in containers:
name, image = ((container["name"],
container["image"]) if isinstance(container, dict) else
(container.name, container.image))
desired_image = _get_desired_image(config, image)
logger.debug("Container Image: %s -> %s", image, desired_image)
desired_containers.append({"name": name, "image": desired_image})
if image != desired_image:
status = _get_replication_status(lock, replications_queue,
replication_statuses, image,
desired_image)
logger.info("Image: %s -> %s, Replication Status: %s", image,
desired_image, status)
statuses.append(status)
return desired_containers, statuses
def _generate_buildspec(repo_host: str, repo_prefix: str, src: str,
dest: str) -> Dict[str, Any]:
repo = dest.replace(f"{repo_host}/", "").split(":")[0]
build_spec = {
"version": 0.2,
"phases": {
"install": {
"runtime-versions": {
"python": 3.7,
"docker": 19
},
"commands": [
("nohup /usr/sbin/dockerd --host=unix:///var/run/docker.sock "
"--host=tcp://0.0.0.0:2375 --storage-driver=overlay&"),
'timeout 15 sh -c "until docker info; do echo .; sleep 1; done"',
],
},
"pre_build": {
"commands": [
f"aws ecr get-login-password | docker login --username AWS --password-stdin {repo_host}",
(f"aws ecr create-repository --repository-name {repo} "
f"--tags Key=Env,Value={repo_prefix} || echo 'Already exists'"
),
]
},
"build": {
"commands": [
f"docker pull {src}", f"docker tag {src} {dest}",
f"docker push {dest}"
]
},
},
}
logger.debug("BuildSpec: %s", build_spec)
return build_spec
def _inspect_standalone_pods(
config: Dict[str, Any],
lock: synchronize.Lock,
replications_queue: Queue, # type: ignore
replication_statuses: Dict[str, str],
) -> None:
pods = CoreV1Api().list_pod_for_all_namespaces()
for pod in pods.items:
if pod.metadata.owner_references:
owner_reference = pod.metadata.owner_references[0]
if owner_reference.kind != "Job":
logger.debug("Skipping Owned Pod: %s/%s",
pod.metadata.namespace, pod.metadata.name)
continue
else:
logger.info("Found Pod: %s/%s for Job: %s",
pod.metadata.namespace, pod.metadata.name,
owner_reference.name)
spec = pod.spec
containers, statuses = _inspect_containers(config, lock,
replications_queue,
replication_statuses,
spec.containers)
init_containers, init_statuses = (_inspect_containers(
config, lock, replications_queue, replication_statuses,
spec.init_containers) if spec.init_containers else ([], []))
all_statuses = statuses + init_statuses
all_containers = containers + init_containers
if len(all_statuses) > 0 and all(
[status == "Complete" for status in all_statuses]):
with lock:
for container in all_containers:
del replication_statuses[container["image"]]
body = {
"spec": {
"containers": containers,
"initContainers": init_containers,
}
}
CoreV1Api().patch_namespaced_pod(
name=pod.metadata.name,
namespace=pod.metadata.namespace,
body=body,
)
logger.info("Patched Pod: %s, Namespace: %s", pod.metadata.name,
pod.metadata.namespace)
def image_replicated(image: str) -> bool:
try:
repo, tag = image.split(":")
repo = "/".join(repo.split("/")[1:])
client = boto3.client("ecr")
paginator = client.get_paginator("list_images")
for page in paginator.paginate(repositoryName=repo):
for imageId in page["imageIds"]:
if imageId.get("imageTag", None) == tag:
logger.debug("ECR Repository contains Image: %s", image)
return True
logger.debug("Tag %s not found in ECR Repository %s", tag, repo)
return False
except Exception as e:
logger.exception(e)
return False