def _mk_transport(
image_name: str,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool],
oa.OciConfig],
action: str = docker_http.PULL,
privileges: oa.Privileges = oa.Privileges.READONLY,
):
if isinstance(image_name, str):
image_name = ou.normalise_image_reference(image_name)
credentials = _mk_credentials(
image_reference=str(image_name),
credentials_lookup=credentials_lookup,
privileges=privileges,
)
if isinstance(image_name, str):
image_name = docker_name.from_string(name=image_name)
transport_pool = _mk_transport_pool(size=1)
transport = docker_http.Transport(
name=image_name,
creds=credentials,
transport=transport_pool,
action=docker_http.PULL,
)
return transport
def pulled_image(
image_reference: str,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool],
oa.OciConfig],
):
_tag_or_digest_reference(image_reference)
transport = _mk_transport_pool()
image_reference = ou.normalise_image_reference(image_reference)
image_reference = docker_name.from_string(image_reference)
creds = _mk_credentials(
image_reference=image_reference,
credentials_lookup=credentials_lookup,
)
# OCI Image Manifest is compatible with Docker Image Manifest Version 2,
# Schema 2. We indicate support for both formats by passing both media types
# as 'Accept' headers.
#
# For reference:
# OCI: https://github.com/opencontainers/image-spec
# Docker: https://docs.docker.com/registry/spec/manifest-v2-2/
accept = docker_http.SUPPORTED_MANIFEST_MIMES
try:
logger.info(f'Pulling v2.2 image from {image_reference}..')
with v2_2_image.FromRegistry(image_reference, creds, transport,
accept) as v2_2_img:
if v2_2_img.exists():
yield v2_2_img
return
# XXX TODO: use streaming rather than writing to local FS
# if outfile is given, we must use it instead of an ano
logger.debug(f'Pulling manifest list from {image_reference}..')
with image_list.FromRegistry(image_reference, creds,
transport) as img_list:
if img_list.exists():
platform = image_list.Platform({
'architecture': _PROCESSOR_ARCHITECTURE,
'os': _OPERATING_SYSTEM,
})
# pytype: disable=wrong-arg-types
with img_list.resolve(platform) as default_child:
yield default_child
return
logger.info(f'Pulling v2 image from {image_reference}..')
with v2_image.FromRegistry(image_reference, creds,
transport) as v2_img:
if v2_img.exists():
with v2_compat.V22FromV2(v2_img) as v2_2_img:
yield v2_2_img
return
raise om.OciImageNotFoundException(
f'failed to retrieve {image_reference=} - does it exist?')
except Exception as e:
raise e
def _mk_credentials(
image_reference,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges],
oa.OciConfig],
privileges: oa.Privileges = None):
if isinstance(image_reference, str):
image_reference = docker_name.from_string(name=image_reference)
try:
# first try container_registry cfgs from available cfg
creds = _credentials(
image_reference=str(image_reference),
credentials_lookup=credentials_lookup,
privileges=privileges,
absent_ok=True,
)
if not creds:
logger.warning(f'could not find rw-creds for {image_reference}')
# fall-back to default docker lookup
creds = docker_creds.DefaultKeychain.Resolve(image_reference)
return creds
except Exception as e:
logger.warning(
f'Error resolving credentials for {image_reference}: {e}')
raise e
def _put_raw_image_manifest(
image_reference: str,
raw_contents: bytes,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool],
oa.OciConfig],
):
image_name = docker_name.from_string(image_reference)
push_sess = docker_session.Push(
name=image_name,
creds=_mk_credentials(
image_reference=image_reference,
credentials_lookup=credentials_lookup,
privileges=oa.Privileges.READWRITE,
),
transport=_mk_transport_pool(),
)
class ImageMock:
def digest(self):
return image_name.tag
def manifest(self):
return raw_contents
def media_type(self):
return docker_http.MANIFEST_SCHEMA2_MIME
image_mock = ImageMock()
push_sess._put_manifest(image=image_mock, use_digest=True)
def tags(
image_name: str,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool],
oa.OciConfig],
) -> typing.Sequence[str]:
'''
returns a sequence of all `tags` for the given image_name
'''
if isinstance(image_name, str):
image_name = ou.normalise_image_reference(image_name)
from containerregistry.client.v2_2 import docker_http
transport = _ou._mk_transport(
image_name=image_name,
credentials_lookup=credentials_lookup,
action=docker_http.PULL,
)
if isinstance(image_name, str):
from containerregistry.client import docker_name
image_name = docker_name.from_string(image_name)
url = f'https://{image_name.registry}/v2/{image_name.repository}/tags/list'
res, body_bytes = transport.Request(url, (200, ))
parsed = json.loads(body_bytes)
# XXX parsed['manifest'] might be used to e.g. determine stale images, and purge them
tags = parsed['tags']
return tags
def _tag_or_digest_reference(image_reference):
if isinstance(image_reference, str):
image_reference = docker_name.from_string(image_reference)
ref_type = type(image_reference)
if ref_type in (docker_name.Tag, docker_name.Digest):
return True
raise ValueError(
f'{image_reference=} is does not contain a symbolic or hash tag')
def rm_tag(image_reference: str):
transport = _mk_transport_pool()
image_reference = ou.normalise_image_reference(image_reference)
image_reference = docker_name.from_string(image_reference)
creds = _mk_credentials(image_reference=image_reference)
docker_session.Delete(
name=image_reference,
creds=creds,
transport=transport,
)
logger.info(f'untagged {image_reference=} - note: did not purge blobs!')
def _image_ref_metadata(
self,
resource: gci.componentmodel.Resource,
omit_version: bool,
):
metadata_dict = {
'IMAGE_REFERENCE_NAME': resource.name,
'RESOURCE_TYPE': resource.type.value,
}
if not omit_version:
image_ref_with_digest = docker_name.from_string(
container.registry.to_hash_reference(
resource.access.imageReference, ))
digest = f'@{image_ref_with_digest.digest}'
metadata_dict['IMAGE_REFERENCE'] = resource.access.imageReference
metadata_dict['IMAGE_VERSION'] = resource.version
metadata_dict['IMAGE_DIGEST'] = digest
metadata_dict['DIGEST_IMAGE_REFERENCE'] = str(
image_ref_with_digest)
return metadata_dict
def _pull_image(
image_reference: str,
credentials_lookup: typing.Callable[[image_reference, oa.Privileges, bool],
oa.OciConfig],
outfileobj=None,
):
if not image_reference:
raise ValueError(image_reference)
image_reference = ou.normalise_image_reference(
image_reference=image_reference)
outfileobj = outfileobj if outfileobj else tempfile.TemporaryFile()
with tarfile.open(fileobj=outfileobj, mode='w:') as tar:
with pulled_image(
image_reference=image_reference,
credentials_lookup=credentials_lookup,
) as image:
image_reference = docker_name.from_string(image_reference)
save.tarball(_make_tag_if_digest(image_reference), image, tar)
return outfileobj
def _push_image(image_reference: str,
image_file: str,
credentials_lookup: typing.Callable[
[image_reference, oa.Privileges, bool], oa.OciConfig],
threads=8):
if not image_reference:
raise ValueError(image_reference)
if not os.path.isfile(image_file):
raise ValueError(f'not an exiting file: {image_file=}')
transport = _mk_transport_pool()
image_reference = ou.normalise_image_reference(image_reference)
image_reference = docker_name.from_string(image_reference)
creds = _mk_credentials(
image_reference=image_reference,
credentials_lookup=credentials_lookup,
privileges=oa.Privileges.READWRITE,
)
# XXX fail if no creds were found
with v2_2_image.FromTarball(image_file) as v2_2_img:
try:
with docker_session.Push(
image_reference,
creds,
transport,
threads=threads,
) as session:
session.upload(v2_2_img)
digest = v2_2_img.digest()
logger.info(
f'{image_reference} was uploaded - digest: {digest}')
except Exception as e:
import traceback
traceback.print_exc()
raise e
def to_hash_reference(image_name: str):
transport = _mk_transport_pool(size=1)
image_name = ou.normalise_image_reference(image_name)
image_reference = docker_name.from_string(image_name)
creds = _mk_credentials(image_reference=image_reference)
accept = docker_http.SUPPORTED_MANIFEST_MIMES
digest = None
with image_list.FromRegistry(image_reference, creds,
transport) as img_list:
if img_list.exists():
digest = img_list.digest()
else:
logger.debug('no manifest found')
# look for image
with v2_2_image.FromRegistry(image_reference, creds, transport,
accept) as v2_2_img:
if v2_2_img.exists():
digest = v2_2_img.digest()
else:
logger.debug('no img v2.2 found')
if not digest:
# fallback to v2
with v2_image.FromRegistry(image_reference, creds,
transport) as v2_img:
if v2_img.exists():
digest = v2_img.digest()
else:
logger.debug('no img v2 found')
raise RuntimeError(
f'could not access img-metadata for {image_name}')
name = image_name.rsplit(':', 1)[0]
return f'{name}@{digest}'
def all(
self,
build_project,
registry_project,
remote_fork, # pylint: disable=too-many-statements,too-many-branches
kustomize_file,
add_github_host=False):
"""Build the latest image and update the prototype.
Args:
build_project: GCP project used to build the image.
registry_project: GCP project used to host the image.
remote_fork: Url of the remote fork.
The remote fork used to create the PR;
e.g. [email protected]:jlewi/kubeflow.git. currently only ssh is
supported.
kustomize_file: Path to the kustomize file
add_github_host: If true will add the github ssh host to known ssh hosts.
"""
# TODO(jlewi): How can we automatically determine the root of the git
# repo containing the kustomize_file?
self.manifests_repo_dir = util.run(
["git", "rev-parse", "--show-toplevel"],
cwd=os.path.dirname(kustomize_file))
repo = git.Repo(self.manifests_repo_dir)
util.maybe_activate_service_account()
last_commit = self.last_commit
# Ensure github.com is in the known hosts
if add_github_host:
output = util.run(["ssh-keyscan", "github.com"])
with open(os.path.join(os.getenv("HOME"), ".ssh", "known_hosts"),
mode='a') as hf:
hf.write(output)
if not remote_fork.startswith("*****@*****.**"):
raise ValueError("Remote fork currently only supports ssh")
remote_repo = self._find_remote_repo(repo, remote_fork)
if not remote_repo:
fork_name = remote_fork.split(":", 1)[-1].split("/", 1)[0]
logging.info("Adding remote %s=%s", fork_name, remote_fork)
remote_repo = repo.create_remote(fork_name, remote_fork)
logging.info("Last change to components-jupyter-web-app was %s",
last_commit)
base = "gcr.io/{0}/jupyter-web-app".format(registry_project)
# Check if there is already an image tagged with this commit.
image = base + ":" + self.last_commit
transport = transport_pool.Http(httplib2.Http)
src = docker_name.from_string(image)
creds = docker_creds.DefaultKeychain.Resolve(src)
image_exists = False
try:
with v2_2_image.FromRegistry(src, creds, transport) as src_image:
logging.info("Image %s exists; digest: %s", image,
src_image.digest())
image_exists = True
except docker_http.V2DiagnosticException as e:
if e.status == 404:
logging.info("%s doesn't exist", image)
else:
raise
if not image_exists:
logging.info("Building the image")
image = self.build_image(build_project, registry_project)
logging.info("Created image: %s", image)
else:
logging.info("Image %s already exists", image)
# TODO(jlewi): What if the file was already modified so we didn't
# modify it in this run but we still need to commit it?
image_updated = application_util.set_kustomize_image(
kustomize_file, JUPYTER_WEB_APP_IMAGE_NAME, image)
if not image_updated:
logging.info("kustomization not updated so not creating a PR.")
return
application_util.regenerate_manifest_tests(self.manifests_repo_dir)
branch_name = "update_jupyter_{0}".format(last_commit)
if repo.active_branch.name != branch_name:
logging.info("Creating branch %s", branch_name)
branch_names = [b.name for b in repo.branches]
if branch_name in branch_names:
logging.info("Branch %s exists", branch_name)
util.run(["git", "checkout", branch_name],
cwd=self.manifests_repo_dir)
else:
util.run(["git", "checkout", "-b", branch_name],
cwd=self.manifests_repo_dir)
if self._check_if_pr_exists(commit=last_commit):
# Since a PR already exists updating to the specified commit
# don't create a new one.
# We don't want to just push -f because if the PR already exists
# git push -f will retrigger the tests.
# To force a recreate of the PR someone could close the existing
# PR and a new PR will be created on the next cron run.
return
logging.info("Add file %s to repo", kustomize_file)
repo.index.add([kustomize_file])
repo.index.add([os.path.join(self.manifests_repo_dir, "tests/*")])
repo.index.commit(
"Update the jupyter web app image to {0}".format(image))
util.run([
"git", "push", "-f", remote_repo.name,
"{0}:{0}".format(branch_name)
],
cwd=self.manifests_repo_dir)
self.create_pull_request(commit=last_commit)
def all(self, build_project, registry_project, remote_fork,
add_github_host=False): # pylint: disable=too-many-statements,too-many-branches
"""Build the latest image and update the prototype.
Args:
build_project: GCP project used to build the image.
registry_project: GCP project used to host the image.
remote_fork: Url of the remote fork.
The remote fork used to create the PR;
e.g. [email protected]:jlewi/kubeflow.git. currently only ssh is
supported.
add_github_host: If true will add the github ssh host to known ssh hosts.
"""
repo = git.Repo(self._root_dir())
util.maybe_activate_service_account()
last_commit = self.last_commit
# Ensure github.com is in the known hosts
if add_github_host:
output = util.run(["ssh-keyscan", "github.com"])
with open(os.path.join(os.getenv("HOME"), ".ssh", "known_hosts"),
mode='a') as hf:
hf.write(output)
if not remote_fork.startswith("*****@*****.**"):
raise ValueError("Remote fork currently only supports ssh")
remote_repo = self._find_remote_repo(repo, remote_fork)
if not remote_repo:
fork_name = remote_fork.split(":", 1)[-1].split("/", 1)[0]
logging.info("Adding remote %s=%s", fork_name, remote_fork)
remote_repo = repo.create_remote(fork_name, remote_fork)
logging.info("Last change to components-jupyter-web-app was %s", last_commit)
base = "gcr.io/{0}/jupyter-web-app".format(registry_project)
# Check if there is already an image tagged with this commit.
image = base + ":" + self.last_commit
transport = transport_pool.Http(httplib2.Http)
src = docker_name.from_string(image)
creds = docker_creds.DefaultKeychain.Resolve(src)
image_exists = False
try:
with v2_2_image.FromRegistry(src, creds, transport) as src_image:
logging.info("Image %s exists; digest: %s", image,
src_image.digest())
image_exists = True
except docker_http.V2DiagnosticException as e:
if e.status == 404:
logging.info("%s doesn't exist", image)
else:
raise
if not image_exists:
logging.info("Building the image")
image = self.build_image(build_project, registry_project)
logging.info("Created image: %s", image)
else:
logging.info("Image %s already exists", image)
# We should check what the current image is if and not update it
# if its the existing image
prototype_file = self.update_prototype(image)
if not prototype_file:
logging.info("Prototype not updated so not creating a PR.")
return
branch_name = "update_jupyter_{0}".format(last_commit)
if repo.active_branch.name != branch_name:
logging.info("Creating branch %s", branch_name)
branch_names = [b.name for b in repo.branches]
if branch_name in branch_names:
logging.info("Branch %s exists", branch_name)
util.run(["git", "checkout", branch_name], cwd=self._root_dir())
else:
util.run(["git", "checkout", "-b", branch_name], cwd=self._root_dir())
logging.info("Add file %s to repo", prototype_file)
repo.index.add([prototype_file])
repo.index.commit("Update the jupyter web app image to {0}".format(image))
util.run(["git", "push", "-f", remote_repo.name], cwd=self._root_dir())
self.create_pull_request(commit=last_commit)
def upload_container_image_group(
self,
resource_group: ResourceGroup,
) -> typing.Iterable[UploadResult]:
mk_upload_result = partial(
UploadResult,
component=resource_group.component(),
)
ci.util.info(
f'Processing resource group for component {resource_group.component().name} and image '
f'{resource_group.image_name()} with {len(resource_group.resources())} resources'
)
# depending on upload-mode, determine an upload-action for each related image
# - images to upload
# - protecode-apps to remove
# - triages to import
images_to_upload = list()
protecode_apps_to_remove = set()
protecode_apps_to_consider = list(
) # consider to rescan; return results
triages_to_import = set()
metadata = self._image_group_metadata(
resource_group=resource_group,
omit_version=True,
)
existing_products = self._api.list_apps(
group_id=self._group_id,
custom_attribs=metadata,
)
# import triages from local group
scan_results = (self._api.scan_result(product_id=product.product_id())
for product in existing_products)
triages_to_import |= set(self._existing_triages(scan_results))
# import triages from reference groups
def enumerate_reference_triages():
for group_id in self._reference_group_ids:
ref_apps = self._api.list_apps(
group_id=group_id,
custom_attribs=metadata,
)
ref_scan_results = (self._api.scan_result(app.product_id())
for app in ref_apps)
yield from self._existing_triages(ref_scan_results)
triages_to_import |= set(enumerate_reference_triages())
ci.util.info(f'found {len(triages_to_import)} triage(s) to import')
if self._processing_mode is ProcessingMode.FORCE_UPLOAD:
ci.util.info('force-upload - will re-upload all images')
images_to_upload += list(resource_group.resources())
# remove all
protecode_apps_to_remove = set(existing_products)
elif self._processing_mode is ProcessingMode.RESCAN:
for resource in resource_group.resources():
# find matching protecode product (aka app)
for existing_product in existing_products:
product_image_digest = existing_product.custom_data().get(
'IMAGE_DIGEST')
container_image_digest = docker_name.from_string(
container.registry.to_hash_reference(
resource.access.imageReference, ))
if product_image_digest == f'@{container_image_digest.digest}':
existing_products.remove(existing_product)
protecode_apps_to_consider.append(existing_product)
break
else:
ci.util.info(
f'did not find image {resource.access.imageReference} - will upload'
)
# not found -> need to upload
images_to_upload.append(resource)
# all existing products that did not match shall be removed
protecode_apps_to_remove |= set(existing_products)
else:
raise NotImplementedError()
# trigger rescan if recommended
for protecode_app in protecode_apps_to_consider:
scan_result = self._api.scan_result_short(
product_id=protecode_app.product_id())
if not scan_result.is_stale():
continue # protecode does not recommend a rescan
if not scan_result.has_binary():
# scan_result here is an AnalysisResult which lacks our metadata. We need the
# metadata to fetch the image version. Therefore, fetch the proper result
scan_result = self._api.scan_result(
product_id=protecode_app.product_id())
image_digest = scan_result.custom_data().get('IMAGE_DIGEST')
# there should be at most one matching image (by image digest)
for resource in resource_group:
container_image_digest = docker_name.from_string(
container.registry.to_hash_reference(
resource.access.imageReference, ))
if image_digest == f'@{container_image_digest.digest}':
ci.util.info(
f'File for container image "{resource.access.imageReference}" no '
'longer available to protecode - will upload')
images_to_upload.append(resource)
protecode_apps_to_consider.remove(protecode_app)
# xxx - also add app for removal?
break
else:
self._api.rescan(protecode_app.product_id())
# upload new images
for resource in images_to_upload:
try:
scan_result = self._upload_image(
component=resource_group.component(),
resource=resource,
)
except requests.exceptions.HTTPError as e:
# in case the image is currently being scanned, Protecode will answer with HTTP
# code 409 ('conflict'). In this case, fetch the ongoing scan to add it
# to the list of scans to consider. In all other cases re-raise the error.
if e.response.status_code != requests.codes.conflict:
raise e
scan_result = self.retrieve_scan_result(
component=resource_group.component(),
resource=resource,
)
protecode_apps_to_consider.append(scan_result)
# wait for all apps currently being scanned
for protecode_app in protecode_apps_to_consider:
# replace - potentially incomplete - scan result
protecode_apps_to_consider.remove(protecode_app)
ci.util.info(f'waiting for {protecode_app.product_id()}')
protecode_apps_to_consider.append(
self._api.wait_for_scan_result(protecode_app.product_id()))
ci.util.info(f'finished waiting for {protecode_app.product_id()}')
# apply imported triages for all protecode apps
for protecode_app in protecode_apps_to_consider:
product_id = protecode_app.product_id()
scan_result = self._api.scan_result(product_id)
existing_triages = list(self._existing_triages([scan_result]))
new_triages = [
t for t in triages_to_import if t not in existing_triages
]
ci.util.info(
f'transporting triages for {protecode_app.product_id()}')
self._transport_triages(new_triages, product_id)
ci.util.info(
f'done with transporting triages for {protecode_app.product_id()}'
)
# apply triages from GCR
protecode_apps_to_consider = [
self._import_triages_from_gcr(protecode_app)
for protecode_app in protecode_apps_to_consider
]
# yield results
for protecode_app in protecode_apps_to_consider:
scan_result = self._api.scan_result(protecode_app.product_id())
# create closure for pdf retrieval to avoid actually having to store
# all the pdf-reports in memory. Will be called when preparing to send
# the notification emails if reports are to be included
def pdf_retrieval_function():
return self._api.pdf_report(protecode_app.product_id())
yield mk_upload_result(
status=UploadStatus.DONE, # XXX remove this
result=scan_result,
resource=resource,
pdf_report_retrieval_func=pdf_retrieval_function,
)
# rm all outdated protecode apps
for protecode_app in protecode_apps_to_remove:
product_id = protecode_app.product_id()
self._api.delete_product(product_id=product_id)
ci.util.info(f'purged outdated product {product_id}')