def patch_dockerfile(dockerfile, images):
"""
Patch an existing Dockerfile to replace FROM image statements
by their local images with digests. It supports multi-stage images
This is needed to avoid retrieving remote images before checking current img state
Bug https://github.com/genuinetools/img/issues/206
"""
assert os.path.exists(dockerfile), "Missing dockerfile {}".format(
dockerfile)
assert isinstance(images, list)
if not images:
return
def _find_replacement(original):
# Replace an image name by its local version
# when it exists
repo, tag = parse_image_name(original)
for image in images:
if image["repository"] == repo and image["tag"] == tag:
if image["registry"]:
local = "{}/{}@sha256:{}".format(image["registry"],
image["repository"],
image["digest"])
else:
local = "{}@sha256:{}".format(image["repository"],
image["digest"])
logger.info("Replacing image {} by {}".format(original, local))
return local
return original
# Parse the given dockerfile and update its parent images
# with local version given by current img state
# The FROM statement parsing & replacement is provided
# by the DockerfileParser
parser = DockerfileParser()
parser.dockerfile_path = dockerfile
parser.content = open(dockerfile).read()
logger.info("Initial parent images: {}".format(" & ".join(
parser.parent_images)))
parser.parent_images = list(map(_find_replacement, parser.parent_images))
def run(self):
builder = self.workflow.builder
dfp = DockerfileParser(builder.df_path)
df_base = dfp.baseimage
build_base = builder.base_image.to_str()
# do some sanity checks to defend against bugs and rogue plugins
if build_base != builder.parent_images[df_base]:
# something updated parent_images entry for base without updating
# the build's base_image; treat it as an error
raise BaseImageMismatch(
"Parent image '{}' does not match base_image '{}'".format(
builder.parent_images[df_base], build_base))
unresolved = [
key for key, val in builder.parent_images.items() if not val
]
if unresolved:
# this would generally mean pull_base_image didn't run and/or
# custom plugins modified parent_images; treat it as an error.
raise ParentImageUnresolved(
"Parent image(s) unresolved: {}".format(unresolved))
missing = [
df_img for df_img in dfp.parent_images
if df_img not in builder.parent_images
]
if missing:
# this would indicate another plugin modified parent_images out of sync
# with the Dockerfile or some other code bug
raise ParentImageMissing(
"Lost parent image(s) from Dockerfile: {}".format(missing))
# docker inspect all parent images so we can address them by Id
parent_image_ids = {}
for img, new_img in builder.parent_images.items():
inspection = builder.tasker.inspect_image(new_img)
try:
parent_image_ids[img] = inspection['Id']
except KeyError: # unexpected code bugs or maybe docker weirdness
self.log.error(
"Id for image %s is missing in inspection: '%s'", new_img,
inspection)
raise NoIdInspection("Could not inspect Id for image " +
new_img)
# update the parents in Dockerfile
new_parents = []
for parent in dfp.parent_images:
pid = parent_image_ids[parent]
self.log.info("changed FROM: '%s' -> '%s'", parent, pid)
new_parents.append(pid)
dfp.parent_images = new_parents
# update builder's representation of what will be built
builder.parent_images = parent_image_ids
builder.set_base_image(parent_image_ids[df_base])
self.log.debug("for base image '%s' using local image '%s', id '%s'",
df_base, build_base, builder.base_image)
def images_streams_prs(runtime, github_access_token, bug, interstitial,
ignore_ci_master, draft_prs, moist_run, add_labels):
runtime.initialize(clone_distgits=False, clone_source=False)
g = Github(login_or_token=github_access_token)
github_user = g.get_user()
major = runtime.group_config.vars['MAJOR']
minor = runtime.group_config.vars['MINOR']
interstitial = int(interstitial)
master_major, master_minor = extract_version_fields(what_is_in_master(),
at_least=2)
if not ignore_ci_master and (major > master_major or minor > master_minor):
# ART building a release before is is in master. Too early to open PRs.
runtime.logger.warning(
f'Target {major}.{minor} has not been in master yet (it is tracking {master_major}.{master_minor}); skipping PRs'
)
exit(0)
prs_in_master = (major == master_major
and minor == master_minor) and not ignore_ci_master
pr_links = {} # map of distgit_key to PR URLs associated with updates
new_pr_links = {}
skipping_dgks = set(
) # If a distgit key is skipped, it children will see it in this list and skip themselves.
for image_meta in runtime.ordered_image_metas():
dgk = image_meta.distgit_key
logger = image_meta.logger
logger.info('Analyzing image')
alignment_prs_config = image_meta.config.content.source.ci_alignment.streams_prs
if alignment_prs_config and alignment_prs_config.enabled is not Missing and not alignment_prs_config.enabled:
# Make sure this is an explicit False. Missing means the default or True.
logger.info('The image has alignment PRs disabled; ignoring')
continue
from_config = image_meta.config['from']
if not from_config:
logger.info('Skipping PRs since there is no configured .from')
continue
desired_parents = []
builders = from_config.builder or []
for builder in builders:
upstream_image = resolve_upstream_from(runtime, builder)
if not upstream_image:
logger.warning(
f'Unable to resolve upstream image for: {builder}')
break
desired_parents.append(upstream_image)
parent_upstream_image = resolve_upstream_from(runtime, from_config)
if len(desired_parents) != len(builders) or not parent_upstream_image:
logger.warning(
'Unable to find all ART equivalent upstream images for this image'
)
continue
desired_parents.append(parent_upstream_image)
desired_parent_digest = calc_parent_digest(desired_parents)
logger.info(
f'Found desired FROM state of: {desired_parents} with digest: {desired_parent_digest}'
)
source_repo_url, source_repo_branch = _get_upstream_source(
runtime, image_meta)
if not source_repo_url:
# No upstream to clone; no PRs to open
continue
public_repo_url, public_branch = runtime.get_public_upstream(
source_repo_url)
if not public_branch:
public_branch = source_repo_branch
# There are two standard upstream branching styles:
# release-4.x : CI fast-forwards from master when appropriate
# openshift-4.x : Upstream team manages completely.
# For the former style, we may need to open the PRs against master.
# For the latter style, always open directly against named branch
if public_branch.startswith('release-') and prs_in_master:
# TODO: auto-detect default branch for repo instead of assuming master
public_branch = 'master'
_, org, repo_name = split_git_url(public_repo_url)
public_source_repo = g.get_repo(f'{org}/{repo_name}')
try:
fork_repo_name = f'{github_user.login}/{repo_name}'
fork_repo = g.get_repo(fork_repo_name)
except UnknownObjectException:
# Repo doesn't exist; fork it
fork_repo = github_user.create_fork(public_source_repo)
fork_branch_name = f'art-consistency-{runtime.group_config.name}-{dgk}'
fork_branch_head = f'{github_user.login}:{fork_branch_name}'
fork_branch = None
try:
fork_branch = fork_repo.get_branch(fork_branch_name)
except UnknownObjectException:
# Doesn't presently exist and will need to be created
pass
except GithubException as ge:
# This API seems to return 404 instead of UnknownObjectException.
# So allow 404 to pass through as well.
if ge.status != 404:
raise
public_repo_url = convert_remote_git_to_ssh(public_repo_url)
clone_dir = os.path.join(runtime.working_dir, 'clones', dgk)
# Clone the private url to make the best possible use of our doozer_cache
runtime.git_clone(source_repo_url, clone_dir)
with Dir(clone_dir):
exectools.cmd_assert(f'git remote add public {public_repo_url}')
exectools.cmd_assert(
f'git remote add fork {convert_remote_git_to_ssh(fork_repo.git_url)}'
)
exectools.cmd_assert('git fetch --all')
# The path to the Dockerfile in the target branch
if image_meta.config.content.source.dockerfile is not Missing:
# Be aware that this attribute sometimes contains path elements too.
dockerfile_name = image_meta.config.content.source.dockerfile
else:
dockerfile_name = "Dockerfile"
df_path = Dir.getpath()
if image_meta.config.content.source.path:
dockerfile_name = os.path.join(
image_meta.config.content.source.path, dockerfile_name)
df_path = df_path.joinpath(dockerfile_name)
fork_branch_parent_digest = None
fork_branch_parents = None
if fork_branch:
# If there is already an art reconciliation branch, get an MD5
# of the FROM images in the Dockerfile in that branch.
exectools.cmd_assert(f'git checkout fork/{fork_branch_name}')
fork_branch_parent_digest, fork_branch_parents = extract_parent_digest(
df_path)
# Now change over to the target branch in the actual public repo
exectools.cmd_assert(f'git checkout public/{public_branch}')
source_branch_parent_digest, source_branch_parents = extract_parent_digest(
df_path)
if desired_parent_digest == source_branch_parent_digest:
green_print(
'Desired digest and source digest match; Upstream is in a good state'
)
continue
yellow_print(
f'Upstream dockerfile does not match desired state in {public_repo_url}/blob/{public_branch}/{dockerfile_name}'
)
print(
f'Desired parents: {desired_parents} ({desired_parent_digest})'
)
print(
f'Source parents: {source_branch_parents} ({source_branch_parent_digest})'
)
print(
f'Fork branch digest: {fork_branch_parents} ({fork_branch_parent_digest})'
)
first_commit_line = f"Updating {image_meta.name} builder & base images to be consistent with ART"
reconcile_info = f"Reconciling with {convert_remote_git_to_https(runtime.gitdata.origin_url)}/tree/{runtime.gitdata.commit_hash}/images/{os.path.basename(image_meta.config_filename)}"
diff_text = None
if fork_branch_parent_digest != desired_parent_digest:
# The fork branch either does not exist, or does not have the desired parent image state
# Let's create a local branch that will contain the Dockerfile in the state we desire.
work_branch_name = '__mod'
exectools.cmd_assert(f'git checkout public/{public_branch}')
exectools.cmd_assert(f'git checkout -b {work_branch_name}')
with df_path.open(mode='r+') as handle:
dfp = DockerfileParser(cache_content=True,
fileobj=io.BytesIO())
dfp.content = handle.read()
dfp.parent_images = desired_parents
handle.truncate(0)
handle.seek(0)
handle.write(dfp.content)
diff_text, _ = exectools.cmd_assert(f'git diff {str(df_path)}')
if not moist_run:
exectools.cmd_assert(f'git add {str(df_path)}')
commit_prefix = ''
if repo_name.startswith('kubernetes'):
# couple repos have this requirement; openshift/kubernetes & openshift/kubernetes-autoscaler.
# This check may suffice for now, but it may eventually need to be in doozer metadata.
commit_prefix = 'UPSTREAM: <carry>: '
commit_msg = f"""{commit_prefix}{first_commit_line}
{reconcile_info}
"""
exectools.cmd_assert(
f'git commit -m "{commit_msg}"'
) # Add a commit atop the public branch's current state
# Create or update the remote fork branch
exectools.cmd_assert(
f'git push --force fork {work_branch_name}:{fork_branch_name}'
)
# At this point, we have a fork branch in the proper state
pr_body = f"""{first_commit_line}
{reconcile_info}
If you have any questions about this pull request, please reach out to `@art-team` in the `#aos-art` coreos slack channel.
"""
parent_pr_url = None
parent_meta = image_meta.resolve_parent()
if parent_meta:
if parent_meta.distgit_key in skipping_dgks:
skipping_dgks.add(image_meta.distgit_key)
yellow_print(
f'Image has parent {parent_meta.distgit_key} which was skipped; skipping self: {image_meta.distgit_key}'
)
continue
parent_pr_url = pr_links.get(parent_meta.distgit_key, None)
if parent_pr_url:
if parent_meta.config.content.source.ci_alignment.streams_prs.merge_first:
skipping_dgks.add(image_meta.distgit_key)
yellow_print(
f'Image has parent {parent_meta.distgit_key} open PR ({parent_pr_url}) and streams_prs.merge_first==True; skipping PR opening for this image {image_meta.distgit_key}'
)
continue
# If the parent has an open PR associated with it, make sure the
# child PR notes that the parent PR should merge first.
pr_body += f'\nDepends on {parent_pr_url} . Allow it to merge and then run `/test all` on this PR.'
# Let's see if there is a PR opened
open_prs = list(
public_source_repo.get_pulls(state='open',
head=fork_branch_head))
if open_prs:
existing_pr = open_prs[0]
# Update body, but never title; The upstream team may need set something like a Bug XXXX: there.
# Don't muck with it.
if alignment_prs_config.auto_label and add_labels:
# If we are to automatically add labels to this upstream PR, do so.
existing_pr.set_labels(*alignment_prs_config.auto_label)
existing_pr.edit(body=pr_body)
pr_url = existing_pr.html_url
pr_links[dgk] = pr_url
yellow_print(
f'A PR is already open requesting desired reconciliation with ART: {pr_url}'
)
continue
# Otherwise, we need to create a pull request
if moist_run:
pr_links[dgk] = f'MOIST-RUN-PR:{dgk}'
green_print(
f'Would have opened PR against: {public_source_repo.html_url}/blob/{public_branch}/{dockerfile_name}.'
)
if parent_pr_url:
green_print(
f'Would have identified dependency on PR: {parent_pr_url}.'
)
if diff_text:
yellow_print(diff_text)
else:
yellow_print(
f'Fork from which PR would be created ({fork_branch_head}) is populated with desired state.'
)
else:
pr_title = first_commit_line
if bug:
pr_title = f'Bug {bug}: {pr_title}'
new_pr = public_source_repo.create_pull(title=pr_title,
body=pr_body,
base=public_branch,
head=fork_branch_head,
draft=draft_prs)
if alignment_prs_config.auto_label and add_labels:
# If we are to automatically add labels to this upstream PR, do so.
new_pr.set_labels(*alignment_prs_config.auto_label)
pr_msg = f'A new PR has been opened: {new_pr.html_url}'
pr_links[dgk] = new_pr.html_url
new_pr_links[dgk] = new_pr.html_url
logger.info(pr_msg)
yellow_print(pr_msg)
print(
f'Sleeping {interstitial} seconds before opening another PR to prevent flooding prow...'
)
time.sleep(interstitial)
if new_pr_links:
print('Newly opened PRs:')
print(yaml.safe_dump(new_pr_links))
if pr_links:
print('Currently open PRs:')
print(yaml.safe_dump(pr_links))
