class KojiPromotePlugin(ExitPlugin):
"""
Promote this build to Koji
Submits a successful build to Koji using the Content Generator API,
https://fedoraproject.org/wiki/Koji/ContentGenerators
Authentication is with Kerberos unless the koji_ssl_certs
configuration parameter is given, in which case it should be a
path at which 'cert', 'ca', and 'serverca' are the certificates
for SSL authentication.
If Kerberos is used for authentication, the default principal will
be used (from the kernel keyring) unless both koji_keytab and
koji_principal are specified. The koji_keytab parameter is a
keytab name like 'type:name', and so can be used to specify a key
in a Kubernetes secret by specifying 'FILE:/path/to/key'.
If metadata_only is set, the 'docker save' image will not be
uploaded, only the logs. The import will be marked as
metadata-only.
Runs as an exit plugin in order to capture logs from all other
plugins.
"""
key = PLUGIN_KOJI_PROMOTE_PLUGIN_KEY
is_allowed_to_fail = False
def __init__(self, tasker, workflow, kojihub, url,
verify_ssl=True, use_auth=True,
koji_ssl_certs=None, koji_proxy_user=None,
koji_principal=None, koji_keytab=None,
metadata_only=False, blocksize=None,
target=None, poll_interval=5):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param kojihub: string, koji hub (xmlrpc)
:param url: string, URL for OSv3 instance
:param verify_ssl: bool, verify OSv3 SSL certificate?
:param use_auth: bool, initiate authentication with OSv3?
:param koji_ssl_certs: str, path to 'cert', 'ca', 'serverca'
:param koji_proxy_user: str, user to log in as (requires hub config)
:param koji_principal: str, Kerberos principal (must specify keytab)
:param koji_keytab: str, keytab name (must specify principal)
:param metadata_only: bool, whether to omit the 'docker save' image
:param blocksize: int, blocksize to use for uploading files
:param target: str, koji target
:param poll_interval: int, seconds between Koji task status requests
"""
super(KojiPromotePlugin, self).__init__(tasker, workflow)
self.kojihub = kojihub
self.koji_ssl_certs = koji_ssl_certs
self.koji_proxy_user = koji_proxy_user
self.koji_principal = koji_principal
self.koji_keytab = koji_keytab
self.metadata_only = metadata_only
self.blocksize = blocksize
self.target = target
self.poll_interval = poll_interval
self.namespace = get_build_json().get('metadata', {}).get('namespace', None)
osbs_conf = Configuration(conf_file=None, openshift_uri=url,
use_auth=use_auth, verify_ssl=verify_ssl,
namespace=self.namespace)
self.osbs = OSBS(osbs_conf, osbs_conf)
self.build_id = None
self.pullspec_image = None
@staticmethod
def parse_rpm_output(output, tags, separator=';'):
"""
Parse output of the rpm query.
:param output: list, decoded output (str) from the rpm subprocess
:param tags: list, str fields used for query output
:return: list, dicts describing each rpm package
"""
def field(tag):
"""
Get a field value by name
"""
try:
value = fields[tags.index(tag)]
except ValueError:
return None
if value == '(none)':
return None
return value
components = []
sigmarker = 'Key ID '
for rpm in output:
fields = rpm.rstrip('\n').split(separator)
if len(fields) < len(tags):
continue
signature = field('SIGPGP:pgpsig') or field('SIGGPG:pgpsig')
if signature:
parts = signature.split(sigmarker, 1)
if len(parts) > 1:
signature = parts[1]
component_rpm = {
'type': 'rpm',
'name': field('NAME'),
'version': field('VERSION'),
'release': field('RELEASE'),
'arch': field('ARCH'),
'sigmd5': field('SIGMD5'),
'signature': signature,
}
# Special handling for epoch as it must be an integer or None
epoch = field('EPOCH')
if epoch is not None:
epoch = int(epoch)
component_rpm['epoch'] = epoch
if component_rpm['name'] != 'gpg-pubkey':
components.append(component_rpm)
return components
def get_rpms(self):
"""
Build a list of installed RPMs in the format required for the
metadata.
"""
tags = [
'NAME',
'VERSION',
'RELEASE',
'ARCH',
'EPOCH',
'SIGMD5',
'SIGPGP:pgpsig',
'SIGGPG:pgpsig',
]
sep = ';'
fmt = sep.join(["%%{%s}" % tag for tag in tags])
cmd = "/bin/rpm -qa --qf '{0}\n'".format(fmt)
try:
# py3
(status, output) = subprocess.getstatusoutput(cmd)
except AttributeError:
# py2
with open('/dev/null', 'r+') as devnull:
p = subprocess.Popen(cmd,
shell=True,
stdin=devnull,
stdout=subprocess.PIPE,
stderr=devnull)
(stdout, stderr) = p.communicate()
status = p.wait()
output = stdout.decode()
if status != 0:
self.log.debug("%s: stderr output: %s", cmd, stderr)
raise RuntimeError("%s: exit code %s" % (cmd, status))
return self.parse_rpm_output(output.splitlines(), tags, separator=sep)
def get_output_metadata(self, path, filename):
"""
Describe a file by its metadata.
:return: dict
"""
checksums = get_checksums(path, ['md5'])
metadata = {'filename': filename,
'filesize': os.path.getsize(path),
'checksum': checksums['md5sum'],
'checksum_type': 'md5'}
if self.metadata_only:
metadata['metadata_only'] = True
return metadata
def get_builder_image_id(self):
"""
Find out the docker ID of the buildroot image we are in.
"""
try:
buildroot_tag = os.environ["OPENSHIFT_CUSTOM_BUILD_BASE_IMAGE"]
except KeyError:
return ''
try:
pod = self.osbs.get_pod_for_build(self.build_id)
all_images = pod.get_container_image_ids()
except OsbsException as ex:
self.log.error("unable to find image id: %r", ex)
return buildroot_tag
try:
return all_images[buildroot_tag]
except KeyError:
self.log.error("Unable to determine buildroot image ID for %s",
buildroot_tag)
return buildroot_tag
def get_buildroot(self, build_id):
"""
Build the buildroot entry of the metadata.
:return: dict, partial metadata
"""
docker_info = self.tasker.get_info()
host_arch, docker_version = get_docker_architecture(self.tasker)
buildroot = {
'id': 1,
'host': {
'os': docker_info['OperatingSystem'],
'arch': host_arch,
},
'content_generator': {
'name': PROG,
'version': atomic_reactor_version,
},
'container': {
'type': 'docker',
'arch': os.uname()[4],
},
'tools': [
{
'name': tool['name'],
'version': tool['version'],
}
for tool in get_version_of_tools()] + [
{
'name': 'docker',
'version': docker_version,
},
],
'components': self.get_rpms(),
'extra': {
'osbs': {
'build_id': build_id,
'builder_image_id': self.get_builder_image_id(),
}
},
}
return buildroot
def get_logs(self):
"""
Build the logs entry for the metadata 'output' section
:return: list, Output instances
"""
output = []
# Collect logs from server
try:
logs = self.osbs.get_build_logs(self.build_id)
except OsbsException as ex:
self.log.error("unable to get build logs: %r", ex)
else:
# Deleted once closed
logfile = NamedTemporaryFile(prefix=self.build_id,
suffix=".log",
mode='wb')
try:
logfile.write(logs)
except (TypeError, UnicodeEncodeError):
# Older osbs-client versions returned Unicode objects
logfile.write(logs.encode('utf-8'))
logfile.flush()
metadata = self.get_output_metadata(logfile.name,
"openshift-final.log")
output.append(Output(file=logfile, metadata=metadata))
docker_logs = NamedTemporaryFile(prefix="docker-%s" % self.build_id,
suffix=".log",
mode='wb')
docker_logs.write("\n".join(self.workflow.build_result.logs).encode('utf-8'))
docker_logs.flush()
output.append(Output(file=docker_logs,
metadata=self.get_output_metadata(docker_logs.name,
"build.log")))
return output
def get_image_components(self):
"""
Re-package the output of the rpmqa plugin into the format required
for the metadata.
"""
try:
output = self.workflow.postbuild_results[PostBuildRPMqaPlugin.key]
except KeyError:
self.log.error("%s plugin did not run!",
PostBuildRPMqaPlugin.key)
return []
try:
sep = PostBuildRPMqaPlugin.sep
except AttributeError:
# sep instance variable added in Aug 2016
sep = ','
return self.parse_rpm_output(output, PostBuildRPMqaPlugin.rpm_tags,
separator=sep)
def get_image_output(self, arch):
"""
Create the output for the image
This is the Koji Content Generator metadata, along with the
'docker save' output to upload.
For metadata-only builds, an empty file is used instead of the
output of 'docker save'.
:param arch: str, architecture for this output
:return: tuple, (metadata dict, Output instance)
"""
image_id = self.workflow.builder.image_id
saved_image = self.workflow.exported_image_sequence[-1].get('path')
ext = saved_image.split('.', 1)[1]
name_fmt = 'docker-image-{id}.{arch}.{ext}'
image_name = name_fmt.format(id=image_id, arch=arch, ext=ext)
if self.metadata_only:
metadata = self.get_output_metadata(os.path.devnull, image_name)
output = Output(file=None, metadata=metadata)
else:
metadata = self.get_output_metadata(saved_image, image_name)
output = Output(file=open(saved_image), metadata=metadata)
return metadata, output
def get_digests(self):
"""
Returns a map of repositories to digests
"""
digests = {} # repository -> digest
for registry in self.workflow.push_conf.docker_registries:
for image in self.workflow.tag_conf.images:
image_str = image.to_str()
if image_str in registry.digests:
# pulp/crane supports only manifest schema v1
if self.workflow.push_conf.pulp_registries:
digest = registry.digests[image_str].v1
else:
digest = registry.digests[image_str].default
digests[image.to_str(registry=False)] = digest
return digests
def get_repositories(self, digests):
"""
Build the repositories metadata
:param digests: dict, repository -> digest
"""
if self.workflow.push_conf.pulp_registries:
# If pulp was used, only report pulp images
registries = self.workflow.push_conf.pulp_registries
else:
# Otherwise report all the images we pushed
registries = self.workflow.push_conf.all_registries
output_images = []
for registry in registries:
image = self.pullspec_image.copy()
image.registry = registry.uri
pullspec = image.to_str()
output_images.append(pullspec)
digest = digests.get(image.to_str(registry=False))
if digest:
digest_pullspec = image.to_str(tag=False) + "@" + digest
output_images.append(digest_pullspec)
return output_images
def get_output(self, buildroot_id):
"""
Build the 'output' section of the metadata.
:return: list, Output instances
"""
def add_buildroot_id(output):
logfile, metadata = output
metadata.update({'buildroot_id': buildroot_id})
return Output(file=logfile, metadata=metadata)
def add_log_type(output):
logfile, metadata = output
metadata.update({'type': 'log', 'arch': 'noarch'})
return Output(file=logfile, metadata=metadata)
output_files = [add_log_type(add_buildroot_id(metadata))
for metadata in self.get_logs()]
# Parent of squashed built image is base image
image_id = self.workflow.builder.image_id
parent_id = self.workflow.base_image_inspect['Id']
# Read config from the registry using v2 schema 2 digest
registries = self.workflow.push_conf.docker_registries
if registries:
config = copy.deepcopy(registries[0].config)
else:
config = {}
# We don't need container_config section
if config and 'container_config' in config:
del config['container_config']
digests = self.get_digests()
repositories = self.get_repositories(digests)
arch = os.uname()[4]
tags = set(image.tag for image in self.workflow.tag_conf.primary_images)
metadata, output = self.get_image_output(arch)
metadata.update({
'arch': arch,
'type': 'docker-image',
'components': self.get_image_components(),
'extra': {
'image': {
'arch': arch,
},
'docker': {
'id': image_id,
'parent_id': parent_id,
'repositories': repositories,
'tags': list(tags),
'config': config
},
},
})
if not config:
del metadata['extra']['docker']['config']
# Add the 'docker save' image to the output
image = add_buildroot_id(output)
output_files.append(image)
return output_files
def get_build(self, metadata):
start_time = int(atomic_reactor_start_time)
labels = df_parser(self.workflow.builder.df_path, workflow=self.workflow).labels
component = get_preferred_label(labels, 'com.redhat.component')
version = get_preferred_label(labels, 'version')
release = get_preferred_label(labels, 'release')
source = self.workflow.source
if not isinstance(source, GitSource):
raise RuntimeError('git source required')
extra = {'image': {'autorebuild': is_rebuild(self.workflow)}}
koji_task_id = metadata.get('labels', {}).get('koji-task-id')
if koji_task_id is not None:
self.log.info("build configuration created by Koji Task ID %s",
koji_task_id)
try:
extra['container_koji_task_id'] = int(koji_task_id)
except ValueError:
self.log.error("invalid task ID %r", koji_task_id, exc_info=1)
fs_result = self.workflow.prebuild_results.get(AddFilesystemPlugin.key)
if fs_result is not None:
try:
fs_task_id = fs_result['filesystem-koji-task-id']
except KeyError:
self.log.error("%s: expected filesystem-koji-task-id in result",
AddFilesystemPlugin.key)
else:
try:
task_id = int(fs_task_id)
except ValueError:
self.log.error("invalid task ID %r", fs_task_id, exc_info=1)
else:
extra['filesystem_koji_task_id'] = task_id
help_result = self.workflow.prebuild_results.get(AddHelpPlugin.key)
if isinstance(help_result, dict) and 'help_file' in help_result and 'status' in help_result:
if help_result['status'] == AddHelpPlugin.NO_HELP_FILE_FOUND:
extra['image']['help'] = None
elif help_result['status'] == AddHelpPlugin.HELP_GENERATED:
extra['image']['help'] = help_result['help_file']
else:
self.log.error("Unknown result from add_help plugin: %s", help_result)
build = {
'name': component,
'version': version,
'release': release,
'source': "{0}#{1}".format(source.uri, source.commit_id),
'start_time': start_time,
'end_time': int(time.time()),
'extra': extra,
}
if self.metadata_only:
build['metadata_only'] = True
return build
def get_metadata(self):
"""
Build the metadata needed for importing the build
:return: tuple, the metadata and the list of Output instances
"""
try:
metadata = get_build_json()["metadata"]
self.build_id = metadata["name"]
except KeyError:
self.log.error("No build metadata")
raise
for image in self.workflow.tag_conf.unique_images:
self.pullspec_image = image
break
for image in self.workflow.tag_conf.primary_images:
# dash at first/last postition does not count
if '-' in image.tag[1:-1]:
self.pullspec_image = image
break
if not self.pullspec_image:
raise RuntimeError('Unable to determine pullspec_image')
metadata_version = 0
build = self.get_build(metadata)
buildroot = self.get_buildroot(build_id=self.build_id)
output_files = self.get_output(buildroot['id'])
koji_metadata = {
'metadata_version': metadata_version,
'build': build,
'buildroots': [buildroot],
'output': [output.metadata for output in output_files],
}
return koji_metadata, output_files
def upload_file(self, session, output, serverdir):
"""
Upload a file to koji
:return: str, pathname on server
"""
name = output.metadata['filename']
self.log.debug("uploading %r to %r as %r",
output.file.name, serverdir, name)
kwargs = {}
if self.blocksize is not None:
kwargs['blocksize'] = self.blocksize
self.log.debug("using blocksize %d", self.blocksize)
upload_logger = KojiUploadLogger(self.log)
session.uploadWrapper(output.file.name, serverdir, name=name,
callback=upload_logger.callback, **kwargs)
path = os.path.join(serverdir, name)
self.log.debug("uploaded %r", path)
return path
@staticmethod
def get_upload_server_dir():
"""
Create a path name for uploading files to
:return: str, path name expected to be unique
"""
dir_prefix = 'koji-promote'
random_chars = ''.join([random.choice(ascii_letters)
for _ in range(8)])
unique_fragment = '%r.%s' % (time.time(), random_chars)
return os.path.join(dir_prefix, unique_fragment)
def login(self):
"""
Log in to koji
:return: koji.ClientSession instance, logged in
"""
# krbV python library throws an error if these are unicode
auth_info = {
"proxyuser": self.koji_proxy_user,
"ssl_certs_dir": self.koji_ssl_certs,
"krb_principal": str(self.koji_principal),
"krb_keytab": str(self.koji_keytab)
}
return create_koji_session(str(self.kojihub), auth_info)
def run(self):
"""
Run the plugin.
"""
if ((self.koji_principal and not self.koji_keytab) or
(self.koji_keytab and not self.koji_principal)):
raise RuntimeError("specify both koji_principal and koji_keytab "
"or neither")
# Only run if the build was successful
if self.workflow.build_process_failed:
self.log.info("Not promoting failed build to koji")
return
koji_metadata, output_files = self.get_metadata()
try:
session = self.login()
server_dir = self.get_upload_server_dir()
for output in output_files:
if output.file:
self.upload_file(session, output, server_dir)
finally:
for output in output_files:
if output.file:
output.file.close()
try:
build_info = session.CGImport(koji_metadata, server_dir)
except Exception:
self.log.debug("metadata: %r", koji_metadata)
raise
# Older versions of CGImport do not return a value.
build_id = build_info.get("id") if build_info else None
self.log.debug("Build information: %s",
json.dumps(build_info, sort_keys=True, indent=4))
# If configured, koji_tag_build plugin will perform build tagging
tag_later = are_plugins_in_order(self.workflow.exit_plugins_conf,
PLUGIN_KOJI_PROMOTE_PLUGIN_KEY,
PLUGIN_KOJI_TAG_BUILD_KEY)
if not tag_later and build_id is not None and self.target is not None:
tag_koji_build(session, build_id, self.target,
poll_interval=self.poll_interval)
return build_id
class KojiPromotePlugin(ExitPlugin):
"""
Promote this build to Koji
Submits a successful build to Koji using the Content Generator API,
https://fedoraproject.org/wiki/Koji/ContentGenerators
Authentication is with Kerberos unless the koji_ssl_certs
configuration parameter is given, in which case it should be a
path at which 'cert', 'ca', and 'serverca' are the certificates
for SSL authentication.
If Kerberos is used for authentication, the default principal will
be used (from the kernel keyring) unless both koji_keytab and
koji_principal are specified. The koji_keytab parameter is a
keytab name like 'type:name', and so can be used to specify a key
in a Kubernetes secret by specifying 'FILE:/path/to/key'.
If metadata_only is set, the v1 image will not be uploaded, only
the logs. The import will be marked as metadata-only.
Runs as an exit plugin in order to capture logs from all other
plugins.
"""
key = "koji_promote"
is_allowed_to_fail = False
def __init__(self, tasker, workflow, kojihub, url,
verify_ssl=True, use_auth=True,
koji_ssl_certs=None, koji_proxy_user=None,
koji_principal=None, koji_keytab=None,
metadata_only=False):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param kojihub: string, koji hub (xmlrpc)
:param url: string, URL for OSv3 instance
:param verify_ssl: bool, verify OSv3 SSL certificate?
:param use_auth: bool, initiate authentication with OSv3?
:param koji_ssl_certs: str, path to 'cert', 'ca', 'serverca'
:param koji_proxy_user: str, user to log in as (requires hub config)
:param koji_principal: str, Kerberos principal (must specify keytab)
:param koji_keytab: str, keytab name (must specify principal)
:param metadata_only: bool, whether to omit the v1 image
"""
super(KojiPromotePlugin, self).__init__(tasker, workflow)
self.kojihub = kojihub
self.koji_ssl_certs = koji_ssl_certs
self.koji_proxy_user = koji_proxy_user
self.koji_principal = koji_principal
self.koji_keytab = koji_keytab
self.metadata_only = metadata_only
osbs_conf = Configuration(conf_file=None, openshift_uri=url,
use_auth=use_auth, verify_ssl=verify_ssl)
self.osbs = OSBS(osbs_conf, osbs_conf)
self.build_id = None
self.namespace = None
@staticmethod
def parse_rpm_output(output, tags, separator=';'):
"""
Parse output of the rpm query.
:param output: list, decoded output (str) from the rpm subprocess
:param tags: list, str fields used for query output
:return: list, dicts describing each rpm package
"""
def field(tag):
"""
Get a field value by name
"""
try:
value = fields[tags.index(tag)]
except ValueError:
return None
if value == '(none)':
return None
return value
components = []
sigmarker = 'Key ID '
for rpm in output:
fields = rpm.rstrip('\n').split(separator)
if len(fields) < len(tags):
continue
signature = field('SIGPGP:pgpsig') or field('SIGGPG:pgpsig')
if signature:
parts = signature.split(sigmarker, 1)
if len(parts) > 1:
signature = parts[1]
component_rpm = {
'type': 'rpm',
'name': field('NAME'),
'version': field('VERSION'),
'release': field('RELEASE'),
'arch': field('ARCH'),
'sigmd5': field('SIGMD5'),
'signature': signature,
}
# Special handling for epoch as it must be an integer or None
epoch = field('EPOCH')
if epoch is not None:
epoch = int(epoch)
component_rpm['epoch'] = epoch
if component_rpm['name'] != 'gpg-pubkey':
components.append(component_rpm)
return components
def get_rpms(self):
"""
Build a list of installed RPMs in the format required for the
metadata.
"""
tags = [
'NAME',
'VERSION',
'RELEASE',
'ARCH',
'EPOCH',
'SIGMD5',
'SIGPGP:pgpsig',
'SIGGPG:pgpsig',
]
sep = ';'
fmt = sep.join(["%%{%s}" % tag for tag in tags])
cmd = "/bin/rpm -qa --qf '{0}\n'".format(fmt)
try:
# py3
(status, output) = subprocess.getstatusoutput(cmd)
except AttributeError:
# py2
with open('/dev/null', 'r+') as devnull:
p = subprocess.Popen(cmd,
shell=True,
stdin=devnull,
stdout=subprocess.PIPE,
stderr=devnull)
(stdout, stderr) = p.communicate()
status = p.wait()
output = stdout.decode()
if status != 0:
self.log.debug("%s: stderr output: %s", cmd, stderr)
raise RuntimeError("%s: exit code %s" % (cmd, status))
return self.parse_rpm_output(output.splitlines(), tags, separator=sep)
def get_output_metadata(self, path, filename):
"""
Describe a file by its metadata.
:return: dict
"""
checksums = get_checksums(path, ['md5'])
metadata = {'filename': filename,
'filesize': os.path.getsize(path),
'checksum': checksums['md5sum'],
'checksum_type': 'md5'}
if self.metadata_only:
metadata['metadata_only'] = True
return metadata
def get_builder_image_id(self):
"""
Find out the docker ID of the buildroot image we are in.
"""
buildroot_tag = os.environ["OPENSHIFT_CUSTOM_BUILD_BASE_IMAGE"]
kwargs = {}
if self.namespace is not None:
kwargs['namespace'] = self.namespace
pod = self.osbs.get_pod_for_build(self.build_id, **kwargs)
all_images = pod.get_container_image_ids()
try:
return all_images[buildroot_tag]
except KeyError:
self.log.error("Unable to determine buildroot image ID for %s",
buildroot_tag)
return buildroot_tag
def get_buildroot(self, build_id):
"""
Build the buildroot entry of the metadata.
:return: dict, partial metadata
"""
docker_version = self.tasker.get_version()
docker_info = self.tasker.get_info()
host_arch = docker_version['Arch']
if host_arch == 'amd64':
host_arch = 'x86_64'
buildroot = {
'id': 1,
'host': {
'os': docker_info['OperatingSystem'],
'arch': host_arch,
},
'content_generator': {
'name': PROG,
'version': atomic_reactor_version,
},
'container': {
'type': 'docker',
'arch': os.uname()[4],
},
'tools': [
{
'name': tool['name'],
'version': tool['version'],
}
for tool in get_version_of_tools()] + [
{
'name': 'docker',
'version': docker_version['Version'],
},
],
'components': self.get_rpms(),
'extra': {
'osbs': {
'build_id': build_id,
'builder_image_id': self.get_builder_image_id(),
}
},
}
return buildroot
def get_logs(self):
"""
Build the logs entry for the metadata 'output' section
:return: list, Output instances
"""
# Collect logs from server
kwargs = {}
if self.namespace is not None:
kwargs['namespace'] = self.namespace
logs = self.osbs.get_build_logs(self.build_id, **kwargs)
# Deleted once closed
logfile = NamedTemporaryFile(prefix=self.build_id,
suffix=".log",
mode='w')
logfile.write(logs)
logfile.flush()
docker_logs = NamedTemporaryFile(prefix="docker-%s" % self.build_id,
suffix=".log",
mode='w')
docker_logs.write("\n".join(self.workflow.build_logs))
docker_logs.flush()
return [Output(file=docker_logs,
metadata=self.get_output_metadata(docker_logs.name,
"build.log")),
Output(file=logfile,
metadata=self.get_output_metadata(logfile.name,
"openshift-final.log"))]
def get_image_components(self):
"""
Re-package the output of the rpmqa plugin into the format required
for the metadata.
"""
try:
output = self.workflow.postbuild_results[PostBuildRPMqaPlugin.key]
except KeyError:
self.log.error("%s plugin did not run!",
PostBuildRPMqaPlugin.key)
return []
return self.parse_rpm_output(output, PostBuildRPMqaPlugin.rpm_tags,
separator=',')
def get_image_output(self):
"""
Create the output for the image
For v1, this is the v1 image. For v2, this is the v2 metadata
with the checksum of an empty file, and no actual upload.
:return: tuple, (metadata dict, Output instance)
"""
image_id = self.workflow.builder.image_id
v1_image = self.workflow.exported_image_sequence[-1].get('path')
ext = v1_image.split('.', 1)[1]
if self.metadata_only:
v2_image_name = 'docker-v2-image-{0}.{1}'.format(image_id, ext)
metadata = self.get_output_metadata(os.path.devnull, v2_image_name)
output = Output(file=None, metadata=metadata)
else:
v1_image_name = 'docker-v1-image-{0}.{1}'.format(image_id, ext)
metadata = self.get_output_metadata(v1_image, v1_image_name)
output = Output(file=open(v1_image), metadata=metadata)
return metadata, output
def get_output_images(self):
if self.workflow.push_conf.pulp_registries:
# If pulp was used, only report pulp images
registries = self.workflow.push_conf.pulp_registries
else:
# Otherwise report all the images we pushed
registries = self.workflow.push_conf.all_registries
output_images = []
for registry in registries:
for image in (self.workflow.tag_conf.primary_images +
self.workflow.tag_conf.unique_images):
registry_image = image.copy()
registry_image.registry = registry.uri
if registry_image not in output_images:
output_images.append(registry_image)
return output_images
def get_output(self, buildroot_id):
"""
Build the 'output' section of the metadata.
:return: list, Output instances
"""
def add_buildroot_id(output):
logfile, metadata = output
metadata.update({'buildroot_id': buildroot_id})
return Output(file=logfile, metadata=metadata)
def add_log_type(output):
logfile, metadata = output
metadata.update({'type': 'log', 'arch': 'noarch'})
return Output(file=logfile, metadata=metadata)
output_files = [add_log_type(add_buildroot_id(metadata))
for metadata in self.get_logs()]
# Parent of squashed built image is base image
image_id = self.workflow.builder.image_id
parent_id = self.workflow.base_image_inspect['Id']
output_images = self.get_output_images()
repositories = [image.to_str() for image in output_images
if image.tag != 'latest']
arch = os.uname()[4]
metadata, output = self.get_image_output()
metadata.update({
'arch': arch,
'type': 'docker-image',
'components': self.get_image_components(),
'extra': {
'image': {
'arch': arch,
},
'docker': {
'id': image_id,
'parent_id': parent_id,
'repositories': repositories,
},
},
})
# Add the v1 image (or v2 metadata) to the output
image = add_buildroot_id(output)
output_files.append(image)
return output_files
def get_build(self, metadata):
build_start_time = metadata["creationTimestamp"]
try:
# Decode UTC RFC3339 date with no fractional seconds
# (the format we expect)
start_time_struct = time.strptime(build_start_time,
'%Y-%m-%dT%H:%M:%SZ')
start_time = int(time.mktime(start_time_struct))
except ValueError:
self.log.error("Invalid time format (%s)", build_start_time)
raise
name = None
version = None
release = None
for image_name in self.workflow.tag_conf.primary_images:
if '-' in image_name.tag:
name = image_name.repo
version, release = image_name.tag.split('-', 1)
if name is None or version is None or release is None:
raise RuntimeError('Unable to determine name-version-release')
source = self.workflow.source
if not isinstance(source, GitSource):
raise RuntimeError('git source required')
build = {
'name': name,
'version': version,
'release': release,
'source': "{0}#{1}".format(source.uri, source.commit_id),
'start_time': start_time,
'end_time': int(time.time()),
'extra': {
'image': {},
},
}
if self.metadata_only:
build['metadata_only'] = True
return build
def get_metadata(self):
"""
Build the metadata needed for importing the build
:return: tuple, the metadata and the list of Output instances
"""
try:
build_json = json.loads(os.environ["BUILD"])
except KeyError:
self.log.error("No $BUILD env variable. "
"Probably not running in build container.")
raise
try:
metadata = build_json["metadata"]
self.build_id = metadata["name"]
self.namespace = metadata.get("namespace")
except KeyError:
self.log.error("No build metadata")
raise
metadata_version = 0
build = self.get_build(metadata)
buildroot = self.get_buildroot(build_id=self.build_id)
output_files = self.get_output(buildroot['id'])
koji_metadata = {
'metadata_version': metadata_version,
'build': build,
'buildroots': [buildroot],
'output': [output.metadata for output in output_files],
}
return koji_metadata, output_files
def upload_file(self, session, output, serverdir):
"""
Upload a file to koji
:return: str, pathname on server
"""
name = output.metadata['filename']
self.log.debug("uploading %r to %r as %r",
output.file.name, serverdir, name)
session.uploadWrapper(output.file.name, serverdir, name=name)
path = os.path.join(serverdir, name)
self.log.debug("uploaded %r", path)
return path
@staticmethod
def get_upload_server_dir():
"""
Create a path name for uploading files to
:return: str, path name expected to be unique
"""
dir_prefix = 'koji-promote'
random_chars = ''.join([random.choice(ascii_letters)
for _ in range(8)])
unique_fragment = '%r.%s' % (time.time(), random_chars)
return os.path.join(dir_prefix, unique_fragment)
def login(self):
"""
Log in to koji
:return: koji.ClientSession instance, logged in
"""
session = koji.ClientSession(self.kojihub)
kwargs = {}
if self.koji_proxy_user:
kwargs['proxyuser'] = self.koji_proxy_user
if self.koji_ssl_certs:
# Use certificates
self.log.info("Using SSL certificates for Koji authentication")
session.ssl_login(os.path.join(self.koji_ssl_certs, 'cert'),
os.path.join(self.koji_ssl_certs, 'ca'),
os.path.join(self.koji_ssl_certs, 'serverca'),
**kwargs)
else:
# Use Kerberos
self.log.info("Using Kerberos for Koji authentication")
if self.koji_principal and self.koji_keytab:
kwargs['principal'] = self.koji_principal
kwargs['keytab'] = self.koji_keytab
session.krb_login(**kwargs)
return session
def run(self):
"""
Run the plugin.
"""
if ((self.koji_principal and not self.koji_keytab) or
(self.koji_keytab and not self.koji_principal)):
raise RuntimeError("specify both koji_principal and koji_keytab "
"or neither")
# Only run if the build was successful
if self.workflow.build_process_failed:
self.log.info("Not promoting failed build to koji")
return
if not is_rebuild(self.workflow):
self.log.info("Not promoting to koji: not a rebuild")
return
koji_metadata, output_files = self.get_metadata()
try:
session = self.login()
server_dir = self.get_upload_server_dir()
for output in output_files:
if output.file:
self.upload_file(session, output, server_dir)
finally:
for output in output_files:
if output.file:
output.file.close()
session.CGImport(koji_metadata, server_dir)
self.log.debug("Submitted with metadata: %s",
json.dumps(koji_metadata, sort_keys=True, indent=4))
class KojiUploadPlugin(PostBuildPlugin):
"""
Upload this build to Koji
Note: only the image archive is uploaded to Koji at this stage.
Metadata about this image is created and stored in a ConfigMap in
OpenShift, ready for the orchestrator build to collect and use to
actually create the Koji Build together with the uploaded image
archive(s).
Authentication is with Kerberos unless the koji_ssl_certs_dir
configuration parameter is given, in which case it should be a
path at which 'cert', 'ca', and 'serverca' are the certificates
for SSL authentication.
If Kerberos is used for authentication, the default principal will
be used (from the kernel keyring) unless both koji_keytab and
koji_principal are specified. The koji_keytab parameter is a
keytab name like 'type:name', and so can be used to specify a key
in a Kubernetes secret by specifying 'FILE:/path/to/key'.
"""
key = PLUGIN_KOJI_UPLOAD_PLUGIN_KEY
is_allowed_to_fail = False
def __init__(self, tasker, workflow, kojihub, url, build_json_dir,
koji_upload_dir, verify_ssl=True, use_auth=True,
koji_ssl_certs_dir=None, koji_proxy_user=None,
koji_principal=None, koji_keytab=None,
blocksize=None, prefer_schema1_digest=True):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param kojihub: string, koji hub (xmlrpc)
:param url: string, URL for OSv3 instance
:param build_json_dir: str, path to directory with input json
:param koji_upload_dir: str, path to use when uploading to hub
:param verify_ssl: bool, verify OSv3 SSL certificate?
:param use_auth: bool, initiate authentication with OSv3?
:param koji_ssl_certs_dir: str, path to 'cert', 'ca', 'serverca'
:param koji_proxy_user: str, user to log in as (requires hub config)
:param koji_principal: str, Kerberos principal (must specify keytab)
:param koji_keytab: str, keytab name (must specify principal)
:param blocksize: int, blocksize to use for uploading files
:param prefer_schema1_digest: bool, when True, v2 schema 1 digest will
be preferred as the built image digest
"""
super(KojiUploadPlugin, self).__init__(tasker, workflow)
self.kojihub = kojihub
self.koji_ssl_certs_dir = koji_ssl_certs_dir
self.koji_proxy_user = koji_proxy_user
self.koji_principal = koji_principal
self.koji_keytab = koji_keytab
self.blocksize = blocksize
self.build_json_dir = build_json_dir
self.koji_upload_dir = koji_upload_dir
self.prefer_schema1_digest = prefer_schema1_digest
self.namespace = get_build_json().get('metadata', {}).get('namespace', None)
osbs_conf = Configuration(conf_file=None, openshift_uri=url,
use_auth=use_auth, verify_ssl=verify_ssl,
build_json_dir=self.build_json_dir,
namespace=self.namespace)
self.osbs = OSBS(osbs_conf, osbs_conf)
self.build_id = None
self.pullspec_image = None
@staticmethod
def parse_rpm_output(output, tags, separator=';'):
"""
Parse output of the rpm query.
:param output: list, decoded output (str) from the rpm subprocess
:param tags: list, str fields used for query output
:return: list, dicts describing each rpm package
"""
def field(tag):
"""
Get a field value by name
"""
try:
value = fields[tags.index(tag)]
except ValueError:
return None
if value == '(none)':
return None
return value
components = []
sigmarker = 'Key ID '
for rpm in output:
fields = rpm.rstrip('\n').split(separator)
if len(fields) < len(tags):
continue
signature = field('SIGPGP:pgpsig') or field('SIGGPG:pgpsig')
if signature:
parts = signature.split(sigmarker, 1)
if len(parts) > 1:
signature = parts[1]
component_rpm = {
'type': 'rpm',
'name': field('NAME'),
'version': field('VERSION'),
'release': field('RELEASE'),
'arch': field('ARCH'),
'sigmd5': field('SIGMD5'),
'signature': signature,
}
# Special handling for epoch as it must be an integer or None
epoch = field('EPOCH')
if epoch is not None:
epoch = int(epoch)
component_rpm['epoch'] = epoch
if component_rpm['name'] != 'gpg-pubkey':
components.append(component_rpm)
return components
def get_rpms(self):
"""
Build a list of installed RPMs in the format required for the
metadata.
"""
tags = [
'NAME',
'VERSION',
'RELEASE',
'ARCH',
'EPOCH',
'SIGMD5',
'SIGPGP:pgpsig',
'SIGGPG:pgpsig',
]
sep = ';'
fmt = sep.join(["%%{%s}" % tag for tag in tags])
cmd = "/bin/rpm -qa --qf '{0}\n'".format(fmt)
try:
# py3
(status, output) = subprocess.getstatusoutput(cmd)
except AttributeError:
# py2
with open('/dev/null', 'r+') as devnull:
p = subprocess.Popen(cmd,
shell=True,
stdin=devnull,
stdout=subprocess.PIPE,
stderr=devnull)
(stdout, stderr) = p.communicate()
status = p.wait()
output = stdout.decode()
if status != 0:
self.log.debug("%s: stderr output: %s", cmd, stderr)
raise RuntimeError("%s: exit code %s" % (cmd, status))
return self.parse_rpm_output(output.splitlines(), tags, separator=sep)
def get_output_metadata(self, path, filename):
"""
Describe a file by its metadata.
:return: dict
"""
checksums = get_checksums(path, ['md5'])
metadata = {'filename': filename,
'filesize': os.path.getsize(path),
'checksum': checksums['md5sum'],
'checksum_type': 'md5'}
return metadata
def get_builder_image_id(self):
"""
Find out the docker ID of the buildroot image we are in.
"""
try:
buildroot_tag = os.environ["OPENSHIFT_CUSTOM_BUILD_BASE_IMAGE"]
except KeyError:
return ''
try:
pod = self.osbs.get_pod_for_build(self.build_id)
all_images = pod.get_container_image_ids()
except OsbsException as ex:
self.log.error("unable to find image id: %r", ex)
return buildroot_tag
try:
return all_images[buildroot_tag]
except KeyError:
self.log.error("Unable to determine buildroot image ID for %s",
buildroot_tag)
return buildroot_tag
def get_buildroot(self, build_id):
"""
Build the buildroot entry of the metadata.
:return: dict, partial metadata
"""
docker_info = self.tasker.get_info()
host_arch, docker_version = get_docker_architecture(self.tasker)
buildroot = {
'id': 1,
'host': {
'os': docker_info['OperatingSystem'],
'arch': host_arch,
},
'content_generator': {
'name': PROG,
'version': atomic_reactor_version,
},
'container': {
'type': 'docker',
'arch': os.uname()[4],
},
'tools': [
{
'name': tool['name'],
'version': tool['version'],
}
for tool in get_version_of_tools()] + [
{
'name': 'docker',
'version': docker_version,
},
],
'components': self.get_rpms(),
'extra': {
'osbs': {
'build_id': build_id,
'builder_image_id': self.get_builder_image_id(),
}
},
}
return buildroot
def get_logs(self):
"""
Build the logs entry for the metadata 'output' section
:return: list, Output instances
"""
output = []
# Collect logs from server
try:
logs = self.osbs.get_build_logs(self.build_id)
except OsbsException as ex:
self.log.error("unable to get build logs: %r", ex)
else:
# Deleted once closed
logfile = NamedTemporaryFile(prefix=self.build_id,
suffix=".log",
mode='wb')
try:
logfile.write(logs)
except (TypeError, UnicodeEncodeError):
# Older osbs-client versions returned Unicode objects
logfile.write(logs.encode('utf-8'))
logfile.flush()
metadata = self.get_output_metadata(logfile.name,
"openshift-final.log")
output.append(Output(file=logfile, metadata=metadata))
docker_logs = NamedTemporaryFile(prefix="docker-%s" % self.build_id,
suffix=".log",
mode='wb')
docker_logs.write("\n".join(self.workflow.build_result.logs).encode('utf-8'))
docker_logs.flush()
output.append(Output(file=docker_logs,
metadata=self.get_output_metadata(docker_logs.name,
"build.log")))
return output
def get_image_components(self):
"""
Re-package the output of the rpmqa plugin into the format required
for the metadata.
"""
try:
output = self.workflow.postbuild_results[PostBuildRPMqaPlugin.key]
except KeyError:
self.log.error("%s plugin did not run!",
PostBuildRPMqaPlugin.key)
return []
try:
sep = PostBuildRPMqaPlugin.sep
except AttributeError:
# sep instance variable added in Aug 2016
sep = ','
return self.parse_rpm_output(output, PostBuildRPMqaPlugin.rpm_tags,
separator=sep)
def get_image_output(self, arch):
"""
Create the output for the image
This is the Koji Content Generator metadata, along with the
'docker save' output to upload.
:param arch: str, architecture for this output
:return: tuple, (metadata dict, Output instance)
"""
image_id = self.workflow.builder.image_id
saved_image = self.workflow.exported_image_sequence[-1].get('path')
ext = saved_image.split('.', 1)[1]
name_fmt = 'docker-image-{id}.{arch}.{ext}'
image_name = name_fmt.format(id=image_id, arch=arch, ext=ext)
metadata = self.get_output_metadata(saved_image, image_name)
output = Output(file=open(saved_image), metadata=metadata)
return metadata, output
def get_digests(self):
"""
Returns a map of repositories to digests
"""
digests = {} # repository -> digest
for registry in self.workflow.push_conf.docker_registries:
for image in self.workflow.tag_conf.images:
image_str = image.to_str()
if image_str in registry.digests:
digest = self.select_digest(registry.digests[image_str])
digests[image.to_str(registry=False)] = digest
return digests
def select_digest(self, digests):
digest = digests.default
# pulp/crane supports only manifest schema v1
if self.prefer_schema1_digest:
if self.workflow.push_conf.pulp_registries:
self.log.info('Using schema v1 digest because of older Pulp integration')
digest = digests.v1
else:
self.log.info('Schema v1 preferred, but not used')
return digest
def get_repositories(self, digests):
"""
Build the repositories metadata
:param digests: dict, repository -> digest
"""
if self.workflow.push_conf.pulp_registries:
# If pulp was used, only report pulp images
registries = self.workflow.push_conf.pulp_registries
else:
# Otherwise report all the images we pushed
registries = self.workflow.push_conf.all_registries
output_images = []
for registry in registries:
image = self.pullspec_image.copy()
image.registry = registry.uri
pullspec = image.to_str()
output_images.append(pullspec)
digest = digests.get(image.to_str(registry=False))
if digest:
digest_pullspec = image.to_str(tag=False) + "@" + digest
output_images.append(digest_pullspec)
return output_images
def get_output(self, buildroot_id):
"""
Build the 'output' section of the metadata.
:return: list, Output instances
"""
def add_buildroot_id(output):
logfile, metadata = output
metadata.update({'buildroot_id': buildroot_id})
return Output(file=logfile, metadata=metadata)
def add_log_type(output, arch):
logfile, metadata = output
metadata.update({'type': 'log', 'arch': arch})
return Output(file=logfile, metadata=metadata)
arch = os.uname()[4]
output_files = [add_log_type(add_buildroot_id(metadata), arch)
for metadata in self.get_logs()]
# Parent of squashed built image is base image
image_id = self.workflow.builder.image_id
parent_id = self.workflow.base_image_inspect['Id']
# Read config from the registry using v2 schema 2 digest
registries = self.workflow.push_conf.docker_registries
if registries:
config = copy.deepcopy(registries[0].config)
else:
config = {}
# We don't need container_config section
if config and 'container_config' in config:
del config['container_config']
digests = self.get_digests()
repositories = self.get_repositories(digests)
tags = set(image.tag for image in self.workflow.tag_conf.images)
metadata, output = self.get_image_output(arch)
metadata.update({
'arch': arch,
'type': 'docker-image',
'components': self.get_image_components(),
'extra': {
'image': {
'arch': arch,
},
'docker': {
'id': image_id,
'parent_id': parent_id,
'repositories': repositories,
'tags': list(tags),
'config': config
},
},
})
if not config:
del metadata['extra']['docker']['config']
# Add the 'docker save' image to the output
image = add_buildroot_id(output)
output_files.append(image)
return output_files
def get_metadata(self):
"""
Build the metadata needed for importing the build
:return: tuple, the metadata and the list of Output instances
"""
try:
metadata = get_build_json()["metadata"]
self.build_id = metadata["name"]
except KeyError:
self.log.error("No build metadata")
raise
for image in self.workflow.tag_conf.unique_images:
self.pullspec_image = image
break
for image in self.workflow.tag_conf.primary_images:
# dash at first/last postition does not count
if '-' in image.tag[1:-1]:
self.pullspec_image = image
break
if not self.pullspec_image:
raise RuntimeError('Unable to determine pullspec_image')
metadata_version = 0
buildroot = self.get_buildroot(build_id=self.build_id)
output_files = self.get_output(buildroot['id'])
koji_metadata = {
'metadata_version': metadata_version,
'buildroots': [buildroot],
'output': [output.metadata for output in output_files],
}
return koji_metadata, output_files
def upload_file(self, session, output, serverdir):
"""
Upload a file to koji
:return: str, pathname on server
"""
name = output.metadata['filename']
self.log.debug("uploading %r to %r as %r",
output.file.name, serverdir, name)
kwargs = {}
if self.blocksize is not None:
kwargs['blocksize'] = self.blocksize
self.log.debug("using blocksize %d", self.blocksize)
upload_logger = KojiUploadLogger(self.log)
session.uploadWrapper(output.file.name, serverdir, name=name,
callback=upload_logger.callback, **kwargs)
path = os.path.join(serverdir, name)
self.log.debug("uploaded %r", path)
return path
def login(self):
"""
Log in to koji
:return: koji.ClientSession instance, logged in
"""
# krbV python library throws an error if these are unicode
auth_info = {
"proxyuser": self.koji_proxy_user,
"ssl_certs_dir": self.koji_ssl_certs_dir,
"krb_principal": str(self.koji_principal),
"krb_keytab": str(self.koji_keytab)
}
return create_koji_session(str(self.kojihub), auth_info)
def run(self):
"""
Run the plugin.
"""
if ((self.koji_principal and not self.koji_keytab) or
(self.koji_keytab and not self.koji_principal)):
raise RuntimeError("specify both koji_principal and koji_keytab "
"or neither")
# Only run if the build was successful
if self.workflow.build_process_failed:
self.log.info("Not promoting failed build to koji")
return
koji_metadata, output_files = self.get_metadata()
try:
session = self.login()
for output in output_files:
if output.file:
self.upload_file(session, output, self.koji_upload_dir)
finally:
for output in output_files:
if output.file:
output.file.close()
md_fragment = "{}-md".format(get_build_json()['metadata']['name'])
md_fragment_key = 'metadata.json'
cm_data = {md_fragment_key: koji_metadata}
annotations = {
"metadata_fragment": "configmap/" + md_fragment,
"metadata_fragment_key": md_fragment_key
}
try:
self.osbs.create_config_map(md_fragment, cm_data)
except OsbsException:
self.log.debug("metadata: %r", koji_metadata)
self.log.debug("annotations: %r", annotations)
raise
return annotations
class MockCreator(object):
def __init__(self):
parser = argparse.ArgumentParser(
description="osbs test harness mock JSON creator")
parser.add_argument(
"user",
action='store',
help="name of user to use for Basic Authentication in OSBS")
parser.add_argument("--config",
action='store',
metavar="PATH",
help="path to configuration file",
default=DEFAULT_CONFIGURATION_FILE)
parser.add_argument(
"--instance",
"-i",
action='store',
metavar="SECTION_NAME",
help="section within config for requested instance",
default="stage")
parser.add_argument(
"--password",
action='store',
help="password to use for Basic Authentication in OSBS")
parser.add_argument("--mock-dir",
metavar="DIR",
action="store",
default=DEFAULT_DIR,
help="mock JSON responses are stored in DIR")
parser.add_argument(
"--imagestream",
metavar="IMAGESTREAM",
action="store",
default=DEFAULT_IMAGESTREAM_FILE,
help="Image name for image stream import. Defaults to " +
DEFAULT_IMAGESTREAM_FILE)
parser.add_argument(
"--image_server",
metavar="IMAGESERVER",
action="store",
default=DEFAULT_IMAGESTREAM_SERVER,
help="Server for image stream import. Defaults to " +
DEFAULT_IMAGESTREAM_SERVER)
parser.add_argument(
"--image_tags",
metavar="IMAGETAGS",
action="store",
nargs=3,
default=DEFAULT_IMAGESTREAM_TAGS,
help="Image stream tags as 3 space separated values.")
parser.add_argument("--os-version",
metavar="OS_VER",
action="store",
default=OS_VERSION,
help="OpenShift version of the mock JSONs")
args = parser.parse_args()
self.user = args.user
self.password = args.password
mock_path = args.mock_dir
self.mock_dir = "/".join([mock_path, args.os_version])
find_or_make_dir(self.mock_dir)
self.capture_dir = tempfile.mkdtemp()
args.git_url = "https://github.com/TomasTomecek/docker-hello-world.git"
args.git_branch = "master"
args.git_commit = "HEAD"
os_conf = Configuration(conf_file=args.config,
conf_section=args.instance,
cli_args=args)
build_conf = Configuration(conf_file=args.config,
conf_section=args.instance,
cli_args=args)
set_logging(level=logging.INFO)
self.osbs = OSBS(os_conf, build_conf)
setup_json_capture(self.osbs, os_conf, self.capture_dir)
self.imagestream_file = args.imagestream
self.imagestream_server = args.image_server
self.imagestream_tags = args.image_tags
self.rh_pattern = re.template("redhat.com")
self.ex_pattern = "(\S+\.)*redhat.com" # noqa:W605
def clean_data(self, out_data):
if isinstance(out_data, dict):
cleaned_data = {}
for key, data in out_data.items():
cleaned_data[key] = self.clean_data(data)
return cleaned_data
elif isinstance(out_data, list):
cleaned_data = []
for data in out_data:
cleaned_data.append(self.clean_data(data))
return cleaned_data
elif isinstance(out_data, str):
if re.search(self.rh_pattern, out_data):
return re.sub(self.ex_pattern, "example.com", out_data)
else:
return out_data
else:
return out_data
def comp_write(self, out_name, out_data):
cleaned_data = self.clean_data(out_data)
out_path = "/".join([self.mock_dir, out_name])
with open(out_path, "w") as outf:
try:
json.dump(cleaned_data, outf, indent=4)
except (ValueError, TypeError):
outf.write(json.dumps(cleaned_data, indent=4))
def create_mock_builds_list(self):
kwargs = {}
# get build list
self.osbs.list_builds(**kwargs)
# find 'get-namespaces_osbs-stage_builds_-000.json' file and parse it into
# 'builds_list.json', 'builds_list_empty.json', 'builds_list_one.json'
all_builds = "get-namespaces_osbs-stage_builds_-000.json"
all_builds_path = "/".join([self.capture_dir, all_builds])
with open(all_builds_path, "r") as infile:
builds_data = json.load(infile)
builds_items = copy.copy(builds_data["items"])
builds_data["items"] = []
self.comp_write("builds_list_empty.json", builds_data)
if not builds_items:
return
builds_data["items"].append(builds_items[0])
self.comp_write("builds_list_one.json", builds_data)
if len(builds_items) < 2:
return
builds_data["items"].append(builds_items[1])
self.comp_write("builds_list.json", builds_data)
os.remove(all_builds_path)
def create_pods_list(self, build_id):
self.osbs.get_pod_for_build(build_id)
pods_pre = "get-namespaces_osbs-stage_pods_?labelSelector=openshift.io%2Fbuild.name%3D"
for i in range(0, 4):
try:
pods_fname = pods_pre + build_id + "-00{}.json".format(i)
pods_inpath = "/".join([self.capture_dir, pods_fname])
os.stat(pods_inpath)
break
except OSError:
continue
with open(pods_inpath, "r") as infile:
pods_data = json.load(infile)
image = "buildroot:latest"
pods_items = pods_data["items"] or []
for pod in pods_items:
pod_containers = pod["status"]["containerStatuses"] or []
for container in pod_containers:
container["imageID"] = "docker-pullable://" + image
container["image"] = image
self.comp_write("pods.json", pods_data)
os.remove(pods_inpath)
def create_mock_get_user(self):
self.osbs.get_user()
user_list = "get-users_~_-000.json"
user_list_path = "/".join([self.capture_dir, user_list])
with open(user_list_path, "r") as infile:
user_data = json.load(infile)
user_data["groups"] = []
user_data["identities"] = None
if "fullName" in user_data:
del user_data["fullName"]
user_data["metadata"]["name"] = "test"
user_data["metadata"][
"selfLink"] = "/apis/user.openshift.io/v1/users/test"
self.comp_write("get_user.json", user_data)
os.remove(user_list_path)
def create_a_mock_build(self, func, build_name, out_tag, build_args):
try:
build = func(**build_args)
build_id = build.get_build_name()
build = self.osbs.wait_for_build_to_get_scheduled(build_id)
self.osbs.watch_builds()
build = self.osbs.wait_for_build_to_finish(build_id)
self.osbs.get_build_logs(build_id)
except (subprocess.CalledProcessError, OsbsException):
pass
watch_data = canonize_data(copy.deepcopy(build.json), build_name,
"Complete")
watch_obj = {"object": watch_data, "type": "MODIFIED"}
out_name = "watch_build_test-" + out_tag + "build-123.json"
self.comp_write(out_name, watch_obj)
build_fname = "get-namespaces_osbs-stage_builds_" + build_id + "_-000.json"
build_path = "/".join([self.capture_dir, build_fname])
with open(build_path, "r") as infile:
build_data = canonize_data(json.load(infile), build_name,
"Complete")
out_name = "build_test-" + out_tag + "build-123.json"
self.comp_write(out_name, build_data)
os.remove(build_path)
return out_name
def create_mock_build(self):
build_kwargs = {
'git_uri': self.osbs.build_conf.get_git_uri(),
'git_ref': self.osbs.build_conf.get_git_ref(),
'git_branch': self.osbs.build_conf.get_git_branch(),
'user': self.osbs.build_conf.get_user(),
'release': TEST_BUILD,
'platform': "x86_64",
'scratch': True,
}
build_name = self.create_a_mock_build(self.osbs.create_worker_build,
TEST_BUILD, "", build_kwargs)
build_path = "/".join([self.mock_dir, build_name])
if os.stat(build_path):
with open(build_path, "r") as infile:
build_data = json.load(infile)
build_data["kind"] = "BuildConfig"
self.comp_write(
"created_build_config_test-build-config-123.json",
build_data)
del build_kwargs['platform']
self.create_a_mock_build(self.osbs.create_orchestrator_build,
TEST_ORCHESTRATOR_BUILD, "orchestrator-",
build_kwargs)
def create_mock_build_other(self):
build_kwargs = {
'git_uri': self.osbs.build_conf.get_git_uri(),
'git_ref': self.osbs.build_conf.get_git_ref(),
'git_branch': self.osbs.build_conf.get_git_branch(),
'user': self.osbs.build_conf.get_user(),
'release': TEST_BUILD,
'platform': "x86_64",
'scratch': True,
}
build_id = ""
try:
build = self.osbs.create_worker_build(**build_kwargs)
build_id = build.get_build_name()
self.osbs.wait_for_build_to_get_scheduled(build_id)
self.create_pods_list(build_id)
self.osbs.cancel_build(build_id)
self.osbs.wait_for_build_to_finish(build_id)
self.osbs.get_build_logs(build_id)
except OsbsException:
self.create_pods_list(build_id)
instant_fname = "post-namespaces_osbs-stage_builds_-000.json"
instant_path = "/".join([self.capture_dir, instant_fname])
with open(instant_path, "r") as infile:
instant_data = canonize_data(json.load(infile))
self.comp_write("instantiated_test-build-config-123.json",
instant_data)
os.remove(instant_path)
cancel_args = [
{
"suffix": "_-000-001.json",
"version": "get",
"phase": None
},
{
"suffix": "_-000-000.json",
"version": "put",
"phase": "Cancelled"
},
]
for data in cancel_args:
build_fname = "get-watch_namespaces_osbs-stage_builds_" + build_id + data[
"suffix"]
build_path = "/".join([self.capture_dir, build_fname])
with open(build_path, "r") as infile:
cancel_obj = json.load(infile)
cancel_data = canonize_data(
copy.deepcopy(cancel_obj["object"]), TEST_CANCELLED_BUILD,
data["phase"])
self.comp_write(
"build_test-build-cancel-123_" + data["version"] + ".json",
cancel_data)
os.remove(build_path)
def create_mock_static_files(self):
# these aren't JSON, so just write them out
out_path = "/".join(
[self.mock_dir, "build_test-orchestrator-build-123_logs.txt"])
with open(out_path, "w") as outf:
outf.write(ORCH_BUILD_LOG)
out_path = "/".join([self.mock_dir, "build_test-build-123_logs.txt"])
with open(out_path, "wb") as outf:
outf.write(BASE_BUILD_LOG)
self.comp_write("create_config_map.json", MOCK_CONFIG_MAP)
class KojiPromotePlugin(ExitPlugin):
"""
Promote this build to Koji
Submits a successful build to Koji using the Content Generator API,
https://fedoraproject.org/wiki/Koji/ContentGenerators
Authentication is with Kerberos unless the koji_ssl_certs
configuration parameter is given, in which case it should be a
path at which 'cert', 'ca', and 'serverca' are the certificates
for SSL authentication.
If Kerberos is used for authentication, the default principal will
be used (from the kernel keyring) unless both koji_keytab and
koji_principal are specified. The koji_keytab parameter is a
keytab name like 'type:name', and so can be used to specify a key
in a Kubernetes secret by specifying 'FILE:/path/to/key'.
If metadata_only is set, the v1 image will not be uploaded, only
the logs. The import will be marked as metadata-only.
Runs as an exit plugin in order to capture logs from all other
plugins.
"""
key = "koji_promote"
is_allowed_to_fail = False
def __init__(self,
tasker,
workflow,
kojihub,
url,
verify_ssl=True,
use_auth=True,
koji_ssl_certs=None,
koji_proxy_user=None,
koji_principal=None,
koji_keytab=None,
metadata_only=False):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param kojihub: string, koji hub (xmlrpc)
:param url: string, URL for OSv3 instance
:param verify_ssl: bool, verify OSv3 SSL certificate?
:param use_auth: bool, initiate authentication with OSv3?
:param koji_ssl_certs: str, path to 'cert', 'ca', 'serverca'
:param koji_proxy_user: str, user to log in as (requires hub config)
:param koji_principal: str, Kerberos principal (must specify keytab)
:param koji_keytab: str, keytab name (must specify principal)
:param metadata_only: bool, whether to omit the v1 image
"""
super(KojiPromotePlugin, self).__init__(tasker, workflow)
self.kojihub = kojihub
self.koji_ssl_certs = koji_ssl_certs
self.koji_proxy_user = koji_proxy_user
self.koji_principal = koji_principal
self.koji_keytab = koji_keytab
self.metadata_only = metadata_only
osbs_conf = Configuration(conf_file=None,
openshift_uri=url,
use_auth=use_auth,
verify_ssl=verify_ssl)
self.osbs = OSBS(osbs_conf, osbs_conf)
self.build_id = None
self.namespace = None
@staticmethod
def parse_rpm_output(output, tags, separator=';'):
"""
Parse output of the rpm query.
:param output: list, decoded output (str) from the rpm subprocess
:param tags: list, str fields used for query output
:return: list, dicts describing each rpm package
"""
def field(tag):
"""
Get a field value by name
"""
try:
value = fields[tags.index(tag)]
except ValueError:
return None
if value == '(none)':
return None
return value
components = []
sigmarker = 'Key ID '
for rpm in output:
fields = rpm.rstrip('\n').split(separator)
if len(fields) < len(tags):
continue
signature = field('SIGPGP:pgpsig') or field('SIGGPG:pgpsig')
if signature:
parts = signature.split(sigmarker, 1)
if len(parts) > 1:
signature = parts[1]
component_rpm = {
'type': 'rpm',
'name': field('NAME'),
'version': field('VERSION'),
'release': field('RELEASE'),
'arch': field('ARCH'),
'sigmd5': field('SIGMD5'),
'signature': signature,
}
# Special handling for epoch as it must be an integer or None
epoch = field('EPOCH')
if epoch is not None:
epoch = int(epoch)
component_rpm['epoch'] = epoch
if component_rpm['name'] != 'gpg-pubkey':
components.append(component_rpm)
return components
def get_rpms(self):
"""
Build a list of installed RPMs in the format required for the
metadata.
"""
tags = [
'NAME',
'VERSION',
'RELEASE',
'ARCH',
'EPOCH',
'SIGMD5',
'SIGPGP:pgpsig',
'SIGGPG:pgpsig',
]
sep = ';'
fmt = sep.join(["%%{%s}" % tag for tag in tags])
cmd = "/bin/rpm -qa --qf '{0}\n'".format(fmt)
try:
# py3
(status, output) = subprocess.getstatusoutput(cmd)
except AttributeError:
# py2
with open('/dev/null', 'r+') as devnull:
p = subprocess.Popen(cmd,
shell=True,
stdin=devnull,
stdout=subprocess.PIPE,
stderr=devnull)
(stdout, stderr) = p.communicate()
status = p.wait()
output = stdout.decode()
if status != 0:
self.log.debug("%s: stderr output: %s", cmd, stderr)
raise RuntimeError("%s: exit code %s" % (cmd, status))
return self.parse_rpm_output(output.splitlines(), tags, separator=sep)
def get_output_metadata(self, path, filename):
"""
Describe a file by its metadata.
:return: dict
"""
checksums = get_checksums(path, ['md5'])
metadata = {
'filename': filename,
'filesize': os.path.getsize(path),
'checksum': checksums['md5sum'],
'checksum_type': 'md5'
}
if self.metadata_only:
metadata['metadata_only'] = True
return metadata
def get_builder_image_id(self):
"""
Find out the docker ID of the buildroot image we are in.
"""
buildroot_tag = os.environ["OPENSHIFT_CUSTOM_BUILD_BASE_IMAGE"]
kwargs = {}
if self.namespace is not None:
kwargs['namespace'] = self.namespace
pod = self.osbs.get_pod_for_build(self.build_id, **kwargs)
all_images = pod.get_container_image_ids()
try:
return all_images[buildroot_tag]
except KeyError:
self.log.error("Unable to determine buildroot image ID for %s",
buildroot_tag)
return buildroot_tag
def get_buildroot(self, build_id):
"""
Build the buildroot entry of the metadata.
:return: dict, partial metadata
"""
docker_version = self.tasker.get_version()
docker_info = self.tasker.get_info()
host_arch = docker_version['Arch']
if host_arch == 'amd64':
host_arch = 'x86_64'
buildroot = {
'id':
1,
'host': {
'os': docker_info['OperatingSystem'],
'arch': host_arch,
},
'content_generator': {
'name': PROG,
'version': atomic_reactor_version,
},
'container': {
'type': 'docker',
'arch': os.uname()[4],
},
'tools': [{
'name': tool['name'],
'version': tool['version'],
} for tool in get_version_of_tools()] + [
{
'name': 'docker',
'version': docker_version['Version'],
},
],
'components':
self.get_rpms(),
'extra': {
'osbs': {
'build_id': build_id,
'builder_image_id': self.get_builder_image_id(),
}
},
}
return buildroot
def get_logs(self):
"""
Build the logs entry for the metadata 'output' section
:return: list, Output instances
"""
# Collect logs from server
kwargs = {}
if self.namespace is not None:
kwargs['namespace'] = self.namespace
logs = self.osbs.get_build_logs(self.build_id, **kwargs)
# Deleted once closed
logfile = NamedTemporaryFile(prefix=self.build_id,
suffix=".log",
mode='w')
logfile.write(logs)
logfile.flush()
docker_logs = NamedTemporaryFile(prefix="docker-%s" % self.build_id,
suffix=".log",
mode='w')
docker_logs.write("\n".join(self.workflow.build_logs))
docker_logs.flush()
return [
Output(file=docker_logs,
metadata=self.get_output_metadata(docker_logs.name,
"build.log")),
Output(file=logfile,
metadata=self.get_output_metadata(logfile.name,
"openshift-final.log"))
]
def get_image_components(self):
"""
Re-package the output of the rpmqa plugin into the format required
for the metadata.
"""
try:
output = self.workflow.postbuild_results[PostBuildRPMqaPlugin.key]
except KeyError:
self.log.error("%s plugin did not run!", PostBuildRPMqaPlugin.key)
return []
return self.parse_rpm_output(output,
PostBuildRPMqaPlugin.rpm_tags,
separator=',')
def get_image_output(self):
"""
Create the output for the image
For v1, this is the v1 image. For v2, this is the v2 metadata
with the checksum of an empty file, and no actual upload.
:return: tuple, (metadata dict, Output instance)
"""
image_id = self.workflow.builder.image_id
v1_image = self.workflow.exported_image_sequence[-1].get('path')
ext = v1_image.split('.', 1)[1]
if self.metadata_only:
v2_image_name = 'docker-v2-image-{0}.{1}'.format(image_id, ext)
metadata = self.get_output_metadata(os.path.devnull, v2_image_name)
output = Output(file=None, metadata=metadata)
else:
v1_image_name = 'docker-v1-image-{0}.{1}'.format(image_id, ext)
metadata = self.get_output_metadata(v1_image, v1_image_name)
output = Output(file=open(v1_image), metadata=metadata)
return metadata, output
def get_output_images(self):
if self.workflow.push_conf.pulp_registries:
# If pulp was used, only report pulp images
registries = self.workflow.push_conf.pulp_registries
else:
# Otherwise report all the images we pushed
registries = self.workflow.push_conf.all_registries
output_images = []
for registry in registries:
for image in (self.workflow.tag_conf.primary_images +
self.workflow.tag_conf.unique_images):
registry_image = image.copy()
registry_image.registry = registry.uri
if registry_image not in output_images:
output_images.append(registry_image)
return output_images
def get_output(self, buildroot_id):
"""
Build the 'output' section of the metadata.
:return: list, Output instances
"""
def add_buildroot_id(output):
logfile, metadata = output
metadata.update({'buildroot_id': buildroot_id})
return Output(file=logfile, metadata=metadata)
def add_log_type(output):
logfile, metadata = output
metadata.update({'type': 'log', 'arch': 'noarch'})
return Output(file=logfile, metadata=metadata)
output_files = [
add_log_type(add_buildroot_id(metadata))
for metadata in self.get_logs()
]
# Parent of squashed built image is base image
image_id = self.workflow.builder.image_id
parent_id = self.workflow.base_image_inspect['Id']
output_images = self.get_output_images()
repositories = [
image.to_str() for image in output_images if image.tag != 'latest'
]
arch = os.uname()[4]
metadata, output = self.get_image_output()
metadata.update({
'arch': arch,
'type': 'docker-image',
'components': self.get_image_components(),
'extra': {
'image': {
'arch': arch,
},
'docker': {
'id': image_id,
'parent_id': parent_id,
'repositories': repositories,
},
},
})
# Add the v1 image (or v2 metadata) to the output
image = add_buildroot_id(output)
output_files.append(image)
return output_files
def get_build(self, metadata):
build_start_time = metadata["creationTimestamp"]
try:
# Decode UTC RFC3339 date with no fractional seconds
# (the format we expect)
start_time_struct = time.strptime(build_start_time,
'%Y-%m-%dT%H:%M:%SZ')
start_time = int(time.mktime(start_time_struct))
except ValueError:
self.log.error("Invalid time format (%s)", build_start_time)
raise
name = None
version = None
release = None
for image_name in self.workflow.tag_conf.primary_images:
if '-' in image_name.tag:
name = image_name.repo
version, release = image_name.tag.split('-', 1)
if name is None or version is None or release is None:
raise RuntimeError('Unable to determine name-version-release')
source = self.workflow.source
if not isinstance(source, GitSource):
raise RuntimeError('git source required')
build = {
'name': name,
'version': version,
'release': release,
'source': "{0}#{1}".format(source.uri, source.commit_id),
'start_time': start_time,
'end_time': int(time.time()),
'extra': {
'image': {},
},
}
if self.metadata_only:
build['metadata_only'] = True
return build
def get_metadata(self):
"""
Build the metadata needed for importing the build
:return: tuple, the metadata and the list of Output instances
"""
try:
build_json = json.loads(os.environ["BUILD"])
except KeyError:
self.log.error("No $BUILD env variable. "
"Probably not running in build container.")
raise
try:
metadata = build_json["metadata"]
self.build_id = metadata["name"]
self.namespace = metadata.get("namespace")
except KeyError:
self.log.error("No build metadata")
raise
metadata_version = 0
build = self.get_build(metadata)
buildroot = self.get_buildroot(build_id=self.build_id)
output_files = self.get_output(buildroot['id'])
koji_metadata = {
'metadata_version': metadata_version,
'build': build,
'buildroots': [buildroot],
'output': [output.metadata for output in output_files],
}
return koji_metadata, output_files
def upload_file(self, session, output, serverdir):
"""
Upload a file to koji
:return: str, pathname on server
"""
name = output.metadata['filename']
self.log.debug("uploading %r to %r as %r", output.file.name, serverdir,
name)
session.uploadWrapper(output.file.name, serverdir, name=name)
path = os.path.join(serverdir, name)
self.log.debug("uploaded %r", path)
return path
@staticmethod
def get_upload_server_dir():
"""
Create a path name for uploading files to
:return: str, path name expected to be unique
"""
dir_prefix = 'koji-promote'
random_chars = ''.join(
[random.choice(ascii_letters) for _ in range(8)])
unique_fragment = '%r.%s' % (time.time(), random_chars)
return os.path.join(dir_prefix, unique_fragment)
def login(self):
"""
Log in to koji
:return: koji.ClientSession instance, logged in
"""
session = koji.ClientSession(self.kojihub)
kwargs = {}
if self.koji_proxy_user:
kwargs['proxyuser'] = self.koji_proxy_user
if self.koji_ssl_certs:
# Use certificates
self.log.info("Using SSL certificates for Koji authentication")
session.ssl_login(os.path.join(self.koji_ssl_certs, 'cert'),
os.path.join(self.koji_ssl_certs, 'ca'),
os.path.join(self.koji_ssl_certs, 'serverca'),
**kwargs)
else:
# Use Kerberos
self.log.info("Using Kerberos for Koji authentication")
if self.koji_principal and self.koji_keytab:
kwargs['principal'] = self.koji_principal
kwargs['keytab'] = self.koji_keytab
session.krb_login(**kwargs)
return session
def run(self):
"""
Run the plugin.
"""
if ((self.koji_principal and not self.koji_keytab)
or (self.koji_keytab and not self.koji_principal)):
raise RuntimeError("specify both koji_principal and koji_keytab "
"or neither")
# Only run if the build was successful
if self.workflow.build_process_failed:
self.log.info("Not promoting failed build to koji")
return
if not is_rebuild(self.workflow):
self.log.info("Not promoting to koji: not a rebuild")
return
koji_metadata, output_files = self.get_metadata()
try:
session = self.login()
server_dir = self.get_upload_server_dir()
for output in output_files:
if output.file:
self.upload_file(session, output, server_dir)
finally:
for output in output_files:
if output.file:
output.file.close()
session.CGImport(koji_metadata, server_dir)
self.log.debug("Submitted with metadata: %s",
json.dumps(koji_metadata, sort_keys=True, indent=4))
class KojiPromotePlugin(ExitPlugin):
"""
Promote this build to Koji
Submits a successful build to Koji using the Content Generator API,
https://fedoraproject.org/wiki/Koji/ContentGenerators
Authentication is with Kerberos unless the koji_ssl_certs
configuration parameter is given, in which case it should be a
path at which 'cert', 'ca', and 'serverca' are the certificates
for SSL authentication.
If Kerberos is used for authentication, the default principal will
be used (from the kernel keyring) unless both koji_keytab and
koji_principal are specified. The koji_keytab parameter is a
keytab name like 'type:name', and so can be used to specify a key
in a Kubernetes secret by specifying 'FILE:/path/to/key'.
If metadata_only is set, the 'docker save' image will not be
uploaded, only the logs. The import will be marked as
metadata-only.
Runs as an exit plugin in order to capture logs from all other
plugins.
"""
key = PLUGIN_KOJI_PROMOTE_PLUGIN_KEY
is_allowed_to_fail = False
def __init__(self, tasker, workflow, kojihub, url,
verify_ssl=True, use_auth=True,
koji_ssl_certs=None, koji_proxy_user=None,
koji_principal=None, koji_keytab=None,
metadata_only=False, blocksize=None,
target=None, poll_interval=5):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param kojihub: string, koji hub (xmlrpc)
:param url: string, URL for OSv3 instance
:param verify_ssl: bool, verify OSv3 SSL certificate?
:param use_auth: bool, initiate authentication with OSv3?
:param koji_ssl_certs: str, path to 'cert', 'ca', 'serverca'
:param koji_proxy_user: str, user to log in as (requires hub config)
:param koji_principal: str, Kerberos principal (must specify keytab)
:param koji_keytab: str, keytab name (must specify principal)
:param metadata_only: bool, whether to omit the 'docker save' image
:param blocksize: int, blocksize to use for uploading files
:param target: str, koji target
:param poll_interval: int, seconds between Koji task status requests
"""
super(KojiPromotePlugin, self).__init__(tasker, workflow)
self.kojihub = kojihub
self.koji_ssl_certs = koji_ssl_certs
self.koji_proxy_user = koji_proxy_user
self.koji_principal = koji_principal
self.koji_keytab = koji_keytab
self.metadata_only = metadata_only
self.blocksize = blocksize
self.target = target
self.poll_interval = poll_interval
self.namespace = get_build_json().get('metadata', {}).get('namespace', None)
osbs_conf = Configuration(conf_file=None, openshift_uri=url,
use_auth=use_auth, verify_ssl=verify_ssl,
namespace=self.namespace)
self.osbs = OSBS(osbs_conf, osbs_conf)
self.build_id = None
self.pullspec_image = None
def get_rpms(self):
"""
Build a list of installed RPMs in the format required for the
metadata.
"""
tags = [
'NAME',
'VERSION',
'RELEASE',
'ARCH',
'EPOCH',
'SIGMD5',
'SIGPGP:pgpsig',
'SIGGPG:pgpsig',
]
cmd = "/bin/rpm " + rpm_qf_args(tags)
try:
# py3
(status, output) = subprocess.getstatusoutput(cmd)
except AttributeError:
# py2
with open('/dev/null', 'r+') as devnull:
p = subprocess.Popen(cmd,
shell=True,
stdin=devnull,
stdout=subprocess.PIPE,
stderr=devnull)
(stdout, stderr) = p.communicate()
status = p.wait()
output = stdout.decode()
if status != 0:
self.log.debug("%s: stderr output: %s", cmd, stderr)
raise RuntimeError("%s: exit code %s" % (cmd, status))
return parse_rpm_output(output.splitlines(), tags)
def get_output_metadata(self, path, filename):
"""
Describe a file by its metadata.
:return: dict
"""
checksums = get_checksums(path, ['md5'])
metadata = {'filename': filename,
'filesize': os.path.getsize(path),
'checksum': checksums['md5sum'],
'checksum_type': 'md5'}
if self.metadata_only:
metadata['metadata_only'] = True
return metadata
def get_builder_image_id(self):
"""
Find out the docker ID of the buildroot image we are in.
"""
try:
buildroot_tag = os.environ["OPENSHIFT_CUSTOM_BUILD_BASE_IMAGE"]
except KeyError:
return ''
try:
pod = self.osbs.get_pod_for_build(self.build_id)
all_images = pod.get_container_image_ids()
except OsbsException as ex:
self.log.error("unable to find image id: %r", ex)
return buildroot_tag
try:
return all_images[buildroot_tag]
except KeyError:
self.log.error("Unable to determine buildroot image ID for %s",
buildroot_tag)
return buildroot_tag
def get_buildroot(self, build_id):
"""
Build the buildroot entry of the metadata.
:return: dict, partial metadata
"""
docker_info = self.tasker.get_info()
host_arch, docker_version = get_docker_architecture(self.tasker)
buildroot = {
'id': 1,
'host': {
'os': docker_info['OperatingSystem'],
'arch': host_arch,
},
'content_generator': {
'name': PROG,
'version': atomic_reactor_version,
},
'container': {
'type': 'docker',
'arch': os.uname()[4],
},
'tools': [
{
'name': tool['name'],
'version': tool['version'],
}
for tool in get_version_of_tools()] + [
{
'name': 'docker',
'version': docker_version,
},
],
'components': self.get_rpms(),
'extra': {
'osbs': {
'build_id': build_id,
'builder_image_id': self.get_builder_image_id(),
}
},
}
return buildroot
def get_logs(self):
"""
Build the logs entry for the metadata 'output' section
:return: list, Output instances
"""
output = []
# Collect logs from server
try:
logs = self.osbs.get_build_logs(self.build_id)
except OsbsException as ex:
self.log.error("unable to get build logs: %r", ex)
else:
# Deleted once closed
logfile = NamedTemporaryFile(prefix=self.build_id,
suffix=".log",
mode='wb')
try:
logfile.write(logs)
except (TypeError, UnicodeEncodeError):
# Older osbs-client versions returned Unicode objects
logfile.write(logs.encode('utf-8'))
logfile.flush()
metadata = self.get_output_metadata(logfile.name,
"openshift-final.log")
output.append(Output(file=logfile, metadata=metadata))
docker_logs = NamedTemporaryFile(prefix="docker-%s" % self.build_id,
suffix=".log",
mode='wb')
docker_logs.write("\n".join(self.workflow.build_result.logs).encode('utf-8'))
docker_logs.flush()
output.append(Output(file=docker_logs,
metadata=self.get_output_metadata(docker_logs.name,
"build.log")))
return output
def get_image_components(self):
"""
Re-package the output of the rpmqa plugin into the format required
for the metadata.
"""
output = self.workflow.image_components
if output is None:
self.log.error("%s plugin did not run!",
PostBuildRPMqaPlugin.key)
output = []
return output
def get_image_output(self, arch):
"""
Create the output for the image
This is the Koji Content Generator metadata, along with the
'docker save' output to upload.
For metadata-only builds, an empty file is used instead of the
output of 'docker save'.
:param arch: str, architecture for this output
:return: tuple, (metadata dict, Output instance)
"""
saved_image = self.workflow.exported_image_sequence[-1].get('path')
image_name = get_image_upload_filename(self.workflow.exported_image_sequence[-1],
self.workflow.builder.image_id,
arch)
if self.metadata_only:
metadata = self.get_output_metadata(os.path.devnull, image_name)
output = Output(file=None, metadata=metadata)
else:
metadata = self.get_output_metadata(saved_image, image_name)
output = Output(file=open(saved_image), metadata=metadata)
return metadata, output
def get_digests(self):
"""
Returns a map of images to their digests
"""
try:
pulp = get_manifests_in_pulp_repository(self.workflow)
except KeyError:
pulp = None
digests = {} # repository -> digests
for registry in self.workflow.push_conf.docker_registries:
for image in self.workflow.tag_conf.images:
image_str = image.to_str()
if image_str in registry.digests:
image_digests = registry.digests[image_str]
if pulp is None:
digest_list = [image_digests.default]
else:
# If Pulp is enabled, only report digests that
# were synced into Pulp. This may not be all
# of them, depending on whether Pulp has
# schema 2 support.
digest_list = [digest for digest in (image_digests.v1,
image_digests.v2)
if digest in pulp]
digests[image.to_str(registry=False)] = digest_list
return digests
def get_repositories(self, digests):
"""
Build the repositories metadata
:param digests: dict, image -> digests
"""
if self.workflow.push_conf.pulp_registries:
# If pulp was used, only report pulp images
registries = self.workflow.push_conf.pulp_registries
else:
# Otherwise report all the images we pushed
registries = self.workflow.push_conf.all_registries
output_images = []
for registry in registries:
image = self.pullspec_image.copy()
image.registry = registry.uri
pullspec = image.to_str()
output_images.append(pullspec)
digest_list = digests.get(image.to_str(registry=False), ())
for digest in digest_list:
digest_pullspec = image.to_str(tag=False) + "@" + digest
output_images.append(digest_pullspec)
return output_images
def get_output(self, buildroot_id):
"""
Build the 'output' section of the metadata.
:return: list, Output instances
"""
def add_buildroot_id(output):
logfile, metadata = output
metadata.update({'buildroot_id': buildroot_id})
return Output(file=logfile, metadata=metadata)
def add_log_type(output):
logfile, metadata = output
metadata.update({'type': 'log', 'arch': 'noarch'})
return Output(file=logfile, metadata=metadata)
output_files = [add_log_type(add_buildroot_id(metadata))
for metadata in self.get_logs()]
# Parent of squashed built image is base image
image_id = self.workflow.builder.image_id
parent_id = self.workflow.base_image_inspect['Id']
# Read config from the registry using v2 schema 2 digest
registries = self.workflow.push_conf.docker_registries
if registries:
config = copy.deepcopy(registries[0].config)
else:
config = {}
# We don't need container_config section
if config and 'container_config' in config:
del config['container_config']
digests = self.get_digests()
repositories = self.get_repositories(digests)
arch = os.uname()[4]
tags = set(image.tag for image in self.workflow.tag_conf.primary_images)
metadata, output = self.get_image_output(arch)
metadata.update({
'arch': arch,
'type': 'docker-image',
'components': self.get_image_components(),
'extra': {
'image': {
'arch': arch,
},
'docker': {
'id': image_id,
'parent_id': parent_id,
'repositories': repositories,
'layer_sizes': self.workflow.layer_sizes,
'tags': list(tags),
'config': config
},
},
})
annotations = self.workflow.build_result.annotations
if annotations and 'digests' in annotations:
digests = get_digests_map_from_annotations(annotations['digests'])
metadata['extra']['docker']['digests'] = digests
if not config:
del metadata['extra']['docker']['config']
# Add the 'docker save' image to the output
image = add_buildroot_id(output)
output_files.append(image)
return output_files
def get_build(self, metadata):
start_time = int(atomic_reactor_start_time)
labels = df_parser(self.workflow.builder.df_path, workflow=self.workflow).labels
component = get_preferred_label(labels, 'com.redhat.component')
version = get_preferred_label(labels, 'version')
release = get_preferred_label(labels, 'release')
source = self.workflow.source
if not isinstance(source, GitSource):
raise RuntimeError('git source required')
extra = {'image': {'autorebuild': is_rebuild(self.workflow)}}
koji_task_id = metadata.get('labels', {}).get('koji-task-id')
if koji_task_id is not None:
self.log.info("build configuration created by Koji Task ID %s",
koji_task_id)
try:
extra['container_koji_task_id'] = int(koji_task_id)
except ValueError:
self.log.error("invalid task ID %r", koji_task_id, exc_info=1)
fs_result = self.workflow.prebuild_results.get(AddFilesystemPlugin.key)
if fs_result is not None:
try:
fs_task_id = fs_result['filesystem-koji-task-id']
except KeyError:
self.log.error("%s: expected filesystem-koji-task-id in result",
AddFilesystemPlugin.key)
else:
try:
task_id = int(fs_task_id)
except ValueError:
self.log.error("invalid task ID %r", fs_task_id, exc_info=1)
else:
extra['filesystem_koji_task_id'] = task_id
# Append media_types from pulp pull
pulp_pull_results = self.workflow.postbuild_results.get(PLUGIN_PULP_PULL_KEY)
if pulp_pull_results:
extra['image']['media_types'] = sorted(list(set(pulp_pull_results)))
# Append parent_build_id from koji parent
parent_results = self.workflow.prebuild_results.get(PLUGIN_KOJI_PARENT_KEY) or {}
parent_id = parent_results.get('parent-image-koji-build', {}).get('id')
if parent_id is not None:
try:
parent_id = int(parent_id)
except ValueError:
self.log.exception("invalid koji parent id %r", parent_id)
else:
extra['image']['parent_build_id'] = parent_id
# Append isolated build flag
try:
isolated = str(metadata['labels']['isolated']).lower() == 'true'
except (IndexError, AttributeError, KeyError):
isolated = False
self.log.info("build is isolated: %r", isolated)
extra['image']['isolated'] = isolated
help_result = self.workflow.prebuild_results.get(AddHelpPlugin.key)
if isinstance(help_result, dict) and 'help_file' in help_result and 'status' in help_result:
if help_result['status'] == AddHelpPlugin.NO_HELP_FILE_FOUND:
extra['image']['help'] = None
elif help_result['status'] == AddHelpPlugin.HELP_GENERATED:
extra['image']['help'] = help_result['help_file']
else:
self.log.error("Unknown result from add_help plugin: %s", help_result)
flatpak_source_info = get_flatpak_source_info(self.workflow)
if flatpak_source_info is not None:
extra['image'].update(flatpak_source_info.koji_metadata())
resolve_comp_result = self.workflow.prebuild_results.get(PLUGIN_RESOLVE_COMPOSES_KEY)
if resolve_comp_result:
extra['image']['odcs'] = {
'compose_ids': [item['id'] for item in resolve_comp_result['composes']],
'signing_intent': resolve_comp_result['signing_intent'],
'signing_intent_overridden': resolve_comp_result['signing_intent_overridden'],
}
build = {
'name': component,
'version': version,
'release': release,
'source': "{0}#{1}".format(source.uri, source.commit_id),
'start_time': start_time,
'end_time': int(time.time()),
'extra': extra,
}
if self.metadata_only:
build['metadata_only'] = True
return build
def get_metadata(self):
"""
Build the metadata needed for importing the build
:return: tuple, the metadata and the list of Output instances
"""
try:
metadata = get_build_json()["metadata"]
self.build_id = metadata["name"]
except KeyError:
self.log.error("No build metadata")
raise
for image in self.workflow.tag_conf.unique_images:
self.pullspec_image = image
break
for image in self.workflow.tag_conf.primary_images:
# dash at first/last postition does not count
if '-' in image.tag[1:-1]:
self.pullspec_image = image
break
if not self.pullspec_image:
raise RuntimeError('Unable to determine pullspec_image')
metadata_version = 0
build = self.get_build(metadata)
buildroot = self.get_buildroot(build_id=self.build_id)
output_files = self.get_output(buildroot['id'])
koji_metadata = {
'metadata_version': metadata_version,
'build': build,
'buildroots': [buildroot],
'output': [output.metadata for output in output_files],
}
return koji_metadata, output_files
def upload_file(self, session, output, serverdir):
"""
Upload a file to koji
:return: str, pathname on server
"""
name = output.metadata['filename']
self.log.debug("uploading %r to %r as %r",
output.file.name, serverdir, name)
kwargs = {}
if self.blocksize is not None:
kwargs['blocksize'] = self.blocksize
self.log.debug("using blocksize %d", self.blocksize)
upload_logger = KojiUploadLogger(self.log)
session.uploadWrapper(output.file.name, serverdir, name=name,
callback=upload_logger.callback, **kwargs)
path = os.path.join(serverdir, name)
self.log.debug("uploaded %r", path)
return path
@staticmethod
def get_upload_server_dir():
"""
Create a path name for uploading files to
:return: str, path name expected to be unique
"""
dir_prefix = 'koji-promote'
random_chars = ''.join([random.choice(ascii_letters)
for _ in range(8)])
unique_fragment = '%r.%s' % (time.time(), random_chars)
return os.path.join(dir_prefix, unique_fragment)
def login(self):
"""
Log in to koji
:return: koji.ClientSession instance, logged in
"""
# krbV python library throws an error if these are unicode
auth_info = {
"proxyuser": self.koji_proxy_user,
"ssl_certs_dir": self.koji_ssl_certs,
"krb_principal": str(self.koji_principal),
"krb_keytab": str(self.koji_keytab)
}
return create_koji_session(str(self.kojihub), auth_info)
def run(self):
"""
Run the plugin.
"""
if ((self.koji_principal and not self.koji_keytab) or
(self.koji_keytab and not self.koji_principal)):
raise RuntimeError("specify both koji_principal and koji_keytab "
"or neither")
# Only run if the build was successful
if self.workflow.build_process_failed:
self.log.info("Not promoting failed build to koji")
return
koji_metadata, output_files = self.get_metadata()
try:
session = self.login()
server_dir = self.get_upload_server_dir()
for output in output_files:
if output.file:
self.upload_file(session, output, server_dir)
finally:
for output in output_files:
if output.file:
output.file.close()
try:
build_info = session.CGImport(koji_metadata, server_dir)
except Exception:
self.log.debug("metadata: %r", koji_metadata)
raise
# Older versions of CGImport do not return a value.
build_id = build_info.get("id") if build_info else None
self.log.debug("Build information: %s",
json.dumps(build_info, sort_keys=True, indent=4))
# If configured, koji_tag_build plugin will perform build tagging
tag_later = are_plugins_in_order(self.workflow.exit_plugins_conf,
PLUGIN_KOJI_PROMOTE_PLUGIN_KEY,
PLUGIN_KOJI_TAG_BUILD_KEY)
if not tag_later and build_id is not None and self.target is not None:
tag_koji_build(session, build_id, self.target,
poll_interval=self.poll_interval)
return build_id
class KojiPromotePlugin(ExitPlugin):
"""
Promote this build to Koji
Submits a successful build to Koji using the Content Generator API,
https://fedoraproject.org/wiki/Koji/ContentGenerators
Authentication is with Kerberos unless the koji_ssl_certs
configuration parameter is given, in which case it should be a
path at which 'cert', 'ca', and 'serverca' are the certificates
for SSL authentication.
If Kerberos is used for authentication, the default principal will
be used (from the kernel keyring) unless both koji_keytab and
koji_principal are specified. The koji_keytab parameter is a
keytab name like 'type:name', and so can be used to specify a key
in a Kubernetes secret by specifying 'FILE:/path/to/key'.
If metadata_only is set, the 'docker save' image will not be
uploaded, only the logs. The import will be marked as
metadata-only.
Runs as an exit plugin in order to capture logs from all other
plugins.
"""
key = "koji_promote"
is_allowed_to_fail = False
def __init__(self, tasker, workflow, kojihub, url,
verify_ssl=True, use_auth=True,
koji_ssl_certs=None, koji_proxy_user=None,
koji_principal=None, koji_keytab=None,
metadata_only=False, blocksize=None,
target=None, poll_interval=5):
"""
constructor
:param tasker: DockerTasker instance
:param workflow: DockerBuildWorkflow instance
:param kojihub: string, koji hub (xmlrpc)
:param url: string, URL for OSv3 instance
:param verify_ssl: bool, verify OSv3 SSL certificate?
:param use_auth: bool, initiate authentication with OSv3?
:param koji_ssl_certs: str, path to 'cert', 'ca', 'serverca'
:param koji_proxy_user: str, user to log in as (requires hub config)
:param koji_principal: str, Kerberos principal (must specify keytab)
:param koji_keytab: str, keytab name (must specify principal)
:param metadata_only: bool, whether to omit the 'docker save' image
:param blocksize: int, blocksize to use for uploading files
:param target: str, koji target
:param poll_interval: int, seconds between Koji task status requests
"""
super(KojiPromotePlugin, self).__init__(tasker, workflow)
self.kojihub = kojihub
self.koji_ssl_certs = koji_ssl_certs
self.koji_proxy_user = koji_proxy_user
self.koji_principal = koji_principal
self.koji_keytab = koji_keytab
self.metadata_only = metadata_only
self.blocksize = blocksize
self.target = target
self.poll_interval = poll_interval
self.namespace = get_build_json().get('metadata', {}).get('namespace', None)
osbs_conf = Configuration(conf_file=None, openshift_uri=url,
use_auth=use_auth, verify_ssl=verify_ssl,
namespace=self.namespace)
self.osbs = OSBS(osbs_conf, osbs_conf)
self.build_id = None
self.nvr_image = None
@staticmethod
def parse_rpm_output(output, tags, separator=';'):
"""
Parse output of the rpm query.
:param output: list, decoded output (str) from the rpm subprocess
:param tags: list, str fields used for query output
:return: list, dicts describing each rpm package
"""
def field(tag):
"""
Get a field value by name
"""
try:
value = fields[tags.index(tag)]
except ValueError:
return None
if value == '(none)':
return None
return value
components = []
sigmarker = 'Key ID '
for rpm in output:
fields = rpm.rstrip('\n').split(separator)
if len(fields) < len(tags):
continue
signature = field('SIGPGP:pgpsig') or field('SIGGPG:pgpsig')
if signature:
parts = signature.split(sigmarker, 1)
if len(parts) > 1:
signature = parts[1]
component_rpm = {
'type': 'rpm',
'name': field('NAME'),
'version': field('VERSION'),
'release': field('RELEASE'),
'arch': field('ARCH'),
'sigmd5': field('SIGMD5'),
'signature': signature,
}
# Special handling for epoch as it must be an integer or None
epoch = field('EPOCH')
if epoch is not None:
epoch = int(epoch)
component_rpm['epoch'] = epoch
if component_rpm['name'] != 'gpg-pubkey':
components.append(component_rpm)
return components
def get_rpms(self):
"""
Build a list of installed RPMs in the format required for the
metadata.
"""
tags = [
'NAME',
'VERSION',
'RELEASE',
'ARCH',
'EPOCH',
'SIGMD5',
'SIGPGP:pgpsig',
'SIGGPG:pgpsig',
]
sep = ';'
fmt = sep.join(["%%{%s}" % tag for tag in tags])
cmd = "/bin/rpm -qa --qf '{0}\n'".format(fmt)
try:
# py3
(status, output) = subprocess.getstatusoutput(cmd)
except AttributeError:
# py2
with open('/dev/null', 'r+') as devnull:
p = subprocess.Popen(cmd,
shell=True,
stdin=devnull,
stdout=subprocess.PIPE,
stderr=devnull)
(stdout, stderr) = p.communicate()
status = p.wait()
output = stdout.decode()
if status != 0:
self.log.debug("%s: stderr output: %s", cmd, stderr)
raise RuntimeError("%s: exit code %s" % (cmd, status))
return self.parse_rpm_output(output.splitlines(), tags, separator=sep)
def get_output_metadata(self, path, filename):
"""
Describe a file by its metadata.
:return: dict
"""
checksums = get_checksums(path, ['md5'])
metadata = {'filename': filename,
'filesize': os.path.getsize(path),
'checksum': checksums['md5sum'],
'checksum_type': 'md5'}
if self.metadata_only:
metadata['metadata_only'] = True
return metadata
def get_builder_image_id(self):
"""
Find out the docker ID of the buildroot image we are in.
"""
try:
buildroot_tag = os.environ["OPENSHIFT_CUSTOM_BUILD_BASE_IMAGE"]
except KeyError:
return ''
try:
pod = self.osbs.get_pod_for_build(self.build_id)
all_images = pod.get_container_image_ids()
except OsbsException as ex:
self.log.error("unable to find image id: %r", ex)
return buildroot_tag
try:
return all_images[buildroot_tag]
except KeyError:
self.log.error("Unable to determine buildroot image ID for %s",
buildroot_tag)
return buildroot_tag
def get_buildroot(self, build_id):
"""
Build the buildroot entry of the metadata.
:return: dict, partial metadata
"""
docker_version = self.tasker.get_version()
docker_info = self.tasker.get_info()
host_arch = docker_version['Arch']
if host_arch == 'amd64':
host_arch = 'x86_64'
buildroot = {
'id': 1,
'host': {
'os': docker_info['OperatingSystem'],
'arch': host_arch,
},
'content_generator': {
'name': PROG,
'version': atomic_reactor_version,
},
'container': {
'type': 'docker',
'arch': os.uname()[4],
},
'tools': [
{
'name': tool['name'],
'version': tool['version'],
}
for tool in get_version_of_tools()] + [
{
'name': 'docker',
'version': docker_version['Version'],
},
],
'components': self.get_rpms(),
'extra': {
'osbs': {
'build_id': build_id,
'builder_image_id': self.get_builder_image_id(),
}
},
}
return buildroot
def get_logs(self):
"""
Build the logs entry for the metadata 'output' section
:return: list, Output instances
"""
output = []
# Collect logs from server
try:
logs = self.osbs.get_build_logs(self.build_id)
except OsbsException as ex:
self.log.error("unable to get build logs: %r", ex)
else:
# Deleted once closed
logfile = NamedTemporaryFile(prefix=self.build_id,
suffix=".log",
mode='w')
logfile.write(logs)
logfile.flush()
metadata = self.get_output_metadata(logfile.name,
"openshift-final.log")
output.append(Output(file=logfile, metadata=metadata))
docker_logs = NamedTemporaryFile(prefix="docker-%s" % self.build_id,
suffix=".log",
mode='w')
docker_logs.write("\n".join(self.workflow.build_logs))
docker_logs.flush()
output.append(Output(file=docker_logs,
metadata=self.get_output_metadata(docker_logs.name,
"build.log")))
return output
def get_image_components(self):
"""
Re-package the output of the rpmqa plugin into the format required
for the metadata.
"""
try:
output = self.workflow.postbuild_results[PostBuildRPMqaPlugin.key]
except KeyError:
self.log.error("%s plugin did not run!",
PostBuildRPMqaPlugin.key)
return []
return self.parse_rpm_output(output, PostBuildRPMqaPlugin.rpm_tags,
separator=',')
def get_image_output(self, arch):
"""
Create the output for the image
This is the Koji Content Generator metadata, along with the
'docker save' output to upload.
For metadata-only builds, an empty file is used instead of the
output of 'docker save'.
:param arch: str, architecture for this output
:return: tuple, (metadata dict, Output instance)
"""
image_id = self.workflow.builder.image_id
saved_image = self.workflow.exported_image_sequence[-1].get('path')
ext = saved_image.split('.', 1)[1]
name_fmt = 'docker-image-{id}.{arch}.{ext}'
image_name = name_fmt.format(id=image_id, arch=arch, ext=ext)
if self.metadata_only:
metadata = self.get_output_metadata(os.path.devnull, image_name)
output = Output(file=None, metadata=metadata)
else:
metadata = self.get_output_metadata(saved_image, image_name)
output = Output(file=open(saved_image), metadata=metadata)
return metadata, output
def get_digests(self):
"""
Returns a map of repositories to digests
"""
digests = {} # repository -> digest
for registry in self.workflow.push_conf.docker_registries:
for image in self.workflow.tag_conf.images:
image_str = image.to_str()
if image_str in registry.digests:
digest = registry.digests[image_str]
digests[image.to_str(registry=False)] = digest
return digests
def get_repositories(self, digests):
"""
Build the repositories metadata
:param digests: dict, repository -> digest
"""
if self.workflow.push_conf.pulp_registries:
# If pulp was used, only report pulp images
registries = self.workflow.push_conf.pulp_registries
else:
# Otherwise report all the images we pushed
registries = self.workflow.push_conf.all_registries
output_images = []
for registry in registries:
image = self.nvr_image.copy()
image.registry = registry.uri
pullspec = image.to_str()
output_images.append(pullspec)
digest = digests.get(image.to_str(registry=False))
if digest:
digest_pullspec = image.to_str(tag=False) + "@" + digest
output_images.append(digest_pullspec)
return output_images
def get_output(self, buildroot_id):
"""
Build the 'output' section of the metadata.
:return: list, Output instances
"""
def add_buildroot_id(output):
logfile, metadata = output
metadata.update({'buildroot_id': buildroot_id})
return Output(file=logfile, metadata=metadata)
def add_log_type(output):
logfile, metadata = output
metadata.update({'type': 'log', 'arch': 'noarch'})
return Output(file=logfile, metadata=metadata)
output_files = [add_log_type(add_buildroot_id(metadata))
for metadata in self.get_logs()]
# Parent of squashed built image is base image
image_id = self.workflow.builder.image_id
parent_id = self.workflow.base_image_inspect['Id']
digests = self.get_digests()
repositories = self.get_repositories(digests)
arch = os.uname()[4]
metadata, output = self.get_image_output(arch)
metadata.update({
'arch': arch,
'type': 'docker-image',
'components': self.get_image_components(),
'extra': {
'image': {
'arch': arch,
},
'docker': {
'id': image_id,
'parent_id': parent_id,
'repositories': repositories,
},
},
})
# Add the 'docker save' image to the output
image = add_buildroot_id(output)
output_files.append(image)
return output_files
def get_build(self, metadata):
start_time = int(atomic_reactor_start_time)
labels = DockerfileParser(self.workflow.builder.df_path).labels
component = get_preferred_label(labels, 'com.redhat.component')
version = get_preferred_label(labels, 'version')
release = get_preferred_label(labels, 'release')
source = self.workflow.source
if not isinstance(source, GitSource):
raise RuntimeError('git source required')
extra = {'image': {}}
koji_task_id = metadata.get('labels', {}).get('koji-task-id')
if koji_task_id is not None:
self.log.info("build configuration created by Koji Task ID %s",
koji_task_id)
extra['container_koji_task_id'] = koji_task_id
fs_result = self.workflow.prebuild_results.get(AddFilesystemPlugin.key)
if fs_result is not None:
try:
task_id = fs_result['filesystem-koji-task-id']
except KeyError:
self.log.error("%s: expected filesystem-koji-task-id in result",
AddFilesystemPlugin.key)
else:
extra['filesystem_koji_task_id'] = str(task_id)
build = {
'name': component,
'version': version,
'release': release,
'source': "{0}#{1}".format(source.uri, source.commit_id),
'start_time': start_time,
'end_time': int(time.time()),
'extra': extra,
}
if self.metadata_only:
build['metadata_only'] = True
return build
def get_metadata(self):
"""
Build the metadata needed for importing the build
:return: tuple, the metadata and the list of Output instances
"""
try:
metadata = get_build_json()["metadata"]
self.build_id = metadata["name"]
except KeyError:
self.log.error("No build metadata")
raise
for image in self.workflow.tag_conf.primary_images:
# dash at first/last postition does not count
if '-' in image.tag[1:-1]:
self.nvr_image = image
break
else:
raise RuntimeError('Unable to determine name:version-release')
metadata_version = 0
build = self.get_build(metadata)
buildroot = self.get_buildroot(build_id=self.build_id)
output_files = self.get_output(buildroot['id'])
koji_metadata = {
'metadata_version': metadata_version,
'build': build,
'buildroots': [buildroot],
'output': [output.metadata for output in output_files],
}
return koji_metadata, output_files
def upload_file(self, session, output, serverdir):
"""
Upload a file to koji
:return: str, pathname on server
"""
name = output.metadata['filename']
self.log.debug("uploading %r to %r as %r",
output.file.name, serverdir, name)
kwargs = {}
if self.blocksize is not None:
kwargs['blocksize'] = self.blocksize
self.log.debug("using blocksize %d", self.blocksize)
upload_logger = KojiUploadLogger(self.log)
session.uploadWrapper(output.file.name, serverdir, name=name,
callback=upload_logger.callback, **kwargs)
path = os.path.join(serverdir, name)
self.log.debug("uploaded %r", path)
return path
@staticmethod
def get_upload_server_dir():
"""
Create a path name for uploading files to
:return: str, path name expected to be unique
"""
dir_prefix = 'koji-promote'
random_chars = ''.join([random.choice(ascii_letters)
for _ in range(8)])
unique_fragment = '%r.%s' % (time.time(), random_chars)
return os.path.join(dir_prefix, unique_fragment)
def login(self):
"""
Log in to koji
:return: koji.ClientSession instance, logged in
"""
auth_info = {
"proxyuser": self.koji_proxy_user,
"ssl_certs_dir": self.koji_ssl_certs,
"krb_principal": self.koji_principal,
"krb_keytab": self.koji_keytab
}
return create_koji_session(self.kojihub, auth_info)
def run(self):
"""
Run the plugin.
"""
if ((self.koji_principal and not self.koji_keytab) or
(self.koji_keytab and not self.koji_principal)):
raise RuntimeError("specify both koji_principal and koji_keytab "
"or neither")
# Only run if the build was successful
if self.workflow.build_process_failed:
self.log.info("Not promoting failed build to koji")
return
koji_metadata, output_files = self.get_metadata()
try:
session = self.login()
server_dir = self.get_upload_server_dir()
for output in output_files:
if output.file:
self.upload_file(session, output, server_dir)
finally:
for output in output_files:
if output.file:
output.file.close()
try:
build_info = session.CGImport(koji_metadata, server_dir)
except Exception:
self.log.debug("metadata: %r", koji_metadata)
raise
# Older versions of CGImport do not return a value.
build_id = build_info.get("id") if build_info else None
self.log.debug("Build information: %s",
json.dumps(build_info, sort_keys=True, indent=4))
# Tag the build
if build_id is not None and self.target is not None:
self.log.debug("Finding build tag for target %s", self.target)
target_info = session.getBuildTarget(self.target)
build_tag = target_info['dest_tag_name']
self.log.info("Tagging build with %s", build_tag)
task_id = session.tagBuild(build_tag, build_id)
task = TaskWatcher(session, task_id,
poll_interval=self.poll_interval)
task.wait()
if task.failed():
raise RuntimeError("Task %s failed to tag koji build" % task_id)
return build_id
