def test_spec_name(self):
kwargs = {
# Params needed to avoid exceptions.
'user': '******',
'base_image': 'base_image',
'name_label': 'name_label',
'source_registry_uri': 'source_registry_uri',
# Params relevant to this test.
'git_uri': 'https://github.com/user/reponame.git',
'git_branch': 'master',
}
spec = ProdSpec()
spec.set_params(**kwargs)
assert spec.name.value == 'reponame-master'
class ProductionBuild(CommonProductionBuild):
key = PROD_BUILD_TYPE
def __init__(self, build_json_store, **kwargs):
super(ProductionBuild, self).__init__(build_json_store, **kwargs)
self.spec = ProdSpec()
def set_params(self, **kwargs):
"""
set parameters according to specification
these parameters are accepted:
:param koji_target: str, koji tag with packages used to build the image
:param kojiroot: str, URL from which koji packages are fetched
:param kojihub: str, URL of the koji hub
:param sources_command: str, command used to fetch dist-git sources
:param architecture: str, architecture we are building for
:param vendor: str, vendor name
:param build_host: str, host the build will run on
:param authoritative_registry: str, the docker registry authoritative for this image
:param metadata_plugin_use_auth: bool, use auth when posting metadata from dock?
"""
logger.debug("setting params '%s' for %s", kwargs, self.spec)
self.spec.set_params(**kwargs)
def render(self, validate=True):
if validate:
self.spec.validate()
super(ProductionBuild, self).render()
dj = DockJsonManipulator(self.template, self.inner_template)
self.template['parameters']['output']['imageTag'] = self.spec.image_tag.value
# if there is yum repo specified, don't pick stuff from koji
if self.spec.yum_repourls.value:
logger.info("removing koji from request, because there is yum repo specified")
dj.remove_plugin("prebuild_plugins", "koji")
else:
dj.dock_json_set_arg('prebuild_plugins', "koji", "target", self.spec.koji_target.value)
dj.dock_json_set_arg('prebuild_plugins', "koji", "root", self.spec.kojiroot.value)
dj.dock_json_set_arg('prebuild_plugins', "koji", "hub", self.spec.kojihub.value)
dj.write_dock_json()
self.build_json = self.template
logger.debug(self.build_json)
return self.build_json
def __init__(self, build_json_store, **kwargs):
super(ProductionBuild, self).__init__(build_json_store, **kwargs)
self.spec = ProdSpec()
class ProductionBuild(CommonBuild):
key = PROD_BUILD_TYPE
def __init__(self, build_json_store, **kwargs):
super(ProductionBuild, self).__init__(build_json_store, **kwargs)
self.spec = ProdSpec()
def set_params(self, **kwargs):
"""
set parameters according to specification
these parameters are accepted:
:param pulp_secret: str, resource name of pulp secret
:param koji_target: str, koji tag with packages used to build the image
:param kojiroot: str, URL from which koji packages are fetched
:param kojihub: str, URL of the koji hub
:param pulp_registry: str, name of pulp registry in dockpulp.conf
:param nfs_server_path: str, NFS server and path
:param nfs_dest_dir: str, directory to create on NFS server
:param sources_command: str, command used to fetch dist-git sources
:param architecture: str, architecture we are building for
:param vendor: str, vendor name
:param build_host: str, host the build will run on
:param authoritative_registry: str, the docker registry authoritative for this image
:param use_auth: bool, use auth from atomic-reactor?
:param git_push_url: str, URL for git push
"""
logger.debug("setting params '%s' for %s", kwargs, self.spec)
self.spec.set_params(**kwargs)
def render(self, validate=True):
if validate:
self.spec.validate()
super(ProductionBuild, self).render()
self.dj.dock_json_set_arg('prebuild_plugins', "distgit_fetch_artefacts",
"command", self.spec.sources_command.value)
self.dj.dock_json_set_arg('prebuild_plugins', "pull_base_image",
"parent_registry", self.spec.registry_uri.value)
implicit_labels = {
'Architecture': self.spec.architecture.value,
'Vendor': self.spec.vendor.value,
'Build_Host': self.spec.build_host.value,
'Authoritative_Registry': self.spec.authoritative_registry.value,
}
self.dj.dock_json_merge_arg('prebuild_plugins', "add_labels_in_dockerfile",
"labels", implicit_labels)
try:
self.dj.dock_json_set_arg('exit_plugins', "store_metadata_in_osv3",
"url", self.spec.openshift_uri.value)
except RuntimeError:
# For compatibility with older osbs.conf files
self.dj.dock_json_set_arg('postbuild_plugins', "store_metadata_in_osv3",
"url", self.spec.openshift_uri.value)
# If there are no triggers set, there is no point in running
# the check_and_set_rebuild, bump_release, or import_image plugins.
triggers = self.template['spec'].get('triggers', [])
if len(triggers) == 0:
for when, which in [("prebuild_plugins", "check_and_set_rebuild"),
("prebuild_plugins", "bump_release"),
("postbuild_plugins", "import_image")]:
logger.info("removing %s from request because there are no triggers",
which)
self.dj.remove_plugin(when, which)
# if there is yum repo specified, don't pick stuff from koji
if self.spec.yum_repourls.value:
logger.info("removing koji from request, because there is yum repo specified")
self.dj.remove_plugin("prebuild_plugins", "koji")
elif not (self.spec.koji_target.value and
self.spec.kojiroot.value and
self.spec.kojihub.value):
logger.info("removing koji from request as not specified")
self.dj.remove_plugin("prebuild_plugins", "koji")
else:
self.dj.dock_json_set_arg('prebuild_plugins', "koji",
"target", self.spec.koji_target.value)
self.dj.dock_json_set_arg('prebuild_plugins', "koji", "root", self.spec.kojiroot.value)
self.dj.dock_json_set_arg('prebuild_plugins', "koji", "hub", self.spec.kojihub.value)
# If the bump_release plugin is present, configure it
if self.dj.dock_json_has_plugin_conf('prebuild_plugins',
'bump_release'):
push_url = self.spec.git_push_url.value
if push_url is not None:
# Do we need to add in a username?
if self.spec.git_push_username.value is not None:
components = urlparse.urlsplit(push_url)
# Remove any existing username
netloc = components.netloc.split('@', 1)[-1]
# Add in the configured username
comps = list(components)
comps[1] = "%s@%s" % (self.spec.git_push_username.value,
netloc)
# Reassemble the URL
push_url = urlparse.urlunsplit(comps)
self.dj.dock_json_set_arg('prebuild_plugins', 'bump_release',
'push_url', push_url)
# Set the source git ref to the branch we're building
# from, but configure the plugin with the commit hash we
# started with.
logger.info("bump_release configured so setting source git ref to %s",
self.spec.git_branch.value)
self.template['spec']['source']['git']['ref'] = self.spec.git_branch.value
self.dj.dock_json_set_arg('prebuild_plugins', 'bump_release',
'git_ref', self.spec.git_ref.value)
# If there is a pulp secret, use it
if self.spec.pulp_secret.value:
name = self.spec.pulp_secret.value
if 'secrets' in self.template['spec']['strategy']['customStrategy']:
# origin 1.0.6 and newer
pulp_secret_path = os.path.join(SECRETS_PATH, name)
logger.info("Configuring pulp secret at %s", pulp_secret_path)
custom = self.template['spec']['strategy']['customStrategy']
custom['secrets'].append({
'secretSource': {
'name': name,
},
'mountPath': pulp_secret_path,
})
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_push',
'pulp_secret_path', pulp_secret_path)
else:
# origin 1.0.5 and earlier
logger.info("Configuring pulp secret as sourceSecret")
if 'sourceSecret' not in self.template['spec']['source']:
raise OsbsValidationException("JSON template does not allow secrets")
self.template['spec']['source']['sourceSecret']['name'] = name
# Don't push to docker registry, we're using pulp here
# but still construct the unique tag
self.template['spec']['output']['to']['name'] = self.spec.image_tag.value
else:
# Otherwise remove references to the secret
if 'sourceSecret' in self.template['spec']['source']:
del self.template['spec']['source']['sourceSecret']
if 'secrets' in self.template['spec']['strategy']['customStrategy']:
del self.template['spec']['strategy']['customStrategy']['secrets']
# If NFS destination set, use it
nfs_server_path = self.spec.nfs_server_path.value
if nfs_server_path:
self.dj.dock_json_set_arg('postbuild_plugins', 'cp_built_image_to_nfs',
'nfs_server_path', nfs_server_path)
self.dj.dock_json_set_arg('postbuild_plugins', 'cp_built_image_to_nfs',
'nfs_dest_dir', self.spec.nfs_dest_dir.value)
else:
# Otherwise, don't run the NFS plugin
self.dj.remove_plugin("postbuild_plugins", "cp_built_image_to_nfs")
# If a pulp registry is specified, use the pulp plugin
pulp_registry = self.spec.pulp_registry.value
if pulp_registry:
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_push',
'pulp_registry_name', pulp_registry)
# Verify we have either a secret or username/password
if self.spec.pulp_secret.value is None:
conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',
'pulp_push')
args = conf.get('args', {})
if 'username' not in args:
raise OsbsValidationException("Pulp registry specified "
"but no auth config")
else:
# If no pulp registry is specified, don't run the pulp plugin
self.dj.remove_plugin("postbuild_plugins", "pulp_push")
# Configure the import_image plugin
if self.dj.dock_json_has_plugin_conf('postbuild_plugins', 'import_image'):
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image', 'imagestream',
self.spec.imagestream_name.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image', 'docker_image_repo',
self.spec.imagestream_url.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image', 'url',
self.spec.openshift_uri.value)
if self.spec.use_auth.value is not None:
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image', 'use_auth',
self.spec.use_auth.value)
self.dj.write_dock_json()
self.build_json = self.template
logger.debug(self.build_json)
return self.build_json
class ProductionBuild(CommonBuild):
key = PROD_BUILD_TYPE
def __init__(self, build_json_store, **kwargs):
super(ProductionBuild, self).__init__(build_json_store, **kwargs)
self.spec = ProdSpec()
def set_params(self, **kwargs):
"""
set parameters according to specification
these parameters are accepted:
:param pulp_secret: str, resource name of pulp secret
:param pdc_secret: str, resource name of pdc secret
:param koji_target: str, koji tag with packages used to build the image
:param kojiroot: str, URL from which koji packages are fetched
:param kojihub: str, URL of the koji hub
:param pulp_registry: str, name of pulp registry in dockpulp.conf
:param nfs_server_path: str, NFS server and path
:param nfs_dest_dir: str, directory to create on NFS server
:param sources_command: str, command used to fetch dist-git sources
:param architecture: str, architecture we are building for
:param vendor: str, vendor name
:param build_host: str, host the build will run on
:param authoritative_registry: str, the docker registry authoritative for this image
:param distribution_scope: str, distribution scope for this image
(private, authoritative-source-only, restricted, public)
:param use_auth: bool, use auth from atomic-reactor?
:param git_push_url: str, URL for git push
"""
logger.debug("setting params '%s' for %s", kwargs, self.spec)
self.spec.set_params(**kwargs)
def set_secret_for_plugin(self, plugin, secret):
has_plugin_conf = self.dj.dock_json_has_plugin_conf(plugin[0],
plugin[1])
if 'secrets' in self.template['spec']['strategy']['customStrategy']:
if has_plugin_conf:
# origin 1.0.6 and newer
secret_path = os.path.join(SECRETS_PATH, secret)
logger.info("Configuring %s secret at %s", secret, secret_path)
custom = self.template['spec']['strategy']['customStrategy']
existing = [secret_mount for secret_mount in custom['secrets']
if secret_mount['secretSource']['name'] == secret]
if existing:
logger.debug("secret %s already set", plugin[1])
else:
custom['secrets'].append({
'secretSource': {
'name': secret,
},
'mountPath': secret_path,
})
self.dj.dock_json_set_arg(*(plugin + (secret_path,)))
else:
logger.debug("not setting secret for unused plugin %s",
plugin[1])
elif plugin[1] in ('pulp_push', 'pulp_sync'):
# setting pulp_push/pulp_sync secret for origin 1.0.5 and earlier
# we only use this way to preserve backwards compat for pulp_push plugin,
# other plugins must use the new secrets way above
logger.info("Configuring %s secret as sourceSecret", secret)
if 'sourceSecret' not in self.template['spec']['source']:
raise OsbsValidationException("JSON template does not allow secrets")
old_secret = self.template['spec']['source']['sourceSecret'].get('name')
if old_secret and old_secret != secret and not old_secret.startswith("{{"):
raise OsbsValidationException("Not possible to set two different source secrets")
self.template['spec']['source']['sourceSecret']['name'] = secret
elif has_plugin_conf:
raise OsbsValidationException("cannot set more than one secret "
"unless using OpenShift >= 1.0.6")
def set_secrets(self, secrets):
"""
:param secrets: dict, {(plugin type, plugin name, argument name): secret name}
for example {('exit_plugins', 'sendmail', 'pdc_secret_path'): 'pdc_secret', ...}
"""
secret_set = False
for (plugin, secret) in secrets.items():
if not isinstance(plugin, tuple) or len(plugin) != 3:
raise ValueError('got "%s" as secrets key, need 3-tuple' % plugin)
if secret is not None:
self.set_secret_for_plugin(plugin, secret)
secret_set = True
if not secret_set:
# remove references to secret if no secret was set
if 'sourceSecret' in self.template['spec']['source']:
del self.template['spec']['source']['sourceSecret']
if 'secrets' in self.template['spec']['strategy']['customStrategy']:
del self.template['spec']['strategy']['customStrategy']['secrets']
@staticmethod
def remove_tag_and_push_registries(tag_and_push_registries, version):
"""
Remove matching entries from tag_and_push_registries (in-place)
:param tag_and_push_registries: dict, uri -> dict
:param version: str, 'version' to match against
"""
registries = [uri
for uri, regdict in tag_and_push_registries.items()
if regdict['version'] == version]
for registry in registries:
logger.info("removing %s registry: %s", version, registry)
del tag_and_push_registries[registry]
def adjust_for_registry_api_versions(self):
"""
Enable/disable plugins depending on supported registry API versions
"""
versions = self.spec.registry_api_versions.value
try:
push_conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',
'tag_and_push')
tag_and_push_registries = push_conf['args']['registries']
except (KeyError, IndexError):
tag_and_push_registries = {}
if 'v1' not in versions:
# Remove v1-only plugins
for phase, name in [('postbuild_plugins', 'compress'),
('postbuild_plugins', 'cp_built_image_to_nfs'),
('postbuild_plugins', 'pulp_push')]:
logger.info("removing v1-only plugin: %s", name)
self.dj.remove_plugin(phase, name)
# remove extra tag_and_push config
self.remove_tag_and_push_registries(tag_and_push_registries, 'v1')
# Enable metadata-only imports to Koji
try:
koji_promote = self.dj.dock_json_get_plugin_conf('exit_plugins',
'koji_promote')
except (KeyError, IndexError):
pass
else:
args = koji_promote.setdefault('args', {})
args['metadata_only'] = True
if 'v2' not in versions:
# Remove v2-only plugins
logger.info("removing v2-only plugin: pulp_sync")
self.dj.remove_plugin('postbuild_plugins', 'pulp_sync')
# remove extra tag_and_push config
self.remove_tag_and_push_registries(tag_and_push_registries, 'v2')
# Remove 'version' from tag_and_push plugin config as it's no
# longer needed
for regdict in tag_and_push_registries.values():
if 'version' in regdict:
del regdict['version']
def adjust_for_triggers(self):
"""
Remove trigger-related plugins if no triggers set
If there are no triggers set, there is no point in running
the check_and_set_rebuild, bump_release, or import_image plugins.
"""
triggers = self.template['spec'].get('triggers', [])
if len(triggers) == 0:
for when, which in [("prebuild_plugins", "check_and_set_rebuild"),
("prebuild_plugins", "stop_autorebuild_if_disabled"),
("prebuild_plugins", "bump_release"),
("postbuild_plugins", "import_image"),
("exit_plugins", "koji_promote"),
("exit_plugins", "sendmail")]:
logger.info("removing %s from request because there are no triggers",
which)
self.dj.remove_plugin(when, which)
def render_add_labels_in_dockerfile(self):
implicit_labels = {
'Vendor': self.spec.vendor.value,
'Build_Host': self.spec.build_host.value,
'Authoritative_Registry': self.spec.authoritative_registry.value,
'distribution-scope': self.spec.distribution_scope.value,
}
architecture = self.spec.architecture.value
if architecture:
implicit_labels['Architecture'] = architecture
self.dj.dock_json_merge_arg('prebuild_plugins',
"add_labels_in_dockerfile",
"labels", implicit_labels)
def render_koji(self):
"""
if there is yum repo specified, don't pick stuff from koji
"""
if self.spec.yum_repourls.value:
logger.info("removing koji from request "
"because there is yum repo specified")
self.dj.remove_plugin("prebuild_plugins", "koji")
elif not (self.spec.koji_target.value and
self.spec.kojiroot.value and
self.spec.kojihub.value):
logger.info("removing koji from request as not specified")
self.dj.remove_plugin("prebuild_plugins", "koji")
else:
self.dj.dock_json_set_arg('prebuild_plugins', "koji",
"target", self.spec.koji_target.value)
self.dj.dock_json_set_arg('prebuild_plugins', "koji",
"root", self.spec.kojiroot.value)
self.dj.dock_json_set_arg('prebuild_plugins', "koji",
"hub", self.spec.kojihub.value)
def render_bump_release(self):
"""
If the bump_release plugin is present, configure it
"""
if self.dj.dock_json_has_plugin_conf('prebuild_plugins',
'bump_release'):
push_url = self.spec.git_push_url.value
if push_url is not None:
# Do we need to add in a username?
if self.spec.git_push_username.value is not None:
components = urlparse.urlsplit(push_url)
# Remove any existing username
netloc = components.netloc.split('@', 1)[-1]
# Add in the configured username
comps = list(components)
comps[1] = "%s@%s" % (self.spec.git_push_username.value,
netloc)
# Reassemble the URL
push_url = urlparse.urlunsplit(comps)
self.dj.dock_json_set_arg('prebuild_plugins', 'bump_release',
'push_url', push_url)
# Set the source git ref to the branch we're building
# from, but configure the plugin with the commit hash we
# started with.
logger.info("bump_release configured so "
"setting source git ref to %s",
self.spec.git_branch.value)
if looks_like_git_hash(self.spec.git_branch.value):
raise OsbsValidationException("git_branch parameter requires "
"branch name not hash")
if not looks_like_git_hash(self.spec.git_ref.value):
raise OsbsValidationException("git_ref parameter requires "
"hash not branch name")
self.template['spec']['source']['git']['ref'] = \
self.spec.git_branch.value
self.dj.dock_json_set_arg('prebuild_plugins', 'bump_release',
'git_ref', self.spec.git_ref.value)
def render_koji_promote(self, use_auth=None):
if not self.dj.dock_json_has_plugin_conf('exit_plugins',
'koji_promote'):
return
if self.spec.kojihub.value:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote', 'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'kojihub', self.spec.kojihub.value)
if use_auth is not None:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'use_auth', use_auth)
else:
logger.info("removing koji_promote from request as no kojihub "
"specified")
self.dj.remove_plugin("exit_plugins", "koji_promote")
def render_sendmail(self):
"""
if we have pdc_url and smtp_uri, configure sendmail plugin,
else remove it
"""
if not self.dj.dock_json_has_plugin_conf('exit_plugins', 'sendmail'):
return
if self.spec.pdc_url.value and self.spec.smtp_uri.value:
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'pdc_url',
self.spec.pdc_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'smtp_uri',
self.spec.smtp_uri.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'submitter',
self.spec.user.value)
# make sure we'll be able to authenticate to PDC
if 'pdc_secret_path' not in \
self.dj.dock_json_get_plugin_conf('exit_plugins',
'sendmail')['args']:
raise OsbsValidationException('sendmail plugin configured, '
'but no pdc_secret_path')
else:
logger.info("removing sendmail from request, "
"requires pdc_url and smtp_uri")
self.dj.remove_plugin('exit_plugins', 'sendmail')
def render_cp_built_image_to_nfs(self):
"""
If NFS destination set, use it
"""
if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'cp_built_image_to_nfs'):
return
nfs_server_path = self.spec.nfs_server_path.value
if nfs_server_path:
self.dj.dock_json_set_arg('postbuild_plugins', 'cp_built_image_to_nfs',
'nfs_server_path', nfs_server_path)
self.dj.dock_json_set_arg('postbuild_plugins',
'cp_built_image_to_nfs',
'dest_dir', self.spec.nfs_dest_dir.value)
else:
# Otherwise, don't run the NFS plugin
logger.info("removing cp_built_image_to_nfs from request, "
"requires nfs_server_path")
self.dj.remove_plugin("postbuild_plugins", "cp_built_image_to_nfs")
def render_pulp_push(self):
"""
If a pulp registry is specified, use the pulp plugin
"""
if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'pulp_push'):
return
pulp_registry = self.spec.pulp_registry.value
if pulp_registry:
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_push',
'pulp_registry_name', pulp_registry)
# Verify we have either a secret or username/password
if self.spec.pulp_secret.value is None:
conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',
'pulp_push')
args = conf.get('args', {})
if 'username' not in args:
raise OsbsValidationException("Pulp registry specified "
"but no auth config")
else:
# If no pulp registry is specified, don't run the pulp plugin
logger.info("removing pulp_push from request, "
"requires pulp_registry")
self.dj.remove_plugin("postbuild_plugins", "pulp_push")
def render_pulp_sync(self):
"""
If a pulp registry is specified, use the pulp plugin
"""
if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'pulp_sync'):
return
pulp_registry = self.spec.pulp_registry.value
docker_v2_registries = [registry
for registry in self.spec.registry_uris.value
if registry.version == 'v2']
if pulp_registry and docker_v2_registries:
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',
'pulp_registry_name', pulp_registry)
# First specified v2 registry is the one we'll tell pulp
# to sync from. Keep the http prefix -- pulp wants it.
docker_registry = docker_v2_registries[0].uri
logger.info("using docker v2 registry %s for pulp_sync",
docker_registry)
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',
'docker_registry', docker_registry)
# Verify we have either a secret or username/password
if self.spec.pulp_secret.value is None:
conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',
'pulp_sync')
args = conf.get('args', {})
if 'username' not in args:
raise OsbsValidationException("Pulp registry specified "
"but no auth config")
else:
# If no pulp registry is specified, don't run the pulp plugin
logger.info("removing pulp_sync from request, "
"requires pulp_registry and a v2 registry")
self.dj.remove_plugin("postbuild_plugins", "pulp_sync")
def render_import_image(self, use_auth=None):
"""
Configure the import_image plugin
"""
if self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'import_image'):
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'imagestream',
self.spec.imagestream_name.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'docker_image_repo',
self.spec.imagestream_url.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'build_json_dir',
self.spec.builder_build_json_dir.value)
use_auth = self.spec.use_auth.value
if use_auth is not None:
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'use_auth', use_auth)
if self.spec.imagestream_insecure_registry.value:
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'insecure_registry', True)
def render(self, validate=True):
if validate:
self.spec.validate()
super(ProductionBuild, self).render()
self.dj.dock_json_set_arg('prebuild_plugins', "distgit_fetch_artefacts",
"command", self.spec.sources_command.value)
# pull_base_image wants a docker URI so strip off the scheme part
source_registry = self.spec.source_registry_uri.value
self.dj.dock_json_set_arg('prebuild_plugins', "pull_base_image", "parent_registry",
source_registry.docker_uri if source_registry else None)
# The rebuild trigger requires git_branch and git_push_url
# parameters, but those parameters are optional. If either was
# not provided, remove the trigger.
remove_triggers = False
for param_name in ['git_branch', 'git_push_url']:
param = getattr(self.spec, param_name)
if not param.value:
logger.info("removing triggers as no %s specified", param_name)
remove_triggers = True
# Continue the loop so we log everything that's missing
if remove_triggers and 'triggers' in self.template['spec']:
del self.template['spec']['triggers']
self.adjust_for_triggers()
# Enable/disable plugins as needed for target registry API versions
self.adjust_for_registry_api_versions()
self.set_secrets({('postbuild_plugins',
'pulp_push',
'pulp_secret_path'):
self.spec.pulp_secret.value,
('postbuild_plugins',
'pulp_sync',
'pulp_secret_path'):
self.spec.pulp_secret.value,
('exit_plugins', 'sendmail', 'pdc_secret_path'):
self.spec.pdc_secret.value})
if self.spec.pulp_secret.value:
# Don't push to docker registry, we're using pulp here
# but still construct the unique tag
self.template['spec']['output']['to']['name'] = \
self.spec.image_tag.value
use_auth = self.spec.use_auth.value
self.render_add_labels_in_dockerfile()
self.render_koji()
self.render_bump_release()
self.render_cp_built_image_to_nfs()
self.render_import_image(use_auth=use_auth)
self.render_pulp_push()
self.render_pulp_sync()
self.render_koji_promote(use_auth=use_auth)
self.render_sendmail()
self.dj.write_dock_json()
self.build_json = self.template
logger.debug(self.build_json)
return self.build_json
def __init__(self, build_json_store, **kwargs):
super(ProductionBuild, self).__init__(build_json_store, **kwargs)
self.spec = ProdSpec()
# For the koji "scratch" build type
self.scratch = False
def __init__(self, build_json_store, **kwargs):
super(ProductionBuild, self).__init__(build_json_store, **kwargs)
self.spec = ProdSpec()
# For the koji "scratch" build type
self.scratch = False
class ProductionBuild(CommonBuild):
key = PROD_BUILD_TYPE
def __init__(self, build_json_store, **kwargs):
super(ProductionBuild, self).__init__(build_json_store, **kwargs)
self.spec = ProdSpec()
# For the koji "scratch" build type
self.scratch = False
def set_params(self, **kwargs):
"""
set parameters according to specification
these parameters are accepted:
:param pulp_secret: str, resource name of pulp secret
:param pdc_secret: str, resource name of pdc secret
:param koji_target: str, koji tag with packages used to build the image
:param kojiroot: str, URL from which koji packages are fetched
:param kojihub: str, URL of the koji hub
:param koji_certs_secret: str, resource name of secret that holds the koji certificates
:param pulp_registry: str, name of pulp registry in dockpulp.conf
:param nfs_server_path: str, NFS server and path
:param nfs_dest_dir: str, directory to create on NFS server
:param sources_command: str, command used to fetch dist-git sources
:param architecture: str, architecture we are building for
:param vendor: str, vendor name
:param build_host: str, host the build will run on or None for auto
:param authoritative_registry: str, the docker registry authoritative for this image
:param distribution_scope: str, distribution scope for this image
(private, authoritative-source-only, restricted, public)
:param use_auth: bool, use auth from atomic-reactor?
:param git_push_url: str, URL for git push
"""
# Here we cater to the koji "scratch" build type, this will disable
# all plugins that might cause importing of data to koji
try:
self.scratch = kwargs.pop("scratch")
except KeyError:
pass
logger.debug("setting params '%s' for %s", kwargs, self.spec)
self.spec.set_params(**kwargs)
def set_secret_for_plugin(self, plugin, secret):
has_plugin_conf = self.dj.dock_json_has_plugin_conf(
plugin[0], plugin[1])
if 'secrets' in self.template['spec']['strategy']['customStrategy']:
if has_plugin_conf:
# origin 1.0.6 and newer
secret_path = os.path.join(SECRETS_PATH, secret)
logger.info("Configuring %s secret at %s", secret, secret_path)
custom = self.template['spec']['strategy']['customStrategy']
existing = [
secret_mount for secret_mount in custom['secrets']
if secret_mount['secretSource']['name'] == secret
]
if existing:
logger.debug("secret %s already set", plugin[1])
else:
custom['secrets'].append({
'secretSource': {
'name': secret,
},
'mountPath': secret_path,
})
self.dj.dock_json_set_arg(*(plugin + (secret_path, )))
else:
logger.debug("not setting secret for unused plugin %s",
plugin[1])
elif plugin[1] in ('pulp_push', 'pulp_sync'):
# setting pulp_push/pulp_sync secret for origin 1.0.5 and earlier
# we only use this way to preserve backwards compat for pulp_push plugin,
# other plugins must use the new secrets way above
logger.info("Configuring %s secret as sourceSecret", secret)
if 'sourceSecret' not in self.template['spec']['source']:
raise OsbsValidationException(
"JSON template does not allow secrets")
old_secret = self.template['spec']['source']['sourceSecret'].get(
'name')
if old_secret and old_secret != secret and not old_secret.startswith(
"{{"):
raise OsbsValidationException(
"Not possible to set two different source secrets")
self.template['spec']['source']['sourceSecret']['name'] = secret
elif has_plugin_conf:
raise OsbsValidationException("cannot set more than one secret "
"unless using OpenShift >= 1.0.6")
def set_secrets(self, secrets):
"""
:param secrets: dict, {(plugin type, plugin name, argument name): secret name}
for example {('exit_plugins', 'sendmail', 'pdc_secret_path'): 'pdc_secret', ...}
"""
secret_set = False
for (plugin, secret) in secrets.items():
if not isinstance(plugin, tuple) or len(plugin) != 3:
raise ValueError('got "%s" as secrets key, need 3-tuple' %
plugin)
if secret is not None:
self.set_secret_for_plugin(plugin, secret)
secret_set = True
if not secret_set:
# remove references to secret if no secret was set
if 'sourceSecret' in self.template['spec']['source']:
del self.template['spec']['source']['sourceSecret']
if 'secrets' in self.template['spec']['strategy'][
'customStrategy']:
del self.template['spec']['strategy']['customStrategy'][
'secrets']
@staticmethod
def remove_tag_and_push_registries(tag_and_push_registries, version):
"""
Remove matching entries from tag_and_push_registries (in-place)
:param tag_and_push_registries: dict, uri -> dict
:param version: str, 'version' to match against
"""
registries = [
uri for uri, regdict in tag_and_push_registries.items()
if regdict['version'] == version
]
for registry in registries:
logger.info("removing %s registry: %s", version, registry)
del tag_and_push_registries[registry]
def adjust_for_registry_api_versions(self):
"""
Enable/disable plugins depending on supported registry API versions
"""
versions = self.spec.registry_api_versions.value
try:
push_conf = self.dj.dock_json_get_plugin_conf(
'postbuild_plugins', 'tag_and_push')
tag_and_push_registries = push_conf['args']['registries']
except (KeyError, IndexError):
tag_and_push_registries = {}
if 'v1' not in versions:
# Remove v1-only plugins
for phase, name in [('postbuild_plugins', 'pulp_push')]:
logger.info("removing v1-only plugin: %s", name)
self.dj.remove_plugin(phase, name)
# remove extra tag_and_push config
self.remove_tag_and_push_registries(tag_and_push_registries, 'v1')
if 'v2' not in versions:
# Remove v2-only plugins
logger.info("removing v2-only plugin: pulp_sync")
self.dj.remove_plugin('postbuild_plugins', 'pulp_sync')
# remove extra tag_and_push config
self.remove_tag_and_push_registries(tag_and_push_registries, 'v2')
# Remove 'version' from tag_and_push plugin config as it's no
# longer needed
for regdict in tag_and_push_registries.values():
if 'version' in regdict:
del regdict['version']
def adjust_for_triggers(self):
"""
Remove trigger-related plugins if no triggers set
If there are no triggers set, there is no point in running
the check_and_set_rebuild, bump_release, or import_image plugins.
"""
triggers = self.template['spec'].get('triggers', [])
if len(triggers) == 0:
for when, which in [("prebuild_plugins", "check_and_set_rebuild"),
("prebuild_plugins",
"stop_autorebuild_if_disabled"),
("prebuild_plugins", "bump_release"),
("postbuild_plugins", "import_image"),
("exit_plugins", "sendmail")]:
logger.info(
"removing %s from request because there are no triggers",
which)
self.dj.remove_plugin(when, which)
def adjust_for_scratch(self):
"""
Remove koji Content Generator related plugins if no triggers set in
order to hadle the "scratch build" scenario
"""
if self.scratch:
# Note: only one for now, but left in a list like other adjust_for_
# functions in the event that this needs to be expanded
for when, which in [
("exit_plugins", "koji_promote"),
]:
logger.info(
"removing %s from request because there are no triggers",
which)
self.dj.remove_plugin(when, which)
def render_add_labels_in_dockerfile(self):
implicit_labels = {
'Vendor': self.spec.vendor.value,
'Authoritative_Registry': self.spec.authoritative_registry.value,
'distribution-scope': self.spec.distribution_scope.value,
}
build_host = self.spec.build_host.value
if build_host:
implicit_labels['Build_Host'] = build_host
architecture = self.spec.architecture.value
if architecture:
implicit_labels['Architecture'] = architecture
self.dj.dock_json_merge_arg('prebuild_plugins',
"add_labels_in_dockerfile", "labels",
implicit_labels)
def render_koji(self):
"""
if there is yum repo specified, don't pick stuff from koji
"""
if self.spec.yum_repourls.value:
logger.info("removing koji from request "
"because there is yum repo specified")
self.dj.remove_plugin("prebuild_plugins", "koji")
elif not (self.spec.koji_target.value and self.spec.kojiroot.value
and self.spec.kojihub.value):
logger.info("removing koji from request as not specified")
self.dj.remove_plugin("prebuild_plugins", "koji")
else:
self.dj.dock_json_set_arg('prebuild_plugins', "koji", "target",
self.spec.koji_target.value)
self.dj.dock_json_set_arg('prebuild_plugins', "koji", "root",
self.spec.kojiroot.value)
self.dj.dock_json_set_arg('prebuild_plugins', "koji", "hub",
self.spec.kojihub.value)
def render_bump_release(self):
"""
If the bump_release plugin is present, configure it
"""
if self.dj.dock_json_has_plugin_conf('prebuild_plugins',
'bump_release'):
push_url = self.spec.git_push_url.value
if push_url is not None:
# Do we need to add in a username?
if self.spec.git_push_username.value is not None:
components = urlparse.urlsplit(push_url)
# Remove any existing username
netloc = components.netloc.split('@', 1)[-1]
# Add in the configured username
comps = list(components)
comps[1] = "%s@%s" % (self.spec.git_push_username.value,
netloc)
# Reassemble the URL
push_url = urlparse.urlunsplit(comps)
self.dj.dock_json_set_arg('prebuild_plugins', 'bump_release',
'push_url', push_url)
# Set the source git ref to the branch we're building
# from, but configure the plugin with the commit hash we
# started with.
logger.info(
"bump_release configured so "
"setting source git ref to %s", self.spec.git_branch.value)
if looks_like_git_hash(self.spec.git_branch.value):
raise OsbsValidationException("git_branch parameter requires "
"branch name not hash")
if not looks_like_git_hash(self.spec.git_ref.value):
raise OsbsValidationException("git_ref parameter requires "
"hash not branch name")
self.template['spec']['source']['git']['ref'] = \
self.spec.git_branch.value
self.dj.dock_json_set_arg('prebuild_plugins', 'bump_release',
'git_ref', self.spec.git_ref.value)
def render_koji_promote(self, use_auth=None):
if not self.dj.dock_json_has_plugin_conf('exit_plugins',
'koji_promote'):
return
if self.spec.kojihub.value:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote', 'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'kojihub', self.spec.kojihub.value)
if use_auth is not None:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'use_auth', use_auth)
else:
logger.info("removing koji_promote from request as no kojihub "
"specified")
self.dj.remove_plugin("exit_plugins", "koji_promote")
def render_sendmail(self):
"""
if we have pdc_url and smtp_uri, configure sendmail plugin,
else remove it
"""
if not self.dj.dock_json_has_plugin_conf('exit_plugins', 'sendmail'):
return
if self.spec.pdc_url.value and self.spec.smtp_uri.value:
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'pdc_url',
self.spec.pdc_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'smtp_uri',
self.spec.smtp_uri.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'submitter',
self.spec.user.value)
# make sure we'll be able to authenticate to PDC
if 'pdc_secret_path' not in \
self.dj.dock_json_get_plugin_conf('exit_plugins',
'sendmail')['args']:
raise OsbsValidationException('sendmail plugin configured, '
'but no pdc_secret_path')
else:
logger.info("removing sendmail from request, "
"requires pdc_url and smtp_uri")
self.dj.remove_plugin('exit_plugins', 'sendmail')
def render_pulp_push(self):
"""
If a pulp registry is specified, use the pulp plugin
"""
if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'pulp_push'):
return
pulp_registry = self.spec.pulp_registry.value
if pulp_registry:
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_push',
'pulp_registry_name', pulp_registry)
# Verify we have either a secret or username/password
if self.spec.pulp_secret.value is None:
conf = self.dj.dock_json_get_plugin_conf(
'postbuild_plugins', 'pulp_push')
args = conf.get('args', {})
if 'username' not in args:
raise OsbsValidationException("Pulp registry specified "
"but no auth config")
else:
# If no pulp registry is specified, don't run the pulp plugin
logger.info("removing pulp_push from request, "
"requires pulp_registry")
self.dj.remove_plugin("postbuild_plugins", "pulp_push")
def render_pulp_sync(self):
"""
If a pulp registry is specified, use the pulp plugin
"""
if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'pulp_sync'):
return
pulp_registry = self.spec.pulp_registry.value
docker_v2_registries = [
registry for registry in self.spec.registry_uris.value
if registry.version == 'v2'
]
if pulp_registry and docker_v2_registries:
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',
'pulp_registry_name', pulp_registry)
# First specified v2 registry is the one we'll tell pulp
# to sync from. Keep the http prefix -- pulp wants it.
docker_registry = docker_v2_registries[0].uri
logger.info("using docker v2 registry %s for pulp_sync",
docker_registry)
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',
'docker_registry', docker_registry)
# Verify we have either a secret or username/password
if self.spec.pulp_secret.value is None:
conf = self.dj.dock_json_get_plugin_conf(
'postbuild_plugins', 'pulp_sync')
args = conf.get('args', {})
if 'username' not in args:
raise OsbsValidationException("Pulp registry specified "
"but no auth config")
else:
# If no pulp registry is specified, don't run the pulp plugin
logger.info("removing pulp_sync from request, "
"requires pulp_registry and a v2 registry")
self.dj.remove_plugin("postbuild_plugins", "pulp_sync")
def render_import_image(self, use_auth=None):
"""
Configure the import_image plugin
"""
if self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'import_image'):
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'imagestream',
self.spec.imagestream_name.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'docker_image_repo',
self.spec.imagestream_url.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'build_json_dir',
self.spec.builder_build_json_dir.value)
use_auth = self.spec.use_auth.value
if use_auth is not None:
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'use_auth', use_auth)
if self.spec.imagestream_insecure_registry.value:
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'insecure_registry', True)
def render(self, validate=True):
if validate:
self.spec.validate()
super(ProductionBuild, self).render()
self.dj.dock_json_set_arg('prebuild_plugins',
"distgit_fetch_artefacts", "command",
self.spec.sources_command.value)
# pull_base_image wants a docker URI so strip off the scheme part
source_registry = self.spec.source_registry_uri.value
self.dj.dock_json_set_arg(
'prebuild_plugins', "pull_base_image", "parent_registry",
source_registry.docker_uri if source_registry else None)
# The rebuild trigger requires git_branch and git_push_url
# parameters, but those parameters are optional. If either was
# not provided, remove the trigger.
remove_triggers = False
for param_name in ['git_branch', 'git_push_url']:
param = getattr(self.spec, param_name)
if not param.value:
logger.info("removing triggers as no %s specified", param_name)
remove_triggers = True
# Continue the loop so we log everything that's missing
if remove_triggers and 'triggers' in self.template['spec']:
del self.template['spec']['triggers']
self.adjust_for_triggers()
self.adjust_for_scratch()
# Enable/disable plugins as needed for target registry API versions
self.adjust_for_registry_api_versions()
self.set_secrets({
('postbuild_plugins', 'pulp_push', 'pulp_secret_path'):
self.spec.pulp_secret.value,
('postbuild_plugins', 'pulp_sync', 'pulp_secret_path'):
self.spec.pulp_secret.value,
('exit_plugins', 'sendmail', 'pdc_secret_path'):
self.spec.pdc_secret.value,
('exit_plugins', 'koji_promote', 'koji_ssl_certs'):
self.spec.koji_certs_secret.value
})
if self.spec.pulp_secret.value:
# Don't push to docker registry, we're using pulp here
# but still construct the unique tag
self.template['spec']['output']['to']['name'] = \
self.spec.image_tag.value
use_auth = self.spec.use_auth.value
self.render_add_labels_in_dockerfile()
self.render_koji()
self.render_bump_release()
self.render_import_image(use_auth=use_auth)
self.render_pulp_push()
self.render_pulp_sync()
self.render_koji_promote(use_auth=use_auth)
self.render_sendmail()
self.dj.write_dock_json()
self.build_json = self.template
logger.debug(self.build_json)
return self.build_json