def test_image_tag(self, rand, timestr, platform, arrangement, has_platform_suffix):
kwargs = self.get_minimal_kwargs()
kwargs.update({
'component': 'foo',
'koji_target': 'tothepoint',
'arrangement_version': arrangement,
})
if platform:
kwargs['platform'] = platform
(flexmock(sys.modules['osbs.build.spec'])
.should_receive('utcnow').once()
.and_return(datetime.datetime.strptime(timestr, '%Y%m%d%H%M%S')))
(flexmock(random)
.should_receive('randrange').once()
.with_args(10**(len(rand) - 1), 10**len(rand))
.and_return(int(rand)))
spec = BuildSpec()
spec.set_params(**kwargs)
img_tag = '{user}/{component}:{koji_target}-{random_number}-{time_string}'
if has_platform_suffix:
img_tag += '-{platform}'
img_tag = img_tag.format(random_number=rand, time_string=timestr, **kwargs)
assert spec.image_tag.value == img_tag
def test_image_tag(self, rand, timestr, platform, arrangement,
has_platform_suffix):
kwargs = self.get_minimal_kwargs()
kwargs.update({
'component': 'foo',
'koji_target': 'tothepoint',
'arrangement_version': arrangement,
})
if platform:
kwargs['platform'] = platform
(flexmock(sys.modules['osbs.build.spec']).should_receive(
'utcnow').once().and_return(
datetime.datetime.strptime(timestr, '%Y%m%d%H%M%S')))
(flexmock(random).should_receive('randrange').once().with_args(
10**(len(rand) - 1), 10**len(rand)).and_return(int(rand)))
spec = BuildSpec()
spec.set_params(**kwargs)
img_tag = '{user}/{component}:{koji_target}-{random_number}-{time_string}'
if has_platform_suffix:
img_tag += '-{platform}'
img_tag = img_tag.format(random_number=rand,
time_string=timestr,
**kwargs)
assert spec.image_tag.value == img_tag
def test_labels(self):
kwargs = self.get_minimal_kwargs()
kwargs.update({
'labels': {'spam': 'maps'},
})
spec = BuildSpec()
spec.set_params(**kwargs)
assert spec.labels.value == {'spam': 'maps'}
def test_labels(self):
kwargs = self.get_minimal_kwargs()
kwargs.update({
'labels': {
'spam': 'maps'
},
})
spec = BuildSpec()
spec.set_params(**kwargs)
assert spec.labels.value == {'spam': 'maps'}
def test_spec_name(self):
kwargs = self.get_minimal_kwargs()
kwargs.update({
'git_uri': 'https://github.com/user/reponame.git',
'git_branch': 'master',
})
spec = BuildSpec()
spec.set_params(**kwargs)
assert spec.name.value.startswith('reponame-master')
registry = spec.registry_uris.value[0]
assert registry.uri == 'http://registry.example.com:5000'
assert registry.docker_uri == 'registry.example.com:5000'
assert registry.version == 'v2'
def __init__(self, build_json_store):
"""
:param build_json_store: str, path to directory with JSON build files
"""
self.spec = BuildSpec()
self.build_json_store = build_json_store
self.build_json = None # rendered template
self._template = None # template loaded from filesystem
self._inner_template = None # dock json
self._dj = None
self._resource_limits = None
self._openshift_required_version = parse_version('1.0.6')
# For the koji "scratch" build type
self.scratch = False
self.base_image = None
def test_spec_name(self):
kwargs = self.get_minimal_kwargs()
kwargs.update({
'git_uri': 'https://github.com/user/reponame.git',
'git_branch': 'master',
})
spec = BuildSpec()
spec.set_params(**kwargs)
assert spec.name.value == 'reponame-master'
registry = spec.registry_uris.value[0]
assert registry.uri == 'http://registry.example.com:5000'
assert registry.docker_uri == 'registry.example.com:5000'
assert registry.version == 'v2'
def test_compose_ids_and_signing_intent(self, odcs_enabled, signing_intent, compose_ids,
yum_repourls, exc):
kwargs = self.get_minimal_kwargs()
if odcs_enabled:
kwargs.update({
'odcs_url': "http://odcs.com",
})
if signing_intent:
kwargs['signing_intent'] = signing_intent
if compose_ids:
kwargs['compose_ids'] = compose_ids
if yum_repourls:
kwargs['yum_repourls'] = yum_repourls
kwargs.update({
'git_uri': 'https://github.com/user/reponame.git',
'git_branch': 'master',
})
spec = BuildSpec()
if exc:
with pytest.raises(exc):
spec.set_params(**kwargs)
else:
spec.set_params(**kwargs)
if yum_repourls:
assert spec.yum_repourls.value == yum_repourls
if signing_intent:
assert spec.signing_intent.value == signing_intent
if compose_ids:
assert spec.compose_ids.value == compose_ids
def __init__(self, build_json_store, inner_template=None,
outer_template=None, customize_conf=None):
"""
:param build_json_store: str, path to directory with JSON build files
:param inner_template: str, path to inner template JSON
:param outer_template: str, path to outer template JSON
:param customize_conf: str, path to customize configuration JSON
"""
self.spec = BuildSpec()
self.build_json_store = build_json_store
self._inner_template_path = inner_template or DEFAULT_INNER_TEMPLATE
self._outer_template_path = outer_template or DEFAULT_OUTER_TEMPLATE
self._customize_conf_path = customize_conf or DEFAULT_CUSTOMIZE_CONF
self.build_json = None # rendered template
self._template = None # template loaded from filesystem
self._inner_template = None # dock json
self._customize_conf = None # site customize conf for _inner_template
self._dj = None
self._resource_limits = None
self._openshift_required_version = parse_version('1.0.6')
# For the koji "scratch" build type
self.scratch = None
self.base_image = None
def __init__(self, build_json_store):
"""
:param build_json_store: str, path to directory with JSON build files
"""
self.spec = BuildSpec()
self.build_json_store = build_json_store
self.build_json = None # rendered template
self._template = None # template loaded from filesystem
self._inner_template = None # dock json
self._dj = None
self._resource_limits = None
self._openshift_required_version = parse_version('1.0.6')
# For the koji "scratch" build type
self.scratch = False
self.base_image = None
class BuildRequest(object):
"""
Wraps logic for creating build inputs
"""
def __init__(self, build_json_store):
"""
:param build_json_store: str, path to directory with JSON build files
"""
self.spec = BuildSpec()
self.build_json_store = build_json_store
self.build_json = None # rendered template
self._template = None # template loaded from filesystem
self._inner_template = None # dock json
self._dj = None
self._resource_limits = None
self._openshift_required_version = parse_version('1.0.6')
# For the koji "scratch" build type
self.scratch = False
self.base_image = None
def set_params(self, **kwargs):
"""
set parameters according to specification
these parameters are accepted:
:param pulp_secret: str, resource name of pulp secret
:param pdc_secret: str, resource name of pdc secret
:param koji_target: str, koji tag with packages used to build the image
:param kojiroot: str, URL from which koji packages are fetched
:param kojihub: str, URL of the koji hub
:param koji_certs_secret: str, resource name of secret that holds the koji certificates
:param koji_task_id: int, Koji Task that created this build config
:param pulp_registry: str, name of pulp registry in dockpulp.conf
:param nfs_server_path: str, NFS server and path
:param nfs_dest_dir: str, directory to create on NFS server
:param sources_command: str, command used to fetch dist-git sources
:param architecture: str, architecture we are building for
:param vendor: str, vendor name
:param build_host: str, host the build will run on or None for auto
:param authoritative_registry: str, the docker registry authoritative for this image
:param distribution_scope: str, distribution scope for this image
(private, authoritative-source-only, restricted, public)
:param use_auth: bool, use auth from atomic-reactor?
:param git_push_url: str, URL for git push
"""
# Here we cater to the koji "scratch" build type, this will disable
# all plugins that might cause importing of data to koji
try:
self.scratch = kwargs.pop("scratch")
except KeyError:
pass
self.base_image = kwargs.get('base_image')
logger.debug("setting params '%s' for %s", kwargs, self.spec)
self.spec.set_params(**kwargs)
def set_resource_limits(self, cpu=None, memory=None, storage=None):
if self._resource_limits is None:
self._resource_limits = {}
if cpu is not None:
self._resource_limits['cpu'] = cpu
if memory is not None:
self._resource_limits['memory'] = memory
if storage is not None:
self._resource_limits['storage'] = storage
def set_openshift_required_version(self, openshift_required_version):
if openshift_required_version is not None:
self._openshift_required_version = openshift_required_version
@property
def build_id(self):
return self.build_json['metadata']['name']
@property
def template(self):
if self._template is None:
path = os.path.join(self.build_json_store, DEFAULT_OUTER_TEMPLATE)
logger.debug("loading template from path %s", path)
try:
with open(path, "r") as fp:
self._template = json.load(fp)
except (IOError, OSError) as ex:
raise OsbsException("Can't open template '%s': %s" %
(path, repr(ex)))
return self._template
@property
def inner_template(self):
if self._inner_template is None:
path = os.path.join(self.build_json_store, DEFAULT_INNER_TEMPLATE)
logger.debug("loading inner template from path %s", path)
with open(path, "r") as fp:
self._inner_template = json.load(fp)
return self._inner_template
@property
def dj(self):
if self._dj is None:
self._dj = DockJsonManipulator(self.template, self.inner_template)
return self._dj
def is_auto_instantiated(self):
"""Return True if this BuildConfig will be automatically instantiated when created."""
triggers = self.template['spec'].get('triggers', [])
for trigger in triggers:
if trigger['type'] == 'ImageChange' and \
trigger['imageChange']['from']['kind'] == 'ImageStreamTag':
return True
return False
def set_label(self, name, value):
self.template['metadata'].setdefault('labels', {})
self.template['metadata']['labels'][name] = value
def render_resource_limits(self):
if self._resource_limits is not None:
resources = self.template['spec'].get('resources', {})
limits = resources.get('limits', {})
limits.update(self._resource_limits)
resources['limits'] = limits
self.template['spec']['resources'] = resources
def render_tag_and_push_registries(self):
if self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'tag_and_push'):
push_conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',
'tag_and_push')
args = push_conf.setdefault('args', {})
registries = args.setdefault('registries', {})
placeholder = '{{REGISTRY_URI}}'
if placeholder in registries:
for registry, secret in zip_longest(self.spec.registry_uris.value,
self.spec.registry_secrets.value):
if not registry.uri:
continue
regdict = registries[placeholder].copy()
regdict['version'] = registry.version
if secret:
regdict['secret'] = os.path.join(SECRETS_PATH, secret)
registries[registry.docker_uri] = regdict
del registries[placeholder]
def render_add_yum_repo_by_url(self):
if (self.spec.yum_repourls.value is not None and
self.dj.dock_json_has_plugin_conf('prebuild_plugins',
"add_yum_repo_by_url")):
self.dj.dock_json_set_arg('prebuild_plugins',
"add_yum_repo_by_url", "repourls",
self.spec.yum_repourls.value)
if self.spec.proxy.value:
self.dj.dock_json_set_arg('prebuild_plugins',
"add_yum_repo_by_url", "inject_proxy",
self.spec.proxy.value)
def render_check_and_set_rebuild(self, use_auth=None):
if self.dj.dock_json_has_plugin_conf('prebuild_plugins',
'check_and_set_rebuild'):
self.dj.dock_json_set_arg('prebuild_plugins',
'check_and_set_rebuild', 'url',
self.spec.builder_openshift_url.value)
if use_auth is not None:
self.dj.dock_json_set_arg('prebuild_plugins',
'check_and_set_rebuild',
'use_auth', use_auth)
def render_store_metadata_in_osv3(self, use_auth=None):
self.dj.dock_json_set_arg('exit_plugins', "store_metadata_in_osv3",
"url",
self.spec.builder_openshift_url.value)
if use_auth is not None:
self.dj.dock_json_set_arg('exit_plugins',
"store_metadata_in_osv3",
"use_auth", use_auth)
def set_secret_for_plugin(self, plugin, secret):
has_plugin_conf = self.dj.dock_json_has_plugin_conf(plugin[0],
plugin[1])
if 'secrets' in self.template['spec']['strategy']['customStrategy']:
if has_plugin_conf:
# origin 1.0.6 and newer
secret_path = os.path.join(SECRETS_PATH, secret)
logger.info("Configuring %s secret at %s", secret, secret_path)
custom = self.template['spec']['strategy']['customStrategy']
existing = [secret_mount for secret_mount in custom['secrets']
if secret_mount['secretSource']['name'] == secret]
if existing:
logger.debug("secret %s already set", plugin[1])
else:
custom['secrets'].append({
'secretSource': {
'name': secret,
},
'mountPath': secret_path,
})
# there's no need to set args if no plugin secret specified
# this is used in tag_and_push plugin, as it sets secret path
# for each registry separately
if plugin[2] is not None:
self.dj.dock_json_set_arg(*(plugin + (secret_path,)))
else:
logger.debug("not setting secret for unused plugin %s",
plugin[1])
elif plugin[1] in ('pulp_push', 'pulp_sync'):
# setting pulp_push/pulp_sync secret for origin 1.0.5 and earlier
# we only use this way to preserve backwards compat for pulp_push plugin,
# other plugins must use the new secrets way above
logger.info("Configuring %s secret as sourceSecret", secret)
if 'sourceSecret' not in self.template['spec']['source']:
raise OsbsValidationException("JSON template does not allow secrets")
old_secret = self.template['spec']['source']['sourceSecret'].get('name')
if old_secret and old_secret != secret and not old_secret.startswith("{{"):
raise OsbsValidationException("Not possible to set two different source secrets")
self.template['spec']['source']['sourceSecret']['name'] = secret
elif has_plugin_conf:
raise OsbsValidationException("cannot set more than one secret "
"unless using OpenShift >= 1.0.6")
def set_secrets(self, secrets):
"""
:param secrets: dict, {(plugin type, plugin name, argument name): secret name}
for example {('exit_plugins', 'sendmail', 'pdc_secret_path'): 'pdc_secret', ...}
"""
secret_set = False
for (plugin, secret) in secrets.items():
if not isinstance(plugin, tuple) or len(plugin) != 3:
raise ValueError('got "%s" as secrets key, need 3-tuple' % plugin)
if secret is not None:
if isinstance(secret, list):
for secret_item in secret:
self.set_secret_for_plugin(plugin, secret_item)
else:
self.set_secret_for_plugin(plugin, secret)
secret_set = True
if not secret_set:
# remove references to secret if no secret was set
if 'sourceSecret' in self.template['spec']['source']:
del self.template['spec']['source']['sourceSecret']
if 'secrets' in self.template['spec']['strategy']['customStrategy']:
del self.template['spec']['strategy']['customStrategy']['secrets']
@staticmethod
def remove_tag_and_push_registries(tag_and_push_registries, version):
"""
Remove matching entries from tag_and_push_registries (in-place)
:param tag_and_push_registries: dict, uri -> dict
:param version: str, 'version' to match against
"""
registries = [uri
for uri, regdict in tag_and_push_registries.items()
if regdict['version'] == version]
for registry in registries:
logger.info("removing %s registry: %s", version, registry)
del tag_and_push_registries[registry]
def is_custom_base_image(self):
"""
Returns whether or not this is a build from a custom base image
"""
return bool(re.match('^koji/image-build(:.*)?$',
self.base_image or ''))
def adjust_for_registry_api_versions(self):
"""
Enable/disable plugins depending on supported registry API versions
"""
versions = self.spec.registry_api_versions.value
try:
push_conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',
'tag_and_push')
tag_and_push_registries = push_conf['args']['registries']
except (KeyError, IndexError):
tag_and_push_registries = {}
if 'v1' not in versions:
# Remove v1-only plugins
for phase, name in [('postbuild_plugins', 'pulp_push')]:
logger.info("removing v1-only plugin: %s", name)
self.dj.remove_plugin(phase, name)
# remove extra tag_and_push config
self.remove_tag_and_push_registries(tag_and_push_registries, 'v1')
if 'v2' not in versions:
# Remove v2-only plugins
logger.info("removing v2-only plugin: pulp_sync")
self.dj.remove_plugin('postbuild_plugins', 'pulp_sync')
# remove extra tag_and_push config
self.remove_tag_and_push_registries(tag_and_push_registries, 'v2')
# Remove 'version' from tag_and_push plugin config as it's no
# longer needed
for regdict in tag_and_push_registries.values():
if 'version' in regdict:
del regdict['version']
def adjust_for_triggers(self):
"""
Remove trigger-related plugins if no triggers set
"""
triggers = self.template['spec'].get('triggers', [])
if not triggers:
for when, which in [("prebuild_plugins", "check_and_set_rebuild"),
("prebuild_plugins", "stop_autorebuild_if_disabled"),
("postbuild_plugins", "import_image"),
("exit_plugins", "sendmail")]:
logger.info("removing %s from request because there are no triggers",
which)
self.dj.remove_plugin(when, which)
def adjust_for_scratch(self):
"""
Remove koji Content Generator related plugins if no triggers set in
order to hadle the "scratch build" scenario
"""
if self.scratch:
# Note: only one for now, but left in a list like other adjust_for_
# functions in the event that this needs to be expanded
for when, which in [
("exit_plugins", "koji_promote"),
]:
logger.info("removing %s from request because there are no triggers",
which)
self.dj.remove_plugin(when, which)
def adjust_for_custom_base_image(self):
"""
Disable plugins to handle builds depending on whether
or not this is a build from a custom base image.
"""
plugins = []
if self.is_custom_base_image():
# Plugins irrelevant to building base images.
plugins.append(("prebuild_plugins", "pull_base_image"))
msg = "removing %s from custom image build request"
else:
# Plugins not needed for building non base images.
plugins.append(("prebuild_plugins", "add_filesystem"))
msg = "removing %s from non custom image build request"
for when, which in plugins:
logger.info(msg, which)
self.dj.remove_plugin(when, which)
def render_add_filesystem(self):
phase = 'prebuild_plugins'
plugin = 'add_filesystem'
if self.dj.dock_json_has_plugin_conf(phase, plugin):
if not self.spec.kojihub.value:
raise OsbsValidationException(
'Custom base image builds require kojihub to be defined')
self.dj.dock_json_set_arg(phase, plugin, 'koji_hub',
self.spec.kojihub.value)
if self.spec.proxy.value:
self.dj.dock_json_set_arg(phase, plugin, 'koji_proxyuser',
self.spec.proxy.value)
def render_add_labels_in_dockerfile(self):
phase = 'prebuild_plugins'
plugin = 'add_labels_in_dockerfile'
implicit_labels = {}
label_spec = {
'Vendor': self.spec.vendor,
'Authoritative_Registry': self.spec.authoritative_registry,
'distribution-scope': self.spec.distribution_scope,
'Build_Host': self.spec.build_host,
'Architecture': self.spec.architecture,
}
for label, spec in label_spec.items():
if spec.value is not None:
implicit_labels[label] = spec.value
self.dj.dock_json_merge_arg(phase, plugin, 'labels', implicit_labels)
explicit_labels = self.spec.labels.value
if explicit_labels:
logger.debug('Adding requested labels: %r', explicit_labels)
self.dj.dock_json_merge_arg(phase, plugin, "labels",
explicit_labels)
def render_koji(self):
"""
if there is yum repo specified, don't pick stuff from koji
"""
if self.spec.yum_repourls.value:
logger.info("removing koji from request "
"because there is yum repo specified")
self.dj.remove_plugin("prebuild_plugins", "koji")
elif not (self.spec.koji_target.value and
self.spec.kojiroot.value and
self.spec.kojihub.value):
logger.info("removing koji from request as not specified")
self.dj.remove_plugin("prebuild_plugins", "koji")
else:
self.dj.dock_json_set_arg('prebuild_plugins', "koji",
"target", self.spec.koji_target.value)
self.dj.dock_json_set_arg('prebuild_plugins', "koji",
"root", self.spec.kojiroot.value)
self.dj.dock_json_set_arg('prebuild_plugins', "koji",
"hub", self.spec.kojihub.value)
if self.spec.proxy.value:
self.dj.dock_json_set_arg('prebuild_plugins', "koji",
"proxy", self.spec.proxy.value)
def render_bump_release(self):
"""
If the bump_release plugin is present, configure it
"""
phase = 'prebuild_plugins'
plugin = 'bump_release'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
target = self.spec.koji_target.value
hub = self.spec.kojihub.value
if not (target and hub):
logger.info('removing %s from request as koji info not specified',
plugin)
self.dj.remove_plugin(phase, plugin)
return
self.dj.dock_json_set_arg(phase, plugin, 'target', target)
self.dj.dock_json_set_arg(phase, plugin, 'hub', hub)
def render_koji_promote(self, use_auth=None):
if not self.dj.dock_json_has_plugin_conf('exit_plugins',
'koji_promote'):
return
if self.spec.kojihub.value:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote', 'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'kojihub', self.spec.kojihub.value)
koji_target = self.spec.koji_target.value
if koji_target is not None:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'target', koji_target)
if use_auth is not None:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'use_auth', use_auth)
else:
logger.info("removing koji_promote from request as no kojihub "
"specified")
self.dj.remove_plugin("exit_plugins", "koji_promote")
def render_sendmail(self):
"""
if we have pdc_url and smtp_uri, configure sendmail plugin,
else remove it
"""
if not self.dj.dock_json_has_plugin_conf('exit_plugins', 'sendmail'):
return
if self.spec.pdc_url.value and self.spec.smtp_uri.value:
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'pdc_url',
self.spec.pdc_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'smtp_uri',
self.spec.smtp_uri.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'submitter',
self.spec.user.value)
# make sure we'll be able to authenticate to PDC
if 'pdc_secret_path' not in \
self.dj.dock_json_get_plugin_conf('exit_plugins',
'sendmail')['args']:
raise OsbsValidationException('sendmail plugin configured, '
'but no pdc_secret_path')
else:
logger.info("removing sendmail from request, "
"requires pdc_url and smtp_uri")
self.dj.remove_plugin('exit_plugins', 'sendmail')
def render_pulp_push(self):
"""
If a pulp registry is specified, use the pulp plugin
"""
if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'pulp_push'):
return
pulp_registry = self.spec.pulp_registry.value
if pulp_registry:
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_push',
'pulp_registry_name', pulp_registry)
# Verify we have either a secret or username/password
if self.spec.pulp_secret.value is None:
conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',
'pulp_push')
args = conf.get('args', {})
if 'username' not in args:
raise OsbsValidationException("Pulp registry specified "
"but no auth config")
else:
# If no pulp registry is specified, don't run the pulp plugin
logger.info("removing pulp_push from request, "
"requires pulp_registry")
self.dj.remove_plugin("postbuild_plugins", "pulp_push")
def render_pulp_sync(self):
"""
If a pulp registry is specified, use the pulp plugin
"""
if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'pulp_sync'):
return
pulp_registry = self.spec.pulp_registry.value
# Find which registry to use
docker_registry = None
registry_secret = None
registries = zip_longest(self.spec.registry_uris.value,
self.spec.registry_secrets.value)
for registry, secret in registries:
if registry.version == 'v2':
# First specified v2 registry is the one we'll tell pulp
# to sync from. Keep the http prefix -- pulp wants it.
docker_registry = registry.uri
registry_secret = secret
logger.info("using docker v2 registry %s for pulp_sync",
docker_registry)
break
if pulp_registry and docker_registry:
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',
'pulp_registry_name', pulp_registry)
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',
'docker_registry', docker_registry)
if registry_secret:
self.set_secret_for_plugin(('postbuild_plugins',
'pulp_sync',
'registry_secret_path'),
registry_secret)
# Verify we have a pulp secret
if self.spec.pulp_secret.value is None:
raise OsbsValidationException("Pulp registry specified "
"but no auth config")
else:
# If no pulp registry is specified, don't run the pulp plugin
logger.info("removing pulp_sync from request, "
"requires pulp_registry and a v2 registry")
self.dj.remove_plugin("postbuild_plugins", "pulp_sync")
def render_import_image(self, use_auth=None):
"""
Configure the import_image plugin
"""
if self.spec.imagestream_name is None or self.spec.imagestream_url is None:
logger.info("removing import_image from request, "
"registry or repo url is not defined")
self.dj.remove_plugin('postbuild_plugins', 'import_image')
return
if self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'import_image'):
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'imagestream',
self.spec.imagestream_name.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'docker_image_repo',
self.spec.imagestream_url.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'build_json_dir',
self.spec.builder_build_json_dir.value)
use_auth = self.spec.use_auth.value
if use_auth is not None:
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'use_auth', use_auth)
if self.spec.imagestream_insecure_registry.value:
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'insecure_registry', True)
def render(self, validate=True):
if validate:
self.spec.validate()
# !IMPORTANT! can't be too long: https://github.com/openshift/origin/issues/733
self.template['metadata']['name'] = self.spec.name.value
self.render_resource_limits()
self.template['spec']['source']['git']['uri'] = self.spec.git_uri.value
self.template['spec']['source']['git']['ref'] = self.spec.git_ref.value
if self.spec.registry_uris.value:
primary_registry_uri = self.spec.registry_uris.value[0].docker_uri
tag_with_registry = '{0}/{1}'.format(primary_registry_uri,
self.spec.image_tag.value)
self.template['spec']['output']['to']['name'] = tag_with_registry
else:
self.template['spec']['output']['to']['name'] = self.spec.image_tag.value
self.render_tag_and_push_registries()
if 'triggers' in self.template['spec']:
imagechange = self.template['spec']['triggers'][0]['imageChange']
imagechange['from']['name'] = self.spec.trigger_imagestreamtag.value
self.render_add_yum_repo_by_url()
use_auth = self.spec.use_auth.value
self.render_check_and_set_rebuild(use_auth=use_auth)
self.render_store_metadata_in_osv3(use_auth=use_auth)
# For Origin 1.0.6 we'll use the 'secrets' array; for earlier
# versions we'll just use 'sourceSecret'
if self._openshift_required_version < parse_version('1.0.6'):
if 'secrets' in self.template['spec']['strategy']['customStrategy']:
del self.template['spec']['strategy']['customStrategy']['secrets']
else:
if 'sourceSecret' in self.template['spec']['source']:
del self.template['spec']['source']['sourceSecret']
if self.spec.build_imagestream.value:
self.template['spec']['strategy']['customStrategy']['from']['kind'] = 'ImageStreamTag'
self.template['spec']['strategy']['customStrategy']['from']['name'] = \
self.spec.build_imagestream.value
else:
self.template['spec']['strategy']['customStrategy']['from']['name'] = \
self.spec.build_image.value
repo_name = git_repo_humanish_part_from_uri(self.spec.git_uri.value)
# NOTE: Since only the repo name is used, a forked repos will have
# the same git-repo-name tag. This is a known limitation. If this
# use case must be handled properly, the git URI must be taken into
# account.
self.set_label('git-repo-name', repo_name)
self.set_label('git-branch', self.spec.git_branch.value)
if self.spec.sources_command.value is not None:
self.dj.dock_json_set_arg('prebuild_plugins', "distgit_fetch_artefacts",
"command", self.spec.sources_command.value)
else:
logger.info("removing distgit_fetch_artefacts, no sources_command was provided")
self.dj.remove_plugin('prebuild_plugins', 'distgit_fetch_artefacts')
# pull_base_image wants a docker URI so strip off the scheme part
source_registry = self.spec.source_registry_uri.value
self.dj.dock_json_set_arg('prebuild_plugins', "pull_base_image", "parent_registry",
source_registry.docker_uri if source_registry else None)
# The rebuild trigger requires git_branch and git_push_url
# parameters, but those parameters are optional. If either was
# not provided, remove the trigger.
remove_triggers = False
for param_name in ['git_branch', 'git_push_url']:
param = getattr(self.spec, param_name)
if not param.value:
logger.info("removing triggers as no %s specified", param_name)
remove_triggers = True
# Continue the loop so we log everything that's missing
if self.is_custom_base_image():
logger.info('removing triggers for custom base image build')
remove_triggers = True
if remove_triggers and 'triggers' in self.template['spec']:
del self.template['spec']['triggers']
self.adjust_for_triggers()
self.adjust_for_scratch()
self.adjust_for_custom_base_image()
# Enable/disable plugins as needed for target registry API versions
self.adjust_for_registry_api_versions()
self.set_secrets({('postbuild_plugins',
'pulp_push',
'pulp_secret_path'):
self.spec.pulp_secret.value,
('postbuild_plugins',
'pulp_sync',
'pulp_secret_path'):
self.spec.pulp_secret.value,
# pulp_sync registry_secret_path set
# in render_pulp_sync
('exit_plugins', 'sendmail', 'pdc_secret_path'):
self.spec.pdc_secret.value,
('exit_plugins', 'koji_promote', 'koji_ssl_certs'):
self.spec.koji_certs_secret.value,
('prebuild_plugins', 'add_filesystem', 'koji_ssl_certs_dir'):
self.spec.koji_certs_secret.value,
('postbuild_plugins', 'tag_and_push',
# Only set the secrets for the build, don't
# add the path to the plugin's
# configuration. This is done elsewhere.
None):
self.spec.registry_secrets.value})
if self.spec.pulp_secret.value:
# Don't push to docker registry, we're using pulp here
# but still construct the unique tag
self.template['spec']['output']['to']['name'] = \
self.spec.image_tag.value
koji_task_id = self.spec.koji_task_id.value
if koji_task_id is not None:
self.template['metadata'].setdefault('labels', {})
self.template['metadata']['labels']['koji-task-id'] = str(koji_task_id)
use_auth = self.spec.use_auth.value
self.render_add_filesystem()
self.render_add_labels_in_dockerfile()
self.render_koji()
self.render_bump_release()
self.render_import_image(use_auth=use_auth)
self.render_pulp_push()
self.render_pulp_sync()
self.render_koji_promote(use_auth=use_auth)
self.render_sendmail()
self.dj.write_dock_json()
self.build_json = self.template
logger.debug(self.build_json)
return self.build_json
class BuildRequest(object):
"""
Wraps logic for creating build inputs
"""
def __init__(self, build_json_store):
"""
:param build_json_store: str, path to directory with JSON build files
"""
self.spec = BuildSpec()
self.build_json_store = build_json_store
self.build_json = None # rendered template
self._template = None # template loaded from filesystem
self._inner_template = None # dock json
self._dj = None
self._resource_limits = None
self._openshift_required_version = parse_version('1.0.6')
# For the koji "scratch" build type
self.scratch = False
self.base_image = None
def set_params(self, **kwargs):
"""
set parameters according to specification
these parameters are accepted:
:param pulp_secret: str, resource name of pulp secret
:param pdc_secret: str, resource name of pdc secret
:param koji_target: str, koji tag with packages used to build the image
:param kojiroot: str, URL from which koji packages are fetched
:param kojihub: str, URL of the koji hub
:param koji_certs_secret: str, resource name of secret that holds the koji certificates
:param koji_task_id: int, Koji Task that created this build config
:param pulp_registry: str, name of pulp registry in dockpulp.conf
:param nfs_server_path: str, NFS server and path
:param nfs_dest_dir: str, directory to create on NFS server
:param sources_command: str, command used to fetch dist-git sources
:param architecture: str, architecture we are building for
:param vendor: str, vendor name
:param build_host: str, host the build will run on or None for auto
:param authoritative_registry: str, the docker registry authoritative for this image
:param distribution_scope: str, distribution scope for this image
(private, authoritative-source-only, restricted, public)
:param use_auth: bool, use auth from atomic-reactor?
:param git_push_url: str, URL for git push
"""
# Here we cater to the koji "scratch" build type, this will disable
# all plugins that might cause importing of data to koji
try:
self.scratch = kwargs.pop("scratch")
except KeyError:
pass
self.base_image = kwargs.get('base_image')
logger.debug("setting params '%s' for %s", kwargs, self.spec)
self.spec.set_params(**kwargs)
def set_resource_limits(self, cpu=None, memory=None, storage=None):
if self._resource_limits is None:
self._resource_limits = {}
if cpu is not None:
self._resource_limits['cpu'] = cpu
if memory is not None:
self._resource_limits['memory'] = memory
if storage is not None:
self._resource_limits['storage'] = storage
def set_openshift_required_version(self, openshift_required_version):
if openshift_required_version is not None:
self._openshift_required_version = openshift_required_version
@property
def build_id(self):
return self.build_json['metadata']['name']
@property
def template(self):
if self._template is None:
path = os.path.join(self.build_json_store, DEFAULT_OUTER_TEMPLATE)
logger.debug("loading template from path %s", path)
try:
with open(path, "r") as fp:
self._template = json.load(fp)
except (IOError, OSError) as ex:
raise OsbsException("Can't open template '%s': %s" %
(path, repr(ex)))
return self._template
@property
def inner_template(self):
if self._inner_template is None:
path = os.path.join(self.build_json_store, DEFAULT_INNER_TEMPLATE)
logger.debug("loading inner template from path %s", path)
with open(path, "r") as fp:
self._inner_template = json.load(fp)
return self._inner_template
@property
def dj(self):
if self._dj is None:
self._dj = DockJsonManipulator(self.template, self.inner_template)
return self._dj
def is_auto_instantiated(self):
"""Return True if this BuildConfig will be automatically instantiated when created."""
triggers = self.template['spec'].get('triggers', [])
for trigger in triggers:
if trigger['type'] == 'ImageChange' and \
trigger['imageChange']['from']['kind'] == 'ImageStreamTag':
return True
return False
def set_label(self, name, value):
self.template['metadata'].setdefault('labels', {})
self.template['metadata']['labels'][name] = value
def render_resource_limits(self):
if self._resource_limits is not None:
resources = self.template['spec'].get('resources', {})
limits = resources.get('limits', {})
limits.update(self._resource_limits)
resources['limits'] = limits
self.template['spec']['resources'] = resources
def render_tag_and_push_registries(self):
if self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'tag_and_push'):
push_conf = self.dj.dock_json_get_plugin_conf(
'postbuild_plugins', 'tag_and_push')
args = push_conf.setdefault('args', {})
registries = args.setdefault('registries', {})
placeholder = '{{REGISTRY_URI}}'
if placeholder in registries:
for registry, secret in zip_longest(
self.spec.registry_uris.value,
self.spec.registry_secrets.value):
if not registry.uri:
continue
regdict = registries[placeholder].copy()
regdict['version'] = registry.version
if secret:
regdict['secret'] = os.path.join(SECRETS_PATH, secret)
registries[registry.docker_uri] = regdict
del registries[placeholder]
def render_add_yum_repo_by_url(self):
if (self.spec.yum_repourls.value is not None
and self.dj.dock_json_has_plugin_conf('prebuild_plugins',
"add_yum_repo_by_url")):
self.dj.dock_json_set_arg('prebuild_plugins',
"add_yum_repo_by_url", "repourls",
self.spec.yum_repourls.value)
if self.spec.proxy.value:
self.dj.dock_json_set_arg('prebuild_plugins',
"add_yum_repo_by_url",
"inject_proxy",
self.spec.proxy.value)
def render_check_and_set_rebuild(self, use_auth=None):
if self.dj.dock_json_has_plugin_conf('prebuild_plugins',
'check_and_set_rebuild'):
self.dj.dock_json_set_arg('prebuild_plugins',
'check_and_set_rebuild', 'url',
self.spec.builder_openshift_url.value)
if use_auth is not None:
self.dj.dock_json_set_arg('prebuild_plugins',
'check_and_set_rebuild', 'use_auth',
use_auth)
def render_store_metadata_in_osv3(self, use_auth=None):
self.dj.dock_json_set_arg('exit_plugins', "store_metadata_in_osv3",
"url", self.spec.builder_openshift_url.value)
if use_auth is not None:
self.dj.dock_json_set_arg('exit_plugins', "store_metadata_in_osv3",
"use_auth", use_auth)
def set_secret_for_plugin(self, plugin, secret):
has_plugin_conf = self.dj.dock_json_has_plugin_conf(
plugin[0], plugin[1])
if 'secrets' in self.template['spec']['strategy']['customStrategy']:
if has_plugin_conf:
# origin 1.0.6 and newer
secret_path = os.path.join(SECRETS_PATH, secret)
logger.info("Configuring %s secret at %s", secret, secret_path)
custom = self.template['spec']['strategy']['customStrategy']
existing = [
secret_mount for secret_mount in custom['secrets']
if secret_mount['secretSource']['name'] == secret
]
if existing:
logger.debug("secret %s already set", plugin[1])
else:
custom['secrets'].append({
'secretSource': {
'name': secret,
},
'mountPath': secret_path,
})
# there's no need to set args if no plugin secret specified
# this is used in tag_and_push plugin, as it sets secret path
# for each registry separately
if plugin[2] is not None:
self.dj.dock_json_set_arg(*(plugin + (secret_path, )))
else:
logger.debug("not setting secret for unused plugin %s",
plugin[1])
elif plugin[1] in ('pulp_push', 'pulp_sync'):
# setting pulp_push/pulp_sync secret for origin 1.0.5 and earlier
# we only use this way to preserve backwards compat for pulp_push plugin,
# other plugins must use the new secrets way above
logger.info("Configuring %s secret as sourceSecret", secret)
if 'sourceSecret' not in self.template['spec']['source']:
raise OsbsValidationException(
"JSON template does not allow secrets")
old_secret = self.template['spec']['source']['sourceSecret'].get(
'name')
if old_secret and old_secret != secret and not old_secret.startswith(
"{{"):
raise OsbsValidationException(
"Not possible to set two different source secrets")
self.template['spec']['source']['sourceSecret']['name'] = secret
elif has_plugin_conf:
raise OsbsValidationException("cannot set more than one secret "
"unless using OpenShift >= 1.0.6")
def set_secrets(self, secrets):
"""
:param secrets: dict, {(plugin type, plugin name, argument name): secret name}
for example {('exit_plugins', 'sendmail', 'pdc_secret_path'): 'pdc_secret', ...}
"""
secret_set = False
for (plugin, secret) in secrets.items():
if not isinstance(plugin, tuple) or len(plugin) != 3:
raise ValueError('got "%s" as secrets key, need 3-tuple' %
plugin)
if secret is not None:
if isinstance(secret, list):
for secret_item in secret:
self.set_secret_for_plugin(plugin, secret_item)
else:
self.set_secret_for_plugin(plugin, secret)
secret_set = True
if not secret_set:
# remove references to secret if no secret was set
if 'sourceSecret' in self.template['spec']['source']:
del self.template['spec']['source']['sourceSecret']
if 'secrets' in self.template['spec']['strategy'][
'customStrategy']:
del self.template['spec']['strategy']['customStrategy'][
'secrets']
@staticmethod
def remove_tag_and_push_registries(tag_and_push_registries, version):
"""
Remove matching entries from tag_and_push_registries (in-place)
:param tag_and_push_registries: dict, uri -> dict
:param version: str, 'version' to match against
"""
registries = [
uri for uri, regdict in tag_and_push_registries.items()
if regdict['version'] == version
]
for registry in registries:
logger.info("removing %s registry: %s", version, registry)
del tag_and_push_registries[registry]
def is_custom_base_image(self):
"""
Returns whether or not this is a build from a custom base image
"""
return bool(re.match('^koji/image-build(:.*)?$', self.base_image
or ''))
def adjust_for_registry_api_versions(self):
"""
Enable/disable plugins depending on supported registry API versions
"""
versions = self.spec.registry_api_versions.value
try:
push_conf = self.dj.dock_json_get_plugin_conf(
'postbuild_plugins', 'tag_and_push')
tag_and_push_registries = push_conf['args']['registries']
except (KeyError, IndexError):
tag_and_push_registries = {}
if 'v1' not in versions:
# Remove v1-only plugins
for phase, name in [('postbuild_plugins', 'pulp_push')]:
logger.info("removing v1-only plugin: %s", name)
self.dj.remove_plugin(phase, name)
# remove extra tag_and_push config
self.remove_tag_and_push_registries(tag_and_push_registries, 'v1')
if 'v2' not in versions:
# Remove v2-only plugins
logger.info("removing v2-only plugin: pulp_sync")
self.dj.remove_plugin('postbuild_plugins', 'pulp_sync')
# remove extra tag_and_push config
self.remove_tag_and_push_registries(tag_and_push_registries, 'v2')
# Remove 'version' from tag_and_push plugin config as it's no
# longer needed
for regdict in tag_and_push_registries.values():
if 'version' in regdict:
del regdict['version']
def adjust_for_triggers(self):
"""
Remove trigger-related plugins if no triggers set
"""
triggers = self.template['spec'].get('triggers', [])
if not triggers:
for when, which in [("prebuild_plugins", "check_and_set_rebuild"),
("prebuild_plugins",
"stop_autorebuild_if_disabled"),
("postbuild_plugins", "import_image"),
("exit_plugins", "sendmail")]:
logger.info(
"removing %s from request because there are no triggers",
which)
self.dj.remove_plugin(when, which)
def adjust_for_scratch(self):
"""
Remove koji Content Generator related plugins if no triggers set in
order to hadle the "scratch build" scenario
"""
if self.scratch:
# Note: only one for now, but left in a list like other adjust_for_
# functions in the event that this needs to be expanded
for when, which in [
("exit_plugins", "koji_promote"),
]:
logger.info(
"removing %s from request because there are no triggers",
which)
self.dj.remove_plugin(when, which)
def adjust_for_custom_base_image(self):
"""
Disable plugins to handle builds depending on whether
or not this is a build from a custom base image.
"""
plugins = []
if self.is_custom_base_image():
# Plugins irrelevant to building base images.
plugins.append(("prebuild_plugins", "pull_base_image"))
msg = "removing %s from custom image build request"
else:
# Plugins not needed for building non base images.
plugins.append(("prebuild_plugins", "add_filesystem"))
msg = "removing %s from non custom image build request"
for when, which in plugins:
logger.info(msg, which)
self.dj.remove_plugin(when, which)
def render_add_filesystem(self):
phase = 'prebuild_plugins'
plugin = 'add_filesystem'
if self.dj.dock_json_has_plugin_conf(phase, plugin):
if not self.spec.kojihub.value:
raise OsbsValidationException(
'Custom base image builds require kojihub to be defined')
self.dj.dock_json_set_arg(phase, plugin, 'koji_hub',
self.spec.kojihub.value)
if self.spec.proxy.value:
self.dj.dock_json_set_arg(phase, plugin, 'koji_proxyuser',
self.spec.proxy.value)
def render_add_labels_in_dockerfile(self):
phase = 'prebuild_plugins'
plugin = 'add_labels_in_dockerfile'
implicit_labels = {}
label_spec = {
'Vendor': self.spec.vendor,
'Authoritative_Registry': self.spec.authoritative_registry,
'distribution-scope': self.spec.distribution_scope,
'Build_Host': self.spec.build_host,
'Architecture': self.spec.architecture,
}
for label, spec in label_spec.items():
if spec.value is not None:
implicit_labels[label] = spec.value
self.dj.dock_json_merge_arg(phase, plugin, 'labels', implicit_labels)
explicit_labels = self.spec.labels.value
if explicit_labels:
logger.debug('Adding requested labels: %r', explicit_labels)
self.dj.dock_json_merge_arg(phase, plugin, "labels",
explicit_labels)
def render_koji(self):
"""
if there is yum repo specified, don't pick stuff from koji
"""
if self.spec.yum_repourls.value:
logger.info("removing koji from request "
"because there is yum repo specified")
self.dj.remove_plugin("prebuild_plugins", "koji")
elif not (self.spec.koji_target.value and self.spec.kojiroot.value
and self.spec.kojihub.value):
logger.info("removing koji from request as not specified")
self.dj.remove_plugin("prebuild_plugins", "koji")
else:
self.dj.dock_json_set_arg('prebuild_plugins', "koji", "target",
self.spec.koji_target.value)
self.dj.dock_json_set_arg('prebuild_plugins', "koji", "root",
self.spec.kojiroot.value)
self.dj.dock_json_set_arg('prebuild_plugins', "koji", "hub",
self.spec.kojihub.value)
if self.spec.proxy.value:
self.dj.dock_json_set_arg('prebuild_plugins', "koji", "proxy",
self.spec.proxy.value)
def render_bump_release(self):
"""
If the bump_release plugin is present, configure it
"""
phase = 'prebuild_plugins'
plugin = 'bump_release'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
target = self.spec.koji_target.value
hub = self.spec.kojihub.value
if not (target and hub):
logger.info('removing %s from request as koji info not specified',
plugin)
self.dj.remove_plugin(phase, plugin)
return
self.dj.dock_json_set_arg(phase, plugin, 'target', target)
self.dj.dock_json_set_arg(phase, plugin, 'hub', hub)
def render_koji_promote(self, use_auth=None):
if not self.dj.dock_json_has_plugin_conf('exit_plugins',
'koji_promote'):
return
if self.spec.kojihub.value:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote', 'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'kojihub', self.spec.kojihub.value)
koji_target = self.spec.koji_target.value
if koji_target is not None:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'target', koji_target)
if use_auth is not None:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'use_auth', use_auth)
else:
logger.info("removing koji_promote from request as no kojihub "
"specified")
self.dj.remove_plugin("exit_plugins", "koji_promote")
def render_sendmail(self):
"""
if we have pdc_url and smtp_uri, configure sendmail plugin,
else remove it
"""
if not self.dj.dock_json_has_plugin_conf('exit_plugins', 'sendmail'):
return
if self.spec.pdc_url.value and self.spec.smtp_uri.value:
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'pdc_url',
self.spec.pdc_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'smtp_uri',
self.spec.smtp_uri.value)
self.dj.dock_json_set_arg('exit_plugins', 'sendmail', 'submitter',
self.spec.user.value)
# make sure we'll be able to authenticate to PDC
if 'pdc_secret_path' not in \
self.dj.dock_json_get_plugin_conf('exit_plugins',
'sendmail')['args']:
raise OsbsValidationException('sendmail plugin configured, '
'but no pdc_secret_path')
else:
logger.info("removing sendmail from request, "
"requires pdc_url and smtp_uri")
self.dj.remove_plugin('exit_plugins', 'sendmail')
def render_pulp_push(self):
"""
If a pulp registry is specified, use the pulp plugin
"""
if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'pulp_push'):
return
pulp_registry = self.spec.pulp_registry.value
if pulp_registry:
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_push',
'pulp_registry_name', pulp_registry)
# Verify we have either a secret or username/password
if self.spec.pulp_secret.value is None:
conf = self.dj.dock_json_get_plugin_conf(
'postbuild_plugins', 'pulp_push')
args = conf.get('args', {})
if 'username' not in args:
raise OsbsValidationException("Pulp registry specified "
"but no auth config")
else:
# If no pulp registry is specified, don't run the pulp plugin
logger.info("removing pulp_push from request, "
"requires pulp_registry")
self.dj.remove_plugin("postbuild_plugins", "pulp_push")
def render_pulp_sync(self):
"""
If a pulp registry is specified, use the pulp plugin
"""
if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'pulp_sync'):
return
pulp_registry = self.spec.pulp_registry.value
# Find which registry to use
docker_registry = None
registry_secret = None
registries = zip_longest(self.spec.registry_uris.value,
self.spec.registry_secrets.value)
for registry, secret in registries:
if registry.version == 'v2':
# First specified v2 registry is the one we'll tell pulp
# to sync from. Keep the http prefix -- pulp wants it.
docker_registry = registry.uri
registry_secret = secret
logger.info("using docker v2 registry %s for pulp_sync",
docker_registry)
break
if pulp_registry and docker_registry:
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',
'pulp_registry_name', pulp_registry)
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',
'docker_registry', docker_registry)
if registry_secret:
self.set_secret_for_plugin(
('postbuild_plugins', 'pulp_sync', 'registry_secret_path'),
registry_secret)
# Verify we have a pulp secret
if self.spec.pulp_secret.value is None:
raise OsbsValidationException("Pulp registry specified "
"but no auth config")
else:
# If no pulp registry is specified, don't run the pulp plugin
logger.info("removing pulp_sync from request, "
"requires pulp_registry and a v2 registry")
self.dj.remove_plugin("postbuild_plugins", "pulp_sync")
def render_import_image(self, use_auth=None):
"""
Configure the import_image plugin
"""
if self.spec.imagestream_name is None or self.spec.imagestream_url is None:
logger.info("removing import_image from request, "
"registry or repo url is not defined")
self.dj.remove_plugin('postbuild_plugins', 'import_image')
return
if self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'import_image'):
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'imagestream',
self.spec.imagestream_name.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'docker_image_repo',
self.spec.imagestream_url.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'build_json_dir',
self.spec.builder_build_json_dir.value)
use_auth = self.spec.use_auth.value
if use_auth is not None:
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'use_auth', use_auth)
if self.spec.imagestream_insecure_registry.value:
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'insecure_registry', True)
def render(self, validate=True):
if validate:
self.spec.validate()
# !IMPORTANT! can't be too long: https://github.com/openshift/origin/issues/733
self.template['metadata']['name'] = self.spec.name.value
self.render_resource_limits()
self.template['spec']['source']['git']['uri'] = self.spec.git_uri.value
self.template['spec']['source']['git']['ref'] = self.spec.git_ref.value
if self.spec.registry_uris.value:
primary_registry_uri = self.spec.registry_uris.value[0].docker_uri
tag_with_registry = '{0}/{1}'.format(primary_registry_uri,
self.spec.image_tag.value)
self.template['spec']['output']['to']['name'] = tag_with_registry
else:
self.template['spec']['output']['to'][
'name'] = self.spec.image_tag.value
self.render_tag_and_push_registries()
if 'triggers' in self.template['spec']:
imagechange = self.template['spec']['triggers'][0]['imageChange']
imagechange['from'][
'name'] = self.spec.trigger_imagestreamtag.value
self.render_add_yum_repo_by_url()
use_auth = self.spec.use_auth.value
self.render_check_and_set_rebuild(use_auth=use_auth)
self.render_store_metadata_in_osv3(use_auth=use_auth)
# For Origin 1.0.6 we'll use the 'secrets' array; for earlier
# versions we'll just use 'sourceSecret'
if self._openshift_required_version < parse_version('1.0.6'):
if 'secrets' in self.template['spec']['strategy'][
'customStrategy']:
del self.template['spec']['strategy']['customStrategy'][
'secrets']
else:
if 'sourceSecret' in self.template['spec']['source']:
del self.template['spec']['source']['sourceSecret']
if self.spec.build_imagestream.value:
self.template['spec']['strategy']['customStrategy']['from'][
'kind'] = 'ImageStreamTag'
self.template['spec']['strategy']['customStrategy']['from']['name'] = \
self.spec.build_imagestream.value
else:
self.template['spec']['strategy']['customStrategy']['from']['name'] = \
self.spec.build_image.value
repo_name = git_repo_humanish_part_from_uri(self.spec.git_uri.value)
# NOTE: Since only the repo name is used, a forked repos will have
# the same git-repo-name tag. This is a known limitation. If this
# use case must be handled properly, the git URI must be taken into
# account.
self.set_label('git-repo-name', repo_name)
self.set_label('git-branch', self.spec.git_branch.value)
if self.spec.sources_command.value is not None:
self.dj.dock_json_set_arg('prebuild_plugins',
"distgit_fetch_artefacts", "command",
self.spec.sources_command.value)
else:
logger.info(
"removing distgit_fetch_artefacts, no sources_command was provided"
)
self.dj.remove_plugin('prebuild_plugins',
'distgit_fetch_artefacts')
# pull_base_image wants a docker URI so strip off the scheme part
source_registry = self.spec.source_registry_uri.value
self.dj.dock_json_set_arg(
'prebuild_plugins', "pull_base_image", "parent_registry",
source_registry.docker_uri if source_registry else None)
# The rebuild trigger requires git_branch and git_push_url
# parameters, but those parameters are optional. If either was
# not provided, remove the trigger.
remove_triggers = False
for param_name in ['git_branch', 'git_push_url']:
param = getattr(self.spec, param_name)
if not param.value:
logger.info("removing triggers as no %s specified", param_name)
remove_triggers = True
# Continue the loop so we log everything that's missing
if self.is_custom_base_image():
logger.info('removing triggers for custom base image build')
remove_triggers = True
if remove_triggers and 'triggers' in self.template['spec']:
del self.template['spec']['triggers']
self.adjust_for_triggers()
self.adjust_for_scratch()
self.adjust_for_custom_base_image()
# Enable/disable plugins as needed for target registry API versions
self.adjust_for_registry_api_versions()
self.set_secrets({
('postbuild_plugins', 'pulp_push', 'pulp_secret_path'):
self.spec.pulp_secret.value,
('postbuild_plugins', 'pulp_sync', 'pulp_secret_path'):
self.spec.pulp_secret.value,
# pulp_sync registry_secret_path set
# in render_pulp_sync
('exit_plugins', 'sendmail', 'pdc_secret_path'):
self.spec.pdc_secret.value,
('exit_plugins', 'koji_promote', 'koji_ssl_certs'):
self.spec.koji_certs_secret.value,
('prebuild_plugins', 'add_filesystem', 'koji_ssl_certs_dir'):
self.spec.koji_certs_secret.value,
(
'postbuild_plugins',
'tag_and_push',
# Only set the secrets for the build, don't
# add the path to the plugin's
# configuration. This is done elsewhere.
None):
self.spec.registry_secrets.value
})
if self.spec.pulp_secret.value:
# Don't push to docker registry, we're using pulp here
# but still construct the unique tag
self.template['spec']['output']['to']['name'] = \
self.spec.image_tag.value
koji_task_id = self.spec.koji_task_id.value
if koji_task_id is not None:
self.template['metadata'].setdefault('labels', {})
self.template['metadata']['labels']['koji-task-id'] = str(
koji_task_id)
use_auth = self.spec.use_auth.value
self.render_add_filesystem()
self.render_add_labels_in_dockerfile()
self.render_koji()
self.render_bump_release()
self.render_import_image(use_auth=use_auth)
self.render_pulp_push()
self.render_pulp_sync()
self.render_koji_promote(use_auth=use_auth)
self.render_sendmail()
self.dj.write_dock_json()
self.build_json = self.template
logger.debug(self.build_json)
return self.build_json
class BuildRequest(object):
"""
Wraps logic for creating build inputs
"""
def __init__(self, build_json_store, inner_template=None,
outer_template=None, customize_conf=None):
"""
:param build_json_store: str, path to directory with JSON build files
:param inner_template: str, path to inner template JSON
:param outer_template: str, path to outer template JSON
:param customize_conf: str, path to customize configuration JSON
"""
self.spec = BuildSpec()
self.build_json_store = build_json_store
self._inner_template_path = inner_template or DEFAULT_INNER_TEMPLATE
self._outer_template_path = outer_template or DEFAULT_OUTER_TEMPLATE
self._customize_conf_path = customize_conf or DEFAULT_CUSTOMIZE_CONF
self.build_json = None # rendered template
self._template = None # template loaded from filesystem
self._inner_template = None # dock json
self._customize_conf = None # site customize conf for _inner_template
self._dj = None
self._resource_limits = None
self._openshift_required_version = parse_version('1.0.6')
# For the koji "scratch" build type
self.scratch = None
self.base_image = None
def set_params(self, **kwargs):
"""
set parameters according to specification
these parameters are accepted:
:param pulp_secret: str, resource name of pulp secret
:param koji_target: str, koji tag with packages used to build the image
:param kojiroot: str, URL from which koji packages are fetched
:param kojihub: str, URL of the koji hub
:param koji_certs_secret: str, resource name of secret that holds the koji certificates
:param koji_task_id: int, Koji Task that created this build config
:param pulp_registry: str, name of pulp registry in dockpulp.conf
:param nfs_server_path: str, NFS server and path
:param nfs_dest_dir: str, directory to create on NFS server
:param sources_command: str, command used to fetch dist-git sources
:param architecture: str, architecture we are building for
:param vendor: str, vendor name
:param build_host: str, host the build will run on or None for auto
:param authoritative_registry: str, the docker registry authoritative for this image
:param distribution_scope: str, distribution scope for this image
(private, authoritative-source-only, restricted, public)
:param use_auth: bool, use auth from atomic-reactor?
"""
# Here we cater to the koji "scratch" build type, this will disable
# all plugins that might cause importing of data to koji
try:
self.scratch = kwargs.pop("scratch")
except KeyError:
pass
self.base_image = kwargs.get('base_image')
logger.debug("setting params '%s' for %s", kwargs, self.spec)
self.spec.set_params(**kwargs)
def set_resource_limits(self, cpu=None, memory=None, storage=None):
if self._resource_limits is None:
self._resource_limits = {}
if cpu is not None:
self._resource_limits['cpu'] = cpu
if memory is not None:
self._resource_limits['memory'] = memory
if storage is not None:
self._resource_limits['storage'] = storage
def set_openshift_required_version(self, openshift_required_version):
if openshift_required_version is not None:
self._openshift_required_version = openshift_required_version
@property
def build_id(self):
return self.build_json['metadata']['name']
@property
def template(self):
if self._template is None:
path = os.path.join(self.build_json_store, self._outer_template_path)
logger.debug("loading template from path %s", path)
try:
with open(path, "r") as fp:
self._template = json.load(fp)
except (IOError, OSError) as ex:
raise OsbsException("Can't open template '%s': %s" %
(path, repr(ex)))
return self._template
@property
def inner_template(self):
if self._inner_template is None:
path = os.path.join(self.build_json_store, self._inner_template_path)
logger.debug("loading inner template from path %s", path)
with open(path, "r") as fp:
self._inner_template = json.load(fp)
return self._inner_template
@property
def customize_conf(self):
if self._customize_conf is None:
path = os.path.join(self.build_json_store, self._customize_conf_path)
logger.debug("loading customize conf from path %s", path)
try:
with open(path, "r") as fp:
self._customize_conf= json.load(fp)
except IOError:
# File not found, which is perfectly fine. Set to empty string
self._customize_conf = {}
return self._customize_conf
@property
def dj(self):
if self._dj is None:
self._dj = DockJsonManipulator(self.template, self.inner_template)
return self._dj
def has_ist_trigger(self):
"""Return True if this BuildConfig has ImageStreamTag trigger."""
triggers = self.template['spec'].get('triggers', [])
for trigger in triggers:
if trigger['type'] == 'ImageChange' and \
trigger['imageChange']['from']['kind'] == 'ImageStreamTag':
return True
return False
def set_label(self, name, value):
if not value:
value = ''
self.template['metadata'].setdefault('labels', {})
self.template['metadata']['labels'][name] = value
def render_reactor_config(self):
if self.spec.reactor_config_secret.value is None:
logger.debug("removing reactor_config plugin: no secret")
self.dj.remove_plugin('prebuild_plugins', 'reactor_config')
def render_orchestrate_build(self):
phase = 'buildstep_plugins'
plugin = 'orchestrate_build'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
if self.spec.platforms.value is None:
logger.debug('removing %s plugin: no platforms', plugin)
self.dj.remove_plugin(phase, plugin)
return
# Parameters to be used in call to create_worker_build
build_kwargs = {
'component': self.spec.component.value,
'git_branch': self.spec.git_branch.value,
'git_ref': self.spec.git_ref.value,
'git_uri': self.spec.git_uri.value,
'koji_task_id': self.spec.koji_task_id.value,
'scratch': self.scratch,
'target': self.spec.koji_target.value,
'user': self.spec.user.value,
'yum_repourls': self.spec.yum_repourls.value,
'arrangement_version': self.spec.arrangement_version.value,
}
self.dj.dock_json_set_arg(phase, plugin, 'platforms', self.spec.platforms.value)
self.dj.dock_json_set_arg(phase, plugin, 'build_kwargs', build_kwargs)
# Parameters to be used as Configuration overrides for each worker
config_kwargs = {
'authoritative_registry': self.spec.authoritative_registry.value,
'distribution_scope': self.spec.distribution_scope.value,
'info_url_format': self.spec.info_url_format.value,
'koji_hub': self.spec.kojihub.value,
'koji_root': self.spec.kojiroot.value,
'openshift_required_version': self._openshift_required_version,
'pulp_registry_name': self.spec.pulp_registry.value,
'registry_api_versions': ','.join(self.spec.registry_api_versions.value or []) or None,
'smtp_additional_addresses': ','.join(self.spec.smtp_additional_addresses.value or []) or None,
'smtp_email_domain': self.spec.smtp_email_domain.value,
'smtp_error_addresses': ','.join(self.spec.smtp_error_addresses.value or []) or None,
'smtp_from': self.spec.smtp_from.value,
'smtp_host': self.spec.smtp_host.value,
'smtp_to_pkgowner': self.spec.smtp_to_pkgowner.value,
'smtp_to_submitter': self.spec.smtp_to_submitter.value,
'source_registry_uri': self.spec.source_registry_uri.value,
'sources_command': self.spec.sources_command.value,
'vendor': self.spec.vendor.value,
}
# Remove empty values, and always convert to string for better interaction
# with Configuration class and JSON encoding
config_kwargs = dict((k, str(v)) for k, v in config_kwargs.items() if v is not None)
if not self.spec.build_imagestream.value:
config_kwargs['build_image'] = self.spec.build_image.value
self.dj.dock_json_set_arg(phase, plugin, 'config_kwargs', config_kwargs)
def render_resource_limits(self):
if self._resource_limits is not None:
resources = self.template['spec'].get('resources', {})
limits = resources.get('limits', {})
limits.update(self._resource_limits)
resources['limits'] = limits
self.template['spec']['resources'] = resources
def render_tag_and_push_registries(self):
if self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'tag_and_push'):
push_conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',
'tag_and_push')
args = push_conf.setdefault('args', {})
registries = args.setdefault('registries', {})
placeholder = '{{REGISTRY_URI}}'
if placeholder in registries:
for registry, secret in zip_longest(self.spec.registry_uris.value,
self.spec.registry_secrets.value):
if not registry.uri:
continue
regdict = registries[placeholder].copy()
regdict['version'] = registry.version
if secret:
regdict['secret'] = os.path.join(SECRETS_PATH, secret)
registries[registry.docker_uri] = regdict
del registries[placeholder]
def render_add_yum_repo_by_url(self):
if (self.spec.yum_repourls.value is not None and
self.dj.dock_json_has_plugin_conf('prebuild_plugins',
"add_yum_repo_by_url")):
self.dj.dock_json_set_arg('prebuild_plugins',
"add_yum_repo_by_url", "repourls",
self.spec.yum_repourls.value)
if self.spec.proxy.value:
self.dj.dock_json_set_arg('prebuild_plugins',
"add_yum_repo_by_url", "inject_proxy",
self.spec.proxy.value)
def render_check_and_set_rebuild(self, use_auth=None):
if self.dj.dock_json_has_plugin_conf('prebuild_plugins',
'check_and_set_rebuild'):
self.dj.dock_json_set_arg('prebuild_plugins',
'check_and_set_rebuild', 'url',
self.spec.builder_openshift_url.value)
if use_auth is not None:
self.dj.dock_json_set_arg('prebuild_plugins',
'check_and_set_rebuild',
'use_auth', use_auth)
def render_store_metadata_in_osv3(self, use_auth=None):
if not self.dj.dock_json_has_plugin_conf('exit_plugins',
'store_metadata_in_osv3'):
return
self.dj.dock_json_set_arg('exit_plugins', "store_metadata_in_osv3",
"url",
self.spec.builder_openshift_url.value)
if use_auth is not None:
self.dj.dock_json_set_arg('exit_plugins',
"store_metadata_in_osv3",
"use_auth", use_auth)
def set_secret_for_plugin(self, secret, plugin=None, mount_path=None):
"""
Sets secret for plugin, if no plugin specified
it will also set general secret
:param secret: str, secret name
:param plugin: tuple, (plugin type, plugin name, argument name)
:param mount_path: str, mount path of secret
"""
has_plugin_conf = False
if plugin is not None:
has_plugin_conf = self.dj.dock_json_has_plugin_conf(plugin[0],
plugin[1])
if 'secrets' in self.template['spec']['strategy']['customStrategy']:
if not plugin or has_plugin_conf:
custom = self.template['spec']['strategy']['customStrategy']
if mount_path:
secret_path = mount_path
else:
secret_path = os.path.join(SECRETS_PATH, secret)
logger.info("Configuring %s secret at %s", secret, secret_path)
existing = [secret_mount for secret_mount in custom['secrets']
if secret_mount['secretSource']['name'] == secret]
if existing:
logger.debug("secret %s already set", secret)
else:
custom['secrets'].append({
'secretSource': {
'name': secret,
},
'mountPath': secret_path,
})
# there's no need to set args if no plugin secret specified
# this is used in tag_and_push plugin, as it sets secret path
# for each registry separately
if plugin and plugin[2] is not None:
self.dj.dock_json_set_arg(*(plugin + (secret_path,)))
else:
logger.debug("not setting secret for unused plugin %s",
plugin[1])
def set_secrets(self, secrets):
"""
:param secrets: dict, {(plugin type, plugin name, argument name): secret name}
for example {('exit_plugins', 'koji_promote', 'koji_ssl_certs'): 'koji_ssl_certs', ...}
"""
secret_set = False
for (plugin, secret) in secrets.items():
if not isinstance(plugin, tuple) or len(plugin) != 3:
raise ValueError('got "%s" as secrets key, need 3-tuple' % plugin)
if secret is not None:
if isinstance(secret, list):
for secret_item in secret:
self.set_secret_for_plugin(secret_item, plugin=plugin)
else:
self.set_secret_for_plugin(secret, plugin=plugin)
secret_set = True
if not secret_set:
# remove references to secret if no secret was set
if 'secrets' in self.template['spec']['strategy']['customStrategy']:
del self.template['spec']['strategy']['customStrategy']['secrets']
def set_kerberos_auth(self, plugins):
if not self.spec.koji_use_kerberos.value:
return
krb_principal = self.spec.koji_kerberos_principal.value
krb_keytab = self.spec.koji_kerberos_keytab.value
if not (krb_principal and krb_keytab):
logger.debug('Kerberos auth requested but missing principal and/or keytab values')
return
for phase, plugin in plugins:
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
continue
self.dj.dock_json_set_arg(phase, plugin, 'koji_principal', krb_principal)
self.dj.dock_json_set_arg(phase, plugin, 'koji_keytab', krb_keytab)
@staticmethod
def remove_tag_and_push_registries(tag_and_push_registries, version):
"""
Remove matching entries from tag_and_push_registries (in-place)
:param tag_and_push_registries: dict, uri -> dict
:param version: str, 'version' to match against
"""
registries = [uri
for uri, regdict in tag_and_push_registries.items()
if regdict['version'] == version]
for registry in registries:
logger.info("removing %s registry: %s", version, registry)
del tag_and_push_registries[registry]
def is_custom_base_image(self):
"""
Returns whether or not this is a build from a custom base image
"""
return bool(re.match('^koji/image-build(:.*)?$',
self.base_image or ''))
def adjust_for_registry_api_versions(self):
"""
Enable/disable plugins depending on supported registry API versions
"""
versions = self.spec.registry_api_versions.value
try:
push_conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',
'tag_and_push')
tag_and_push_registries = push_conf['args']['registries']
except (KeyError, IndexError):
tag_and_push_registries = {}
if 'v1' not in versions:
# Remove v1-only plugins
for phase, name in [('postbuild_plugins', 'pulp_push')]:
logger.info("removing v1-only plugin: %s", name)
self.dj.remove_plugin(phase, name)
# remove extra tag_and_push config
self.remove_tag_and_push_registries(tag_and_push_registries, 'v1')
if 'v2' not in versions:
# Remove v2-only plugins
logger.info("removing v2-only plugins: pulp_sync, delete_from_registry")
self.dj.remove_plugin('postbuild_plugins', 'pulp_sync')
self.dj.remove_plugin('exit_plugins', 'delete_from_registry')
# remove extra tag_and_push config
self.remove_tag_and_push_registries(tag_and_push_registries, 'v2')
# Remove 'version' from tag_and_push plugin config as it's no
# longer needed
for regdict in tag_and_push_registries.values():
if 'version' in regdict:
del regdict['version']
def adjust_for_triggers(self):
"""Remove trigger-related plugins when needed
If there are no triggers defined, it's assumed the
feature is disabled and all trigger-related plugins
are removed.
If there are triggers defined, and this is a custom
base image, some trigger-related plugins do not apply.
All but import_image are disabled in this case.
Additionally, this method ensures that custom base
images never have triggers since triggering a base
image rebuild is not a valid scenario.
"""
triggers = self.template['spec'].get('triggers', [])
remove_plugins = [
("prebuild_plugins", "check_and_set_rebuild"),
("prebuild_plugins", "stop_autorebuild_if_disabled"),
]
should_remove = False
if triggers and self.is_custom_base_image():
msg = "removing %s from request because custom base image"
del self.template['spec']['triggers']
should_remove = True
elif not triggers:
remove_plugins.append(("postbuild_plugins", "import_image"))
msg = "removing %s from request because there are no triggers"
should_remove = True
if should_remove:
for when, which in remove_plugins:
logger.info(msg, which)
self.dj.remove_plugin(when, which)
def adjust_for_scratch(self):
"""
Remove koji Content Generator related plugins if no triggers set in
order to hadle the "scratch build" scenario
"""
if self.scratch:
for when, which in [
("postbuild_plugins", "compress"),
("postbuild_plugins", "tag_from_config"),
("exit_plugins", "koji_promote"),
("exit_plugins", "koji_tag_build"),
]:
logger.info("removing %s from scratch build request",
which)
self.dj.remove_plugin(when, which)
if self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'tag_by_labels'):
self.dj.dock_json_set_arg('postbuild_plugins', 'tag_by_labels',
'unique_tag_only', True)
def adjust_for_custom_base_image(self):
"""
Disable plugins to handle builds depending on whether
or not this is a build from a custom base image.
"""
plugins = []
if self.is_custom_base_image():
# Plugins irrelevant to building base images.
plugins.append(("prebuild_plugins", "pull_base_image"))
msg = "removing %s from custom image build request"
else:
# Plugins not needed for building non base images.
plugins.append(("prebuild_plugins", "add_filesystem"))
msg = "removing %s from non custom image build request"
for when, which in plugins:
logger.info(msg, which)
self.dj.remove_plugin(when, which)
def render_add_filesystem(self):
phase = 'prebuild_plugins'
plugin = 'add_filesystem'
if self.dj.dock_json_has_plugin_conf(phase, plugin):
if not self.spec.kojihub.value:
raise OsbsValidationException(
'Custom base image builds require kojihub to be defined')
self.dj.dock_json_set_arg(phase, plugin, 'koji_hub',
self.spec.kojihub.value)
if self.spec.proxy.value:
self.dj.dock_json_set_arg(phase, plugin, 'koji_proxyuser',
self.spec.proxy.value)
if self.spec.yum_repourls.value:
self.dj.dock_json_set_arg(phase, plugin, 'repos',
self.spec.yum_repourls.value)
def render_add_labels_in_dockerfile(self):
phase = 'prebuild_plugins'
plugin = 'add_labels_in_dockerfile'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
implicit_labels = {}
label_spec = {
'vendor': self.spec.vendor,
'authoritative-source-url': self.spec.authoritative_registry,
'distribution-scope': self.spec.distribution_scope,
}
for label, spec in label_spec.items():
if spec.value is not None:
implicit_labels[label] = spec.value
self.dj.dock_json_merge_arg(phase, plugin, 'labels', implicit_labels)
if self.spec.info_url_format.value:
self.dj.dock_json_set_arg(phase, plugin, 'info_url_format',
self.spec.info_url_format.value)
def render_koji(self):
"""
if there is yum repo specified, don't pick stuff from koji
"""
phase = 'prebuild_plugins'
plugin = 'koji'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
if self.spec.yum_repourls.value:
logger.info("removing koji from request "
"because there is yum repo specified")
self.dj.remove_plugin(phase, plugin)
elif not (self.spec.koji_target.value and
self.spec.kojiroot.value and
self.spec.kojihub.value):
logger.info("removing koji from request as not specified")
self.dj.remove_plugin(phase, plugin)
else:
self.dj.dock_json_set_arg(phase, plugin,
"target", self.spec.koji_target.value)
self.dj.dock_json_set_arg(phase, plugin,
"root", self.spec.kojiroot.value)
self.dj.dock_json_set_arg(phase, plugin,
"hub", self.spec.kojihub.value)
if self.spec.proxy.value:
self.dj.dock_json_set_arg(phase, plugin,
"proxy", self.spec.proxy.value)
def render_bump_release(self):
"""
If the bump_release plugin is present, configure it
"""
phase = 'prebuild_plugins'
plugin = 'bump_release'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
hub = self.spec.kojihub.value
if not hub:
logger.info('removing %s from request as koji hub not specified',
plugin)
self.dj.remove_plugin(phase, plugin)
return
self.dj.dock_json_set_arg(phase, plugin, 'hub', hub)
def render_koji_promote(self, use_auth=None):
if not self.dj.dock_json_has_plugin_conf('exit_plugins',
'koji_promote'):
return
if self.spec.kojihub.value:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote', 'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'kojihub', self.spec.kojihub.value)
koji_target = self.spec.koji_target.value
if koji_target is not None:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'target', koji_target)
if use_auth is not None:
self.dj.dock_json_set_arg('exit_plugins', 'koji_promote',
'use_auth', use_auth)
else:
logger.info("removing koji_promote from request as no kojihub "
"specified")
self.dj.remove_plugin("exit_plugins", "koji_promote")
def render_koji_tag_build(self):
phase = 'exit_plugins'
plugin = 'koji_tag_build'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
if not self.spec.kojihub.value:
logger.info('Removing %s because no kojihub was specified', plugin)
self.dj.remove_plugin(phase, plugin)
return
if not self.spec.koji_target.value:
logger.info('Removing %s because no koji_target was specified', plugin)
self.dj.remove_plugin(phase, plugin)
return
self.dj.dock_json_set_arg(phase, plugin, 'kojihub', self.spec.kojihub.value)
self.dj.dock_json_set_arg(phase, plugin, 'target', self.spec.koji_target.value)
def render_sendmail(self):
"""
if we have smtp_host and smtp_from, configure sendmail plugin,
else remove it
"""
phase = 'exit_plugins'
plugin = 'sendmail'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
if self.spec.smtp_host.value and self.spec.smtp_from.value:
self.dj.dock_json_set_arg(phase, plugin, 'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg(phase, plugin, 'smtp_host',
self.spec.smtp_host.value)
self.dj.dock_json_set_arg(phase, plugin, 'from_address',
self.spec.smtp_from.value)
else:
logger.info("removing sendmail from request, "
"requires smtp_host and smtp_from")
self.dj.remove_plugin(phase, plugin)
return
if self.spec.kojihub.value and self.spec.kojiroot.value:
self.dj.dock_json_set_arg(phase, plugin,
'koji_hub', self.spec.kojihub.value)
self.dj.dock_json_set_arg(phase, plugin,
"koji_root", self.spec.kojiroot.value)
if self.spec.smtp_to_submitter.value:
self.dj.dock_json_set_arg(phase, plugin, 'to_koji_submitter',
self.spec.smtp_to_submitter.value)
if self.spec.smtp_to_pkgowner.value:
self.dj.dock_json_set_arg(phase, plugin, 'to_koji_pkgowner',
self.spec.smtp_to_pkgowner.value)
if self.spec.smtp_additional_addresses.value:
self.dj.dock_json_set_arg(phase, plugin, 'additional_addresses',
self.spec.smtp_additional_addresses.value)
if self.spec.smtp_error_addresses.value:
self.dj.dock_json_set_arg(phase, plugin,
'error_addresses', self.spec.smtp_error_addresses.value)
if self.spec.smtp_email_domain.value:
self.dj.dock_json_set_arg(phase, plugin,
'email_domain', self.spec.smtp_email_domain.value)
def render_fetch_maven_artifacts(self):
"""Configure fetch_maven_artifacts plugin"""
phase = 'prebuild_plugins'
plugin = 'fetch_maven_artifacts'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
koji_hub = self.spec.kojihub.value
koji_root = self.spec.kojiroot.value
if not koji_hub and not koji_root:
logger.info('Removing %s because kojihub and kojiroot were not specified', plugin)
self.dj.remove_plugin(phase, plugin)
return
self.dj.dock_json_set_arg(phase, plugin, 'koji_hub', koji_hub)
self.dj.dock_json_set_arg(phase, plugin, "koji_root", koji_root)
if self.spec.proxy.value:
self.dj.dock_json_set_arg(phase, plugin, 'koji_proxyuser', self.spec.proxy.value)
if self.spec.artifacts_allowed_domains.value:
self.dj.dock_json_set_arg(phase, plugin, 'allowed_domains',
self.spec.artifacts_allowed_domains.value)
def render_pulp_pull(self):
"""
If a pulp registry is specified, use pulp_pull plugin
"""
phase = 'postbuild_plugins'
plugin = 'pulp_pull'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
pulp_registry = self.spec.pulp_registry.value
if not pulp_registry:
logger.info("removing %s from request, requires pulp_registry", pulp_registry)
self.dj.remove_plugin(phase, plugin)
def render_pulp_push(self):
"""
If a pulp registry is specified, use the pulp plugin
"""
if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'pulp_push'):
return
pulp_registry = self.spec.pulp_registry.value
if pulp_registry:
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_push',
'pulp_registry_name', pulp_registry)
# Verify we have either a secret or username/password
if self.spec.pulp_secret.value is None:
conf = self.dj.dock_json_get_plugin_conf('postbuild_plugins',
'pulp_push')
args = conf.get('args', {})
if 'username' not in args:
raise OsbsValidationException("Pulp registry specified "
"but no auth config")
else:
# If no pulp registry is specified, don't run the pulp plugin
logger.info("removing pulp_push from request, "
"requires pulp_registry")
self.dj.remove_plugin("postbuild_plugins", "pulp_push")
def render_pulp_sync(self):
"""
If a pulp registry is specified, use the pulp plugin as well as the
delete_from_registry to delete the image after sync
"""
if not self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'pulp_sync'):
return
pulp_registry = self.spec.pulp_registry.value
# Find which registry to use
docker_registry = None
registry_secret = None
registries = zip_longest(self.spec.registry_uris.value,
self.spec.registry_secrets.value)
for registry, secret in registries:
if registry.version == 'v2':
# First specified v2 registry is the one we'll tell pulp
# to sync from. Keep the http prefix -- pulp wants it.
docker_registry = registry.uri
registry_secret = secret
logger.info("using docker v2 registry %s for pulp_sync",
docker_registry)
break
if pulp_registry and docker_registry:
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',
'pulp_registry_name', pulp_registry)
self.dj.dock_json_set_arg('postbuild_plugins', 'pulp_sync',
'docker_registry', docker_registry)
if registry_secret:
self.set_secret_for_plugin(registry_secret,
plugin=('postbuild_plugins',
'pulp_sync',
'registry_secret_path'))
# Verify we have a pulp secret
if self.spec.pulp_secret.value is None:
raise OsbsValidationException("Pulp registry specified "
"but no auth config")
source_registry = self.spec.source_registry_uri.value
perform_delete = (source_registry is None or
source_registry.docker_uri != registry.docker_uri)
if perform_delete:
delete_registries = {docker_registry: {}}
if registry_secret:
delete_registries[docker_registry]['secret'] = \
os.path.join(SECRETS_PATH, registry_secret)
# tag_and_push configured the registry secret, no neet to set it again
self.dj.dock_json_set_arg('exit_plugins', 'delete_from_registry',
'registries', delete_registries)
else:
logger.info("removing delete_from_registry from request, "
"source and target registry are identical")
self.dj.remove_plugin("exit_plugins", "delete_from_registry")
else:
# If no pulp registry is specified, don't run the pulp plugin
logger.info("removing pulp_sync+delete_from_registry from request, "
"requires pulp_registry and a v2 registry")
self.dj.remove_plugin("postbuild_plugins", "pulp_sync")
self.dj.remove_plugin("exit_plugins", "delete_from_registry")
def render_import_image(self, use_auth=None):
"""
Configure the import_image plugin
"""
if self.spec.imagestream_name is None or self.spec.imagestream_url is None:
logger.info("removing import_image from request, "
"registry or repo url is not defined")
self.dj.remove_plugin('postbuild_plugins', 'import_image')
return
if self.dj.dock_json_has_plugin_conf('postbuild_plugins',
'import_image'):
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'imagestream',
self.spec.imagestream_name.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'docker_image_repo',
self.spec.imagestream_url.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'url',
self.spec.builder_openshift_url.value)
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'build_json_dir',
self.spec.builder_build_json_dir.value)
use_auth = self.spec.use_auth.value
if use_auth is not None:
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'use_auth', use_auth)
if self.spec.imagestream_insecure_registry.value:
self.dj.dock_json_set_arg('postbuild_plugins', 'import_image',
'insecure_registry', True)
def render_distgit_fetch_artefacts(self):
phase = 'prebuild_plugins'
plugin = 'distgit_fetch_artefacts'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
if self.spec.sources_command.value is not None:
self.dj.dock_json_set_arg(phase, plugin, "command",
self.spec.sources_command.value)
else:
logger.info('removing {0}, no sources_command was provided'.format(plugin))
self.dj.remove_plugin(phase, plugin)
def render_pull_base_image(self):
phase = 'prebuild_plugins'
plugin = 'pull_base_image'
if not self.dj.dock_json_has_plugin_conf(phase, plugin):
return
# pull_base_image wants a docker URI so strip off the scheme part
source_registry = self.spec.source_registry_uri.value
self.dj.dock_json_set_arg(phase, plugin, 'parent_registry',
source_registry.docker_uri if source_registry else None)
def render_customizations(self):
"""
Customize prod_inner for site specific customizations
"""
disable_plugins = self.customize_conf.get('disable_plugins', [])
if not disable_plugins:
logger.debug("No site-specific plugins to disable")
else:
for plugin_dict in disable_plugins:
try:
self.dj.remove_plugin(
plugin_dict['plugin_type'],
plugin_dict['plugin_name']
)
logger.debug(
"site-specific plugin disabled -> Type:{0} Name:{1}".format(
plugin_dict['plugin_type'],
plugin_dict['plugin_name']
)
)
except KeyError:
# Malformed config
logger.debug("Invalid custom configuration found for disable_plugins")
enable_plugins = self.customize_conf.get('enable_plugins', [])
if not enable_plugins:
logger.debug("No site-specific plugins to enable")
else:
for plugin_dict in enable_plugins:
try:
self.dj.add_plugin(
plugin_dict['plugin_type'],
plugin_dict['plugin_name'],
plugin_dict['plugin_args']
)
logger.debug(
"site-specific plugin enabled -> Type:{0} Name:{1} Args: {2}".format(
plugin_dict['plugin_type'],
plugin_dict['plugin_name'],
plugin_dict['plugin_args']
)
)
except KeyError:
# Malformed config
logger.debug("Invalid custom configuration found for enable_plugins")
def render_version(self):
self.dj.dock_json_set_param('client_version', client_version)
def render(self, validate=True):
if validate:
self.spec.validate()
self.render_customizations()
# !IMPORTANT! can't be too long: https://github.com/openshift/origin/issues/733
self.template['metadata']['name'] = self.spec.name.value
self.render_resource_limits()
self.template['spec']['source']['git']['uri'] = self.spec.git_uri.value
self.template['spec']['source']['git']['ref'] = self.spec.git_ref.value
if self.spec.registry_uris.value:
primary_registry_uri = self.spec.registry_uris.value[0].docker_uri
tag_with_registry = '{0}/{1}'.format(primary_registry_uri,
self.spec.image_tag.value)
self.template['spec']['output']['to']['name'] = tag_with_registry
else:
self.template['spec']['output']['to']['name'] = self.spec.image_tag.value
self.render_tag_and_push_registries()
if 'triggers' in self.template['spec']:
imagechange = self.template['spec']['triggers'][0]['imageChange']
imagechange['from']['name'] = self.spec.trigger_imagestreamtag.value
self.render_add_yum_repo_by_url()
use_auth = self.spec.use_auth.value
self.render_check_and_set_rebuild(use_auth=use_auth)
self.render_store_metadata_in_osv3(use_auth=use_auth)
# Remove legacy sourceSecret in case an older template is used.
if 'sourceSecret' in self.template['spec']['source']:
del self.template['spec']['source']['sourceSecret']
if self.spec.build_imagestream.value:
self.template['spec']['strategy']['customStrategy']['from']['kind'] = 'ImageStreamTag'
self.template['spec']['strategy']['customStrategy']['from']['name'] = \
self.spec.build_imagestream.value
else:
self.template['spec']['strategy']['customStrategy']['from']['name'] = \
self.spec.build_image.value
repo_name = git_repo_humanish_part_from_uri(self.spec.git_uri.value)
# NOTE: Since only the repo name is used, a forked repos will have
# the same git-repo-name tag. This is a known limitation. If this
# use case must be handled properly, the git URI must be taken into
# account.
self.set_label('git-repo-name', repo_name)
if self.spec.git_branch.value:
self.set_label('git-branch', self.spec.git_branch.value)
else:
self.set_label('git-branch', 'unknown')
self.render_distgit_fetch_artefacts()
self.render_pull_base_image()
self.adjust_for_triggers()
self.adjust_for_scratch()
self.adjust_for_custom_base_image()
# Enable/disable plugins as needed for target registry API versions
self.adjust_for_registry_api_versions()
self.set_secrets({('prebuild_plugins',
'reactor_config',
'config_path'):
self.spec.reactor_config_secret.value,
('postbuild_plugins',
'pulp_push',
'pulp_secret_path'):
self.spec.pulp_secret.value,
('postbuild_plugins',
'pulp_sync',
'pulp_secret_path'):
self.spec.pulp_secret.value,
# pulp_sync registry_secret_path set
# in render_pulp_sync
('exit_plugins', 'koji_promote', 'koji_ssl_certs'):
self.spec.koji_certs_secret.value,
('exit_plugins', 'koji_tag_build', 'koji_ssl_certs'):
self.spec.koji_certs_secret.value,
('prebuild_plugins', 'add_filesystem', 'koji_ssl_certs_dir'):
self.spec.koji_certs_secret.value,
('prebuild_plugins', 'bump_release', 'koji_ssl_certs_dir'):
self.spec.koji_certs_secret.value,
('prebuild_plugins', 'koji', 'koji_ssl_certs_dir'):
self.spec.koji_certs_secret.value,
('prebuild_plugins', 'fetch_maven_artifacts', 'koji_ssl_certs_dir'):
self.spec.koji_certs_secret.value,
('exit_plugins', 'sendmail', 'koji_ssl_certs_dir'):
self.spec.koji_certs_secret.value,
('postbuild_plugins', 'tag_and_push',
# Only set the secrets for the build, don't
# add the path to the plugin's
# configuration. This is done elsewhere.
None):
self.spec.registry_secrets.value,
('buildstep_plugins', 'orchestrate_build', 'osbs_client_config'):
self.spec.client_config_secret.value})
self.set_kerberos_auth([
('prebuild_plugins', 'fetch_maven_artifacts'),
('exit_plugins', 'koji_promote'),
('exit_plugins', 'koji_tag_build'),
('exit_plugins', 'sendmail')
])
for (secret, path) in self.spec.token_secrets.value.items():
self.set_secret_for_plugin(secret, mount_path=path)
if self.spec.pulp_secret.value:
# Don't push to docker registry, we're using pulp here
# but still construct the unique tag
self.template['spec']['output']['to']['name'] = \
self.spec.image_tag.value
koji_task_id = self.spec.koji_task_id.value
if koji_task_id is not None:
self.template['metadata'].setdefault('labels', {})
self.template['metadata']['labels']['koji-task-id'] = str(koji_task_id)
use_auth = self.spec.use_auth.value
self.render_reactor_config()
self.render_orchestrate_build()
self.render_add_filesystem()
self.render_add_labels_in_dockerfile()
self.render_koji()
self.render_bump_release()
self.render_import_image(use_auth=use_auth)
self.render_pulp_pull()
self.render_pulp_push()
self.render_pulp_sync()
self.render_koji_promote(use_auth=use_auth)
self.render_koji_tag_build()
self.render_sendmail()
self.render_fetch_maven_artifacts()
self.render_version()
self.dj.write_dock_json()
self.build_json = self.template
logger.debug(self.build_json)
return self.build_json