def do_create(tag, build_name, reproducible_artifact_path, commit, variant_arguments, all_completes):
"""Create a installer script for each variant in bootstrap_dict.
Writes a dcos_generate_config.<variant>.sh for each variant in
bootstrap_dict to the working directory, except for the default variant's
script, which is written to dcos_generate_config.sh. Returns a dict mapping
variants to (genconf_version, genconf_filename) tuples.
Outputs the generated dcos_generate_config.sh as it's artifacts.
"""
# TODO(cmaloney): Build installers in parallel.
# Variants are sorted for stable ordering.
for variant, bootstrap_info in sorted(variant_arguments.items(),
key=lambda kv: pkgpanda.util.variant_str(kv[0])):
with logger.scope("Building installer for variant: ".format(pkgpanda.util.variant_name(variant))):
bootstrap_installer_name = '{}installer'.format(pkgpanda.util.variant_prefix(variant))
installer_filename = make_installer_docker(variant, all_completes[variant],
all_completes[bootstrap_installer_name])
yield {
'channel_path': 'dcos_generate_config.{}sh'.format(pkgpanda.util.variant_prefix(variant)),
'local_path': installer_filename
}
# Build dcos-launch
# TODO(cmaloney): This really doesn't belong to here, but it's the best place it fits for now.
# dcos-launch works many places which aren't bash / on-premise installers.
# It also isn't dependent on the "reproducible" artifacts at all. Just the commit...
with logger.scope("building dcos-launch"):
yield {
'channel_path': 'dcos-launch',
'local_path': make_dcos_launch()
}
def do_create(tag, build_name, reproducible_artifact_path, commit, variant_arguments, all_completes):
"""Create a installer script for each variant in bootstrap_dict.
Writes a dcos_generate_config.<variant>.sh for each variant in
bootstrap_dict to the working directory, except for the default variant's
script, which is written to dcos_generate_config.sh. Returns a dict mapping
variants to (genconf_version, genconf_filename) tuples.
Outputs the generated dcos_generate_config.sh as it's artifacts.
"""
# TODO(cmaloney): Build installers in parallel.
# Variants are sorted for stable ordering.
for variant in sorted(variant_arguments.keys(), key=lambda k: pkgpanda.util.variant_str(k)):
variant_name = pkgpanda.util.variant_name(variant)
bootstrap_installer_name = '{}installer'.format(pkgpanda.util.variant_prefix(variant))
if bootstrap_installer_name not in all_completes:
print('WARNING: No installer tree for variant: {}'.format(variant_name))
else:
with logger.scope("Building installer for variant: {}".format(variant_name)):
yield {
'channel_path': 'dcos_generate_config.{}sh'.format(pkgpanda.util.variant_prefix(variant)),
'local_path': make_installer_docker(variant, all_completes[variant],
all_completes[bootstrap_installer_name])
}
def setup_cluster_workload(self, dcos_api: DcosApiSession, healthcheck_app: dict, dns_app: dict):
# Deploy test apps.
# TODO(branden): We ought to be able to deploy these apps concurrently. See
# https://mesosphere.atlassian.net/browse/DCOS-13360.
with logger.scope("deploy apps"):
dcos_api.marathon.deploy_app(healthcheck_app)
dcos_api.marathon.ensure_deployments_complete()
# This is a hack to make sure we don't deploy dns_app before the name it's
# trying to resolve is available.
self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME'])
dcos_api.marathon.deploy_app(dns_app, check_health=False)
dcos_api.marathon.ensure_deployments_complete()
test_apps = [healthcheck_app, dns_app]
self.test_app_ids = [app['id'] for app in test_apps]
self.tasks_start = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids}
self.log.debug('Test app tasks at start:\n' + pprint.pformat(self.tasks_start))
for app in test_apps:
assert app['instances'] == len(self.tasks_start[app['id']])
# Save the master's state of the task to compare with
# the master's view after the upgrade.
# See this issue for why we check for a difference:
# https://issues.apache.org/jira/browse/MESOS-1718
self.task_state_start = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0])
def apply_storage_commands(self, storage_commands):
assert storage_commands.keys() == {'stage1', 'stage2'}
if self.__noop:
return
with logger.scope("Uploading artifacts"):
apply_storage_commands(self.__storage_providers, storage_commands)
def make_stable_artifacts(cache_repository_url):
metadata = {
"commit": util.dcos_image_commit,
"core_artifacts": [],
"packages": set()
}
# TODO(cmaloney): Rather than guessing / reverse-engineering all these paths
# have do_build_packages get them directly from pkgpanda
with logger.scope("Building packages"):
try:
all_completes = do_build_packages(cache_repository_url)
except pkgpanda.build.BuildError as ex:
logger.error("Failure building package(s): {}".format(ex))
raise
# The installer and util are built bootstraps, but not a DC/OS variants. We use
# iteration over the complete_dict to enumerate all variants a whole lot,
# so explicity remove installer/util here so people don't accidentally hit it.
# TODO: make this into a tree option
complete_dict = dict()
for name, info in copy.copy(all_completes).items():
if name is not None and (name.endswith('installer') or name.endswith('util')):
continue
complete_dict[name] = info
metadata["complete_dict"] = complete_dict
metadata["all_completes"] = all_completes
metadata["bootstrap_dict"] = {k: v['bootstrap'] for k, v in complete_dict.items()}
metadata["all_bootstraps"] = {k: v['bootstrap'] for k, v in all_completes.items()}
def add_file(info):
metadata["core_artifacts"].append(info)
def add_package(package_id):
if package_id in metadata['packages']:
return
metadata['packages'].add(package_id)
add_file(get_package_artifact(package_id))
# Add the bootstrap, active.json, packages as reproducible_path artifacts
# Add the <variant>.bootstrap.latest as a channel_path
for name, info in sorted(all_completes.items(), key=lambda kv: pkgpanda.util.variant_str(kv[0])):
for file in make_bootstrap_artifacts(info['bootstrap'], info['packages'], name, 'packages/cache'):
add_file(file)
# Add all the packages which haven't been added yet
for package_id in sorted(info['packages']):
add_package(package_id)
# Sets aren't json serializable, so transform to a list for future use.
metadata['packages'] = list(sorted(metadata['packages']))
return metadata
def make_bootstrap(package_set):
with logger.scope("Making bootstrap variant: {}".format(pkgpanda.util.variant_name(package_set.variant))):
package_paths = list()
for name, pkg_variant in package_set.bootstrap_packages:
package_paths.append(built_packages[name][pkg_variant])
if mkbootstrap:
return make_bootstrap_tarball(
package_store,
list(sorted(package_paths)),
package_set.variant)
def gen_simple_template(variant_prefix, filename, arguments, extra_source):
results = gen.generate(
arguments=arguments,
extra_templates=[
'aws/templates/cloudformation.json',
'aws/dcos-config.yaml',
'coreos-aws/cloud-config.yaml',
'coreos/cloud-config.yaml'],
cc_package_files=[
'/etc/cfn_signal_metadata',
'/etc/adminrouter.env',
'/etc/ui-config.json',
'/etc/dns_config',
'/etc/exhibitor',
'/etc/mesos-master-provider',
'/etc/extra_master_addresses'],
extra_sources=[aws_base_source, aws_simple_source, extra_source])
cloud_config = results.templates['cloud-config.yaml']
# Add general services
cloud_config = results.utils.add_services(cloud_config, 'coreos')
# Specialize for master, slave, slave_public
variant_cloudconfig = {}
for variant, params in cf_instance_groups.items():
cc_variant = deepcopy(cloud_config)
# Specialize the dcos-cfn-signal service
cc_variant = results.utils.add_units(
cc_variant,
yaml.safe_load(gen.template.parse_str(late_services).render(params)))
# Add roles
cc_variant = results.utils.add_roles(cc_variant, params['roles'] + ['aws'])
# NOTE: If this gets printed in string stylerather than '|' the AWS
# parameters which need to be split out for the cloudformation to
# interpret end up all escaped and undoing it would be hard.
variant_cloudconfig[variant] = results.utils.render_cloudconfig(cc_variant)
# Render the cloudformation
cloudformation = render_cloudformation(
results.templates['cloudformation.json'],
master_cloud_config=variant_cloudconfig['master'],
slave_cloud_config=variant_cloudconfig['slave'],
slave_public_cloud_config=variant_cloudconfig['slave_public'])
with logger.scope("Validating CloudFormation"):
validate_cf(cloudformation)
yield from _as_artifact_and_pkg(variant_prefix, filename, (cloudformation, results))
def verify_apps_state(self, dcos_api: DcosApiSession, dns_app: dict):
with logger.scope("verify apps state"):
# nested methods here so we can "close" over external state
def marathon_app_tasks_survive_upgrade():
# Verify that the tasks we started are still running.
tasks_end = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids}
self.log.debug('Test app tasks at end:\n' + pprint.pformat(tasks_end))
if not self.tasks_start == tasks_end:
self.teamcity_msg.testFailed(
"test_upgrade_vpc.marathon_app_tasks_survive_upgrade",
details="expected: {}\nactual: {}".format(self.tasks_start, tasks_end))
def test_mesos_task_state_remains_consistent():
# Verify that the "state" of the task does not change.
task_state_end = self.get_master_task_state(dcos_api, self.tasks_start[self.test_app_ids[0]][0])
if not self.task_state_start == task_state_end:
self.teamcity_msg.testFailed(
"test_upgrade_vpc.test_mesos_task_state_remains_consistent",
details="expected: {}\nactual: {}".format(self.task_state_start, task_state_end))
def test_app_dns_survive_upgrade():
# Verify DNS didn't fail.
marathon_framework_id = dcos_api.marathon.get('/v2/info').json()['frameworkId']
dns_app_task = dcos_api.marathon.get('/v2/apps' + dns_app['id'] + '/tasks').json()['tasks'][0]
dns_log = self.parse_dns_log(dcos_api.mesos_sandbox_file(
dns_app_task['slaveId'],
marathon_framework_id,
dns_app_task['id'],
dns_app['env']['DNS_LOG_FILENAME'],
))
dns_failure_times = [entry[0] for entry in dns_log if entry[1] != 'SUCCESS']
if len(dns_failure_times) > 0:
message = 'Failed to resolve Marathon app hostname {} at least once.'.format(
dns_app['env']['RESOLVE_NAME'])
err_msg = message + ' Hostname failed to resolve at these times:\n' + '\n'.join(dns_failure_times)
self.log.debug(err_msg)
self.teamcity_msg.testFailed("test_upgrade_vpc.test_app_dns_survive_upgrade", details=err_msg)
self.log_test("test_upgrade_vpc.marathon_app_tasks_survive_upgrade", marathon_app_tasks_survive_upgrade)
self.log_test(
"test_upgrade_vpc.test_mesos_task_state_remains_consistent",
test_mesos_task_state_remains_consistent
)
self.log_test("test_upgrade_vpc.test_app_dns_survive_upgrade", test_app_dns_survive_upgrade)
def setup_cluster_workload(self, dcos_api: DcosApiSession,
healthcheck_app: dict, dns_app: dict,
viplisten_app: dict, viptalk_app: dict):
# Deploy test apps.
# TODO(branden): We ought to be able to deploy these apps concurrently. See
# https://mesosphere.atlassian.net/browse/DCOS-13360.
with logger.scope("deploy apps"):
dcos_api.marathon.deploy_app(viplisten_app)
dcos_api.marathon.ensure_deployments_complete()
# viptalk app depends on VIP from viplisten app, which may still fail
# the first try immediately after ensure_deployments_complete
dcos_api.marathon.deploy_app(viptalk_app, ignore_failed_tasks=True)
dcos_api.marathon.ensure_deployments_complete()
dcos_api.marathon.deploy_app(healthcheck_app)
dcos_api.marathon.ensure_deployments_complete()
# This is a hack to make sure we don't deploy dns_app before the name it's
# trying to resolve is available.
self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME'])
dcos_api.marathon.deploy_app(dns_app, check_health=False)
dcos_api.marathon.ensure_deployments_complete()
test_apps = [healthcheck_app, dns_app, viplisten_app, viptalk_app]
self.test_app_ids = [app['id'] for app in test_apps]
self.tasks_start = {
app_id: sorted(self.app_task_ids(dcos_api, app_id))
for app_id in self.test_app_ids
}
self.log.debug('Test app tasks at start:\n' +
pprint.pformat(self.tasks_start))
for app in test_apps:
assert app['instances'] == len(self.tasks_start[app['id']])
# Save the master's state of the task to compare with
# the master's view after the upgrade.
# See this issue for why we check for a difference:
# https://issues.apache.org/jira/browse/MESOS-1718
self.task_state_start = self.get_master_task_state(
dcos_api, self.tasks_start[self.test_app_ids[0]][0])
def setup_cluster_workload(self, dcos_api: DcosApiSession, healthcheck_app: dict, dns_app: dict):
# Deploy test apps.
# TODO(branden): We ought to be able to deploy these apps concurrently. See
# https://mesosphere.atlassian.net/browse/DCOS-13360.
with logger.scope("deploy apps"):
dcos_api.marathon.deploy_app(healthcheck_app)
dcos_api.marathon.ensure_deployments_complete()
# This is a hack to make sure we don't deploy dns_app before the name it's
# trying to resolve is available.
self.wait_for_dns(dcos_api, dns_app['env']['RESOLVE_NAME'])
dcos_api.marathon.deploy_app(dns_app, check_health=False)
dcos_api.marathon.ensure_deployments_complete()
test_apps = [healthcheck_app, dns_app]
self.test_app_ids = [app['id'] for app in test_apps]
self.tasks_start = {app_id: sorted(self.app_task_ids(dcos_api, app_id)) for app_id in self.test_app_ids}
self.log.debug('Test app tasks at start:\n' + pprint.pformat(self.tasks_start))
for app in test_apps:
assert app['instances'] == len(self.tasks_start[app['id']])
def build(package_store: PackageStore, name: str, variant, clean_after_build, recursive=False):
msg = "Building package {} variant {}".format(name, pkgpanda.util.variant_name(variant))
with logger.scope(msg):
return _build(package_store, name, variant, clean_after_build, recursive)
def build_tree(package_store, mkbootstrap, tree_variant):
"""Build packages and bootstrap tarballs for one or all tree variants.
Returns a dict mapping tree variants to bootstrap IDs.
If tree_variant is None, builds all available tree variants.
"""
# TODO(cmaloney): Add support for circular dependencies. They are doable
# long as there is a pre-built version of enough of the packages.
# TODO(cmaloney): Make it so when we're building a treeinfo which has a
# explicit package list we don't build all the other packages.
build_order = list()
visited = set()
built = set()
def visit(pkg_tuple: tuple):
"""Add a package and its requires to the build order.
Raises AssertionError if pkg_tuple is in the set of visited packages.
If the package has any requires, they're recursively visited and added
to the build order depth-first. Then the package itself is added.
"""
# Visit the node for the first (and only) time.
assert pkg_tuple not in visited
visited.add(pkg_tuple)
# Ensure all dependencies are built. Sorted for stability
for require in sorted(package_store.packages[pkg_tuple]['requires']):
require_tuple = expand_require(require)
# If the dependency has already been built, we can move on.
if require_tuple in built:
continue
# If the dependency has not been built but has been visited, then
# there's a cycle in the dependency graph.
if require_tuple in visited:
raise BuildError("Circular dependency. Circular link {0} -> {1}".format(pkg_tuple, require_tuple))
if PackageId.is_id(require_tuple[0]):
raise BuildError("Depending on a specific package id is not supported. Package {} "
"depends on {}".format(pkg_tuple, require_tuple))
if require_tuple not in package_store.packages:
raise BuildError("Package {0} require {1} not buildable from tree.".format(pkg_tuple, require_tuple))
# Add the dependency (after its dependencies, if any) to the build
# order.
visit(require_tuple)
build_order.append(pkg_tuple)
built.add(pkg_tuple)
# Can't compare none to string, so expand none -> "true" / "false", then put
# the string in a field after "" if none, the string if not.
def key_func(elem):
return elem[0], elem[1] is None, elem[1] or ""
def visit_packages(package_tuples):
for pkg_tuple in sorted(package_tuples, key=key_func):
if pkg_tuple in visited:
continue
visit(pkg_tuple)
if tree_variant:
package_sets = [package_store.get_package_set(tree_variant)]
else:
package_sets = package_store.get_all_package_sets()
with logger.scope("resolve package graph"):
# Build all required packages for all tree variants.
for package_set in package_sets:
visit_packages(package_set.all_packages)
built_packages = dict()
for (name, variant) in build_order:
built_packages.setdefault(name, dict())
# Run the build, store the built package path for later use.
# TODO(cmaloney): Only build the requested variants, rather than all variants.
built_packages[name][variant] = build(
package_store,
name,
variant,
True)
# Build bootstrap tarballs for all tree variants.
def make_bootstrap(package_set):
with logger.scope("Making bootstrap variant: {}".format(pkgpanda.util.variant_name(package_set.variant))):
package_paths = list()
for name, pkg_variant in package_set.bootstrap_packages:
package_paths.append(built_packages[name][pkg_variant])
if mkbootstrap:
return make_bootstrap_tarball(
package_store,
list(sorted(package_paths)),
package_set.variant)
# Build bootstraps and and package lists for all variants.
# TODO(cmaloney): Allow distinguishing between "build all" and "build the default one".
complete_cache_dir = package_store.get_complete_cache_dir()
check_call(['mkdir', '-p', complete_cache_dir])
results = {}
for package_set in package_sets:
info = {
'bootstrap': make_bootstrap(package_set),
'packages': sorted(
load_string(package_store.get_last_build_filename(*pkg_tuple))
for pkg_tuple in package_set.all_packages)}
write_json(
complete_cache_dir + '/' + pkgpanda.util.variant_prefix(package_set.variant) + 'complete.latest.json',
info)
results[package_set.variant] = info
return results
def make_channel_artifacts(metadata):
artifacts = []
provider_data = {}
providers = load_providers()
for name, module in sorted(providers.items()):
bootstrap_url = metadata['repository_url']
# If the particular provider has its own storage by the same name then
# Use the storage provider rather
if name in metadata['storage_urls']:
bootstrap_url = metadata['storage_urls'][name] + metadata[
'repository_path']
variant_arguments = dict()
for bootstrap_name, bootstrap_id in metadata['bootstrap_dict'].items():
variant_arguments[bootstrap_name] = copy.deepcopy({
'bootstrap_url':
bootstrap_url,
'provider':
name,
'bootstrap_id':
bootstrap_id,
'bootstrap_variant':
pkgpanda.util.variant_prefix(bootstrap_name)
})
# Load additional default variant arguments out of gen_extra
if os.path.exists('gen_extra/calc.py'):
mod = importlib.machinery.SourceFileLoader(
'gen_extra.calc', 'gen_extra/calc.py').load_module()
variant_arguments[bootstrap_name].update(
mod.provider_template_defaults)
# Add templates for the default variant.
# Use keyword args to make not matching ordering a loud error around changes.
with logger.scope("Creating {} deploy tools".format(module.__name__)):
for built_resource in module.do_create(
tag=metadata['tag'],
build_name=metadata['build_name'],
reproducible_artifact_path=metadata[
'reproducible_artifact_path'],
commit=metadata['commit'],
variant_arguments=variant_arguments,
all_bootstraps=metadata["all_bootstraps"]):
assert isinstance(built_resource, dict), built_resource
# Type switch
if 'packages' in built_resource:
for package in built_resource['packages']:
artifacts.append(get_gen_package_artifact(package))
else:
assert 'packages' not in built_resource
artifacts.append(built_resource)
# TODO(cmaloney): Check the provider artifacts adhere to the artifact template.
artifacts += provider_data.get('artifacts', list())
return artifacts
def _build(package_store, name, variant, clean_after_build, recursive):
assert isinstance(package_store, PackageStore)
tmpdir = tempfile.TemporaryDirectory(prefix="pkgpanda_repo")
repository = Repository(tmpdir.name)
package_dir = package_store.get_package_folder(name)
def src_abs(name):
return package_dir + '/' + name
def cache_abs(filename):
return package_store.get_package_cache_folder(name) + '/' + filename
# Build pkginfo over time, translating fields from buildinfo.
pkginfo = {}
# Build up the docker command arguments over time, translating fields as needed.
cmd = DockerCmd()
assert (name, variant) in package_store.packages, \
"Programming error: name, variant should have been validated to be valid before calling build()."
builder = IdBuilder(package_store.get_buildinfo(name, variant))
final_buildinfo = dict()
builder.add('name', name)
builder.add('variant', pkgpanda.util.variant_str(variant))
# Convert single_source -> sources
if builder.has('sources'):
if builder.has('single_source'):
raise BuildError('Both sources and single_source cannot be specified at the same time')
sources = builder.take('sources')
elif builder.has('single_source'):
sources = {name: builder.take('single_source')}
builder.replace('single_source', 'sources', sources)
else:
builder.add('sources', {})
sources = dict()
print("NOTICE: No sources specified")
final_buildinfo['sources'] = sources
# Construct the source fetchers, gather the checkout ids from them
checkout_ids = dict()
fetchers = dict()
try:
for src_name, src_info in sorted(sources.items()):
# TODO(cmaloney): Switch to a unified top level cache directory shared by all packages
cache_dir = package_store.get_package_cache_folder(name) + '/' + src_name
check_call(['mkdir', '-p', cache_dir])
fetcher = get_src_fetcher(src_info, cache_dir, package_dir)
fetchers[src_name] = fetcher
checkout_ids[src_name] = fetcher.get_id()
except ValidationError as ex:
raise BuildError("Validation error when fetching sources for package: {}".format(ex))
for src_name, checkout_id in checkout_ids.items():
# NOTE: single_source buildinfo was expanded above so the src_name is
# always correct here.
# Make sure we never accidentally overwrite something which might be
# important. Fields should match if specified (And that should be
# tested at some point). For now disallowing identical saves hassle.
assert_no_duplicate_keys(checkout_id, final_buildinfo['sources'][src_name])
final_buildinfo['sources'][src_name].update(checkout_id)
# Add the sha1 of the buildinfo.json + build file to the build ids
builder.update('sources', checkout_ids)
build_script = src_abs(builder.take('build_script'))
# TODO(cmaloney): Change dest name to build_script_sha1
builder.replace('build_script', 'build', pkgpanda.util.sha1(build_script))
builder.add('pkgpanda_version', pkgpanda.build.constants.version)
extra_dir = src_abs("extra")
# Add the "extra" folder inside the package as an additional source if it
# exists
if os.path.exists(extra_dir):
extra_id = hash_folder_abs(extra_dir, package_dir)
builder.add('extra_source', extra_id)
final_buildinfo['extra_source'] = extra_id
# Figure out the docker name.
docker_name = builder.take('docker')
cmd.container = docker_name
# Add the id of the docker build environment to the build_ids.
try:
docker_id = get_docker_id(docker_name)
except CalledProcessError:
# docker pull the container and try again
check_call(['docker', 'pull', docker_name])
docker_id = get_docker_id(docker_name)
builder.update('docker', docker_id)
# TODO(cmaloney): The environment variables should be generated during build
# not live in buildinfo.json.
pkginfo['environment'] = builder.take('environment')
# Whether pkgpanda should on the host make sure a `/var/lib` state directory is available
pkginfo['state_directory'] = builder.take('state_directory')
if pkginfo['state_directory'] not in [True, False]:
raise BuildError("state_directory in buildinfo.json must be a boolean `true` or `false`")
username = None
if builder.has('username'):
username = builder.take('username')
if not isinstance(username, str):
raise BuildError("username in buildinfo.json must be either not set (no user for this"
" package), or a user name string")
try:
pkgpanda.UserManagement.validate_username(username)
except ValidationError as ex:
raise BuildError("username in buildinfo.json didn't meet the validation rules. {}".format(ex))
pkginfo['username'] = username
group = None
if builder.has('group'):
group = builder.take('group')
if not isinstance(group, str):
raise BuildError("group in buildinfo.json must be either not set (use default group for this user)"
", or group must be a string")
try:
pkgpanda.UserManagement.validate_group_name(group)
except ValidationError as ex:
raise BuildError("group in buildinfo.json didn't meet the validation rules. {}".format(ex))
pkginfo['group'] = group
# Packages need directories inside the fake install root (otherwise docker
# will try making the directories on a readonly filesystem), so build the
# install root now, and make the package directories in it as we go.
install_dir = tempfile.mkdtemp(prefix="pkgpanda-")
active_packages = list()
active_package_ids = set()
active_package_variants = dict()
auto_deps = set()
# Final package has the same requires as the build.
requires = builder.take('requires')
pkginfo['requires'] = requires
if builder.has("sysctl"):
pkginfo["sysctl"] = builder.take("sysctl")
# TODO(cmaloney): Pull generating the full set of requires a function.
to_check = copy.deepcopy(requires)
if type(to_check) != list:
raise BuildError("`requires` in buildinfo.json must be an array of dependencies.")
while to_check:
requires_info = to_check.pop(0)
requires_name, requires_variant = expand_require(requires_info)
if requires_name in active_package_variants:
# TODO(cmaloney): If one package depends on the <default>
# variant of a package and 1+ others depends on a non-<default>
# variant then update the dependency to the non-default variant
# rather than erroring.
if requires_variant != active_package_variants[requires_name]:
# TODO(cmaloney): Make this contain the chains of
# dependencies which contain the conflicting packages.
# a -> b -> c -> d {foo}
# e {bar} -> d {baz}
raise BuildError(
"Dependncy on multiple variants of the same package {}. variants: {} {}".format(
requires_name,
requires_variant,
active_package_variants[requires_name]))
# The variant has package {requires_name, variant} already is a
# dependency, don't process it again / move on to the next.
continue
active_package_variants[requires_name] = requires_variant
# Figure out the last build of the dependency, add that as the
# fully expanded dependency.
requires_last_build = package_store.get_last_build_filename(requires_name, requires_variant)
if not os.path.exists(requires_last_build):
if recursive:
# Build the dependency
build(package_store, requires_name, requires_variant, clean_after_build, recursive)
else:
raise BuildError("No last build file found for dependency {} variant {}. Rebuild "
"the dependency".format(requires_name, requires_variant))
try:
pkg_id_str = load_string(requires_last_build)
auto_deps.add(pkg_id_str)
pkg_buildinfo = package_store.get_buildinfo(requires_name, requires_variant)
pkg_requires = pkg_buildinfo['requires']
pkg_path = repository.package_path(pkg_id_str)
pkg_tar = pkg_id_str + '.tar.xz'
if not os.path.exists(package_store.get_package_cache_folder(requires_name) + '/' + pkg_tar):
raise BuildError(
"The build tarball {} refered to by the last_build file of the dependency {} "
"variant {} doesn't exist. Rebuild the dependency.".format(
pkg_tar,
requires_name,
requires_variant))
active_package_ids.add(pkg_id_str)
# Mount the package into the docker container.
cmd.volumes[pkg_path] = "/opt/mesosphere/packages/{}:ro".format(pkg_id_str)
os.makedirs(os.path.join(install_dir, "packages/{}".format(pkg_id_str)))
# Add the dependencies of the package to the set which will be
# activated.
# TODO(cmaloney): All these 'transitive' dependencies shouldn't
# be available to the package being built, only what depends on
# them directly.
to_check += pkg_requires
except ValidationError as ex:
raise BuildError("validating package needed as dependency {0}: {1}".format(requires_name, ex)) from ex
except PackageError as ex:
raise BuildError("loading package needed as dependency {0}: {1}".format(requires_name, ex)) from ex
# Add requires to the package id, calculate the final package id.
# NOTE: active_packages isn't fully constructed here since we lazily load
# packages not already in the repository.
builder.update('requires', list(active_package_ids))
version_extra = None
if builder.has('version_extra'):
version_extra = builder.take('version_extra')
build_ids = builder.get_build_ids()
version_base = hash_checkout(build_ids)
version = None
if builder.has('version_extra'):
version = "{0}-{1}".format(version_extra, version_base)
else:
version = version_base
pkg_id = PackageId.from_parts(name, version)
# Everything must have been extracted by now. If it wasn't, then we just
# had a hard error that it was set but not used, as well as didn't include
# it in the caluclation of the PackageId.
builder = None
# Save the build_ids. Useful for verify exactly what went into the
# package build hash.
final_buildinfo['build_ids'] = build_ids
final_buildinfo['package_version'] = version
# Save the package name and variant. The variant is used when installing
# packages to validate dependencies.
final_buildinfo['name'] = name
final_buildinfo['variant'] = variant
# If the package is already built, don't do anything.
pkg_path = package_store.get_package_cache_folder(name) + '/{}.tar.xz'.format(pkg_id)
# Done if it exists locally
if exists(pkg_path):
print("Package up to date. Not re-building.")
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
write_string(package_store.get_last_build_filename(name, variant), str(pkg_id))
return pkg_path
# Try downloading.
dl_path = package_store.try_fetch_by_id(pkg_id)
if dl_path:
print("Package up to date. Not re-building. Downloaded from repository-url.")
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
write_string(package_store.get_last_build_filename(name, variant), str(pkg_id))
print(dl_path, pkg_path)
assert dl_path == pkg_path
return pkg_path
# Fall out and do the build since it couldn't be downloaded
print("Unable to download from cache. Proceeding to build")
print("Building package {} with buildinfo: {}".format(
pkg_id,
json.dumps(final_buildinfo, indent=2, sort_keys=True)))
# Clean out src, result so later steps can use them freely for building.
def clean():
# Run a docker container to remove src/ and result/
cmd = DockerCmd()
cmd.volumes = {
package_store.get_package_cache_folder(name): "/pkg/:rw",
}
cmd.container = "ubuntu:14.04.4"
cmd.run("package-cleaner", ["rm", "-rf", "/pkg/src", "/pkg/result"])
clean()
# Only fresh builds are allowed which don't overlap existing artifacts.
result_dir = cache_abs("result")
if exists(result_dir):
raise BuildError("result folder must not exist. It will be made when the package is "
"built. {}".format(result_dir))
# 'mkpanda add' all implicit dependencies since we actually need to build.
for dep in auto_deps:
print("Auto-adding dependency: {}".format(dep))
# NOTE: Not using the name pkg_id because that overrides the outer one.
id_obj = PackageId(dep)
add_package_file(repository, package_store.get_package_path(id_obj))
package = repository.load(dep)
active_packages.append(package)
# Checkout all the sources int their respective 'src/' folders.
try:
src_dir = cache_abs('src')
if os.path.exists(src_dir):
raise ValidationError(
"'src' directory already exists, did you have a previous build? " +
"Currently all builds must be from scratch. Support should be " +
"added for re-using a src directory when possible. src={}".format(src_dir))
os.mkdir(src_dir)
for src_name, fetcher in sorted(fetchers.items()):
root = cache_abs('src/' + src_name)
os.mkdir(root)
fetcher.checkout_to(root)
except ValidationError as ex:
raise BuildError("Validation error when fetching sources for package: {}".format(ex))
# Activate the packages so that we have a proper path, environment
# variables.
# TODO(cmaloney): RAII type thing for temproary directory so if we
# don't get all the way through things will be cleaned up?
install = Install(
root=install_dir,
config_dir=None,
rooted_systemd=True,
manage_systemd=False,
block_systemd=True,
fake_path=True,
manage_users=False,
manage_state_dir=False)
install.activate(active_packages)
# Rewrite all the symlinks inside the active path because we will
# be mounting the folder into a docker container, and the absolute
# paths to the packages will change.
# TODO(cmaloney): This isn't very clean, it would be much nicer to
# just run pkgpanda inside the package.
rewrite_symlinks(install_dir, repository.path, "/opt/mesosphere/packages/")
print("Building package in docker")
# TODO(cmaloney): Run as a specific non-root user, make it possible
# for non-root to cleanup afterwards.
# Run the build, prepping the environment as necessary.
mkdir(cache_abs("result"))
# Copy the build info to the resulting tarball
write_json(cache_abs("src/buildinfo.full.json"), final_buildinfo)
write_json(cache_abs("result/buildinfo.full.json"), final_buildinfo)
write_json(cache_abs("result/pkginfo.json"), pkginfo)
# Make the folder for the package we are building. If docker does it, it
# gets auto-created with root permissions and we can't actually delete it.
os.makedirs(os.path.join(install_dir, "packages", str(pkg_id)))
# TOOD(cmaloney): Disallow writing to well known files and directories?
# Source we checked out
cmd.volumes.update({
# TODO(cmaloney): src should be read only...
cache_abs("src"): "/pkg/src:rw",
# The build script
build_script: "/pkg/build:ro",
# Getting the result out
cache_abs("result"): "/opt/mesosphere/packages/{}:rw".format(pkg_id),
install_dir: "/opt/mesosphere:ro"
})
if os.path.exists(extra_dir):
cmd.volumes[extra_dir] = "/pkg/extra:ro"
cmd.environment = {
"PKG_VERSION": version,
"PKG_NAME": name,
"PKG_ID": pkg_id,
"PKG_PATH": "/opt/mesosphere/packages/{}".format(pkg_id),
"PKG_VARIANT": variant if variant is not None else "<default>",
"NUM_CORES": multiprocessing.cpu_count()
}
try:
# TODO(cmaloney): Run a wrapper which sources
# /opt/mesosphere/environment then runs a build. Also should fix
# ownership of /opt/mesosphere/packages/{pkg_id} post build.
cmd.run("package-builder", [
"/bin/bash",
"-o", "nounset",
"-o", "pipefail",
"-o", "errexit",
"/pkg/build"])
except CalledProcessError as ex:
raise BuildError("docker exited non-zero: {}\nCommand: {}".format(ex.returncode, ' '.join(ex.cmd)))
# Clean up the temporary install dir used for dependencies.
# TODO(cmaloney): Move to an RAII wrapper.
check_call(['rm', '-rf', install_dir])
with logger.scope("Build package tarball"):
# Check for forbidden services before packaging the tarball:
try:
check_forbidden_services(cache_abs("result"), RESERVED_UNIT_NAMES)
except ValidationError as ex:
raise BuildError("Package validation failed: {}".format(ex))
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
write_string(package_store.get_last_build_filename(name, variant), str(pkg_id))
# Bundle the artifacts into the pkgpanda package
tmp_name = pkg_path + "-tmp.tar.xz"
make_tar(tmp_name, cache_abs("result"))
os.rename(tmp_name, pkg_path)
print("Package built.")
if clean_after_build:
clean()
return pkg_path
def run_test(self) -> int:
stack_name = 'dcos-ci-test-upgrade-' + random_id(10)
test_id = uuid.uuid4().hex
healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id)
dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id)
with logger.scope("create vpc cf stack '{}'".format(stack_name)):
bw = test_util.aws.BotoWrapper(
region=self.aws_region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key)
ssh_key = bw.create_key_pair(stack_name)
write_string('ssh_key', ssh_key)
vpc, ssh_info = test_util.aws.VpcCfStack.create(
stack_name=stack_name,
instance_type='m4.xlarge',
instance_os='cent-os-7-dcos-prereqs',
# An instance for each cluster node plus the bootstrap.
instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name=stack_name,
boto_wrapper=bw
)
vpc.wait_for_complete()
cluster = test_util.cluster.Cluster.from_vpc(
vpc,
ssh_info,
ssh_key=ssh_key,
num_masters=self.num_masters,
num_agents=self.num_agents,
num_public_agents=self.num_public_agents,
)
with logger.scope("install dcos"):
# Use the CLI installer to set exhibitor_storage_backend = zookeeper.
# Don't install prereqs since stable breaks Docker 1.13. See
# https://jira.mesosphere.com/browse/DCOS_OSS-743.
test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False, install_prereqs=False,
add_config_path=self.config_yaml_override_install)
master_list = [h.private_ip for h in cluster.masters]
dcos_api_install = self.dcos_api_session_factory_install.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_install.wait_for_dcos()
installed_version = dcos_api_install.get_version()
healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id)
dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id)
viplisten_app = create_marathon_viplisten_app()
viptalk_app = create_marathon_viptalk_app()
self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app, viplisten_app, viptalk_app)
with logger.scope("upgrade cluster"):
test_util.cluster.upgrade_dcos(cluster, self.installer_url,
installed_version, add_config_path=self.config_yaml_override_upgrade)
with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel:
bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*'])
# this method invocation looks like it is the same as the one above, and that is partially correct.
# the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster
# the client is being created to interact with. This client is specifically for the cluster after the
# upgrade has taken place, and can account for any possible settings that may change for the client under
# the hood when it probes the cluster.
dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client
self.verify_apps_state(dcos_api_upgrade, dns_app)
with logger.scope("run integration tests"):
# copied from test_util/test_aws_cf.py:96
add_env = []
prefix = 'TEST_ADD_ENV_'
for k, v in os.environ.items():
if k.startswith(prefix):
add_env.append(k.replace(prefix, '') + '=' + v)
test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '')
result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd)
if result == 0:
self.log.info("Test successful! Deleting VPC if provided in this run.")
vpc.delete()
bw.delete_key_pair(stack_name)
else:
self.log.info("Test failed! VPC cluster will remain available for "
"debugging for 2 hour after instantiation.")
return result
def build_tree(package_store, mkbootstrap, tree_variant):
"""Build packages and bootstrap tarballs for one or all tree variants.
Returns a dict mapping tree variants to bootstrap IDs.
If tree_variant is None, builds all available tree variants.
"""
# TODO(cmaloney): Add support for circular dependencies. They are doable
# long as there is a pre-built version of enough of the packages.
# TODO(cmaloney): Make it so when we're building a treeinfo which has a
# explicit package list we don't build all the other packages.
build_order = list()
visited = set()
built = set()
def visit(pkg_tuple: tuple):
"""Add a package and its requires to the build order.
Raises AssertionError if pkg_tuple is in the set of visited packages.
If the package has any requires, they're recursively visited and added
to the build order depth-first. Then the package itself is added.
"""
# Visit the node for the first (and only) time.
assert pkg_tuple not in visited
visited.add(pkg_tuple)
# Ensure all dependencies are built. Sorted for stability
for require in sorted(package_store.packages[pkg_tuple]['requires']):
require_tuple = expand_require(require)
# If the dependency has already been built, we can move on.
if require_tuple in built:
continue
# If the dependency has not been built but has been visited, then
# there's a cycle in the dependency graph.
if require_tuple in visited:
raise BuildError("Circular dependency. Circular link {0} -> {1}".format(pkg_tuple, require_tuple))
if PackageId.is_id(require_tuple[0]):
raise BuildError("Depending on a specific package id is not supported. Package {} "
"depends on {}".format(pkg_tuple, require_tuple))
if require_tuple not in package_store.packages:
raise BuildError("Package {0} require {1} not buildable from tree.".format(pkg_tuple, require_tuple))
# Add the dependency (after its dependencies, if any) to the build
# order.
visit(require_tuple)
build_order.append(pkg_tuple)
built.add(pkg_tuple)
# Can't compare none to string, so expand none -> "true" / "false", then put
# the string in a field after "" if none, the string if not.
def key_func(elem):
return elem[0], elem[1] is None, elem[1] or ""
def visit_packages(package_tuples):
for pkg_tuple in sorted(package_tuples, key=key_func):
if pkg_tuple in visited:
continue
visit(pkg_tuple)
if tree_variant:
package_sets = [package_store.get_package_set(tree_variant)]
else:
package_sets = package_store.get_all_package_sets()
with logger.scope("resolve package graph"):
# Build all required packages for all tree variants.
for package_set in package_sets:
visit_packages(package_set.all_packages)
built_packages = dict()
for (name, variant) in build_order:
built_packages.setdefault(name, dict())
# Run the build, store the built package path for later use.
# TODO(cmaloney): Only build the requested variants, rather than all variants.
built_packages[name][variant] = build(
package_store,
name,
variant,
True)
# Build bootstrap tarballs for all tree variants.
def make_bootstrap(package_set):
with logger.scope("Making bootstrap variant: {}".format(pkgpanda.util.variant_name(package_set.variant))):
package_paths = list()
for name, pkg_variant in package_set.bootstrap_packages:
package_paths.append(built_packages[name][pkg_variant])
if mkbootstrap:
return make_bootstrap_tarball(
package_store,
list(sorted(package_paths)),
package_set.variant)
# Build bootstraps and and package lists for all variants.
# TODO(cmaloney): Allow distinguishing between "build all" and "build the default one".
complete_cache_dir = package_store.get_complete_cache_dir()
check_call(['mkdir', '-p', complete_cache_dir])
results = {}
for package_set in package_sets:
info = {
'bootstrap': make_bootstrap(package_set),
'packages': sorted(
load_string(package_store.get_last_build_filename(*pkg_tuple))
for pkg_tuple in package_set.all_packages)}
write_json(
complete_cache_dir + '/' + pkgpanda.util.variant_prefix(package_set.variant) + 'complete.latest.json',
info)
results[package_set.variant] = info
return results
def build(package_store: PackageStore, name: str, variant, clean_after_build, recursive=False):
msg = "Building package {} variant {}".format(name, pkgpanda.util.variant_name(variant))
with logger.scope(msg):
return _build(package_store, name, variant, clean_after_build, recursive)
def run_test(self) -> int:
stack_name = 'dcos-ci-test-upgrade-' + random_id(10)
test_id = uuid.uuid4().hex
healthcheck_app_id = TEST_APP_NAME_FMT.format('healthcheck-' + test_id)
dns_app_id = TEST_APP_NAME_FMT.format('dns-' + test_id)
with logger.scope("create vpc cf stack '{}'".format(stack_name)):
bw = test_util.aws.BotoWrapper(
region=self.aws_region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key)
ssh_key = bw.create_key_pair(stack_name)
write_string('ssh_key', ssh_key)
vpc, ssh_info = test_util.aws.VpcCfStack.create(
stack_name=stack_name,
instance_type='m4.xlarge',
instance_os='cent-os-7-dcos-prereqs',
# An instance for each cluster node plus the bootstrap.
instance_count=(self.num_masters + self.num_agents + self.num_public_agents + 1),
admin_location='0.0.0.0/0',
key_pair_name=stack_name,
boto_wrapper=bw
)
vpc.wait_for_complete()
cluster = test_util.cluster.Cluster.from_vpc(
vpc,
ssh_info,
ssh_key=ssh_key,
num_masters=self.num_masters,
num_agents=self.num_agents,
num_public_agents=self.num_public_agents,
)
with logger.scope("install dcos"):
# Use the CLI installer to set exhibitor_storage_backend = zookeeper.
test_util.cluster.install_dcos(cluster, self.stable_installer_url, api=False,
add_config_path=self.config_yaml_override_install)
master_list = [h.private_ip for h in cluster.masters]
dcos_api_install = self.dcos_api_session_factory_install.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_install.wait_for_dcos()
installed_version = dcos_api_install.get_version()
healthcheck_app = create_marathon_healthcheck_app(healthcheck_app_id)
dns_app = create_marathon_dns_app(dns_app_id, healthcheck_app_id)
self.setup_cluster_workload(dcos_api_install, healthcheck_app, dns_app)
with logger.scope("upgrade cluster"):
test_util.cluster.upgrade_dcos(cluster, self.installer_url,
installed_version, add_config_path=self.config_yaml_override_upgrade)
with cluster.ssher.tunnel(cluster.bootstrap_host) as bootstrap_host_tunnel:
bootstrap_host_tunnel.remote_cmd(['sudo', 'rm', '-rf', cluster.ssher.home_dir + '/*'])
# this method invocation looks like it is the same as the one above, and that is partially correct.
# the arguments to the invocation are the same, but the thing that changes is the lifecycle of the cluster
# the client is being created to interact with. This client is specifically for the cluster after the
# upgrade has taken place, and can account for any possible settings that may change for the client under
# the hood when it probes the cluster.
dcos_api_upgrade = self.dcos_api_session_factory_upgrade.apply(
'http://{ip}'.format(ip=cluster.masters[0].public_ip),
master_list,
master_list,
[h.private_ip for h in cluster.agents],
[h.private_ip for h in cluster.public_agents],
self.default_os_user)
dcos_api_upgrade.wait_for_dcos() # here we wait for DC/OS to be "up" so that we can auth this new client
self.verify_apps_state(dcos_api_upgrade, dns_app)
with logger.scope("run integration tests"):
# copied from test_util/test_aws_cf.py:96
add_env = []
prefix = 'TEST_ADD_ENV_'
for k, v in os.environ.items():
if k.startswith(prefix):
add_env.append(k.replace(prefix, '') + '=' + v)
test_cmd = ' '.join(add_env) + ' py.test -vv -s -rs ' + os.getenv('CI_FLAGS', '')
result = test_util.cluster.run_integration_tests(cluster, test_cmd=test_cmd)
if result == 0:
self.log.info("Test successful! Deleting VPC if provided in this run.")
vpc.delete()
bw.delete_key_pair(stack_name)
else:
self.log.info("Test failed! VPC cluster will remain available for "
"debugging for 2 hour after instantiation.")
return result
def make_channel_artifacts(metadata):
artifacts = []
# Set logging to debug so we get gen error messages, since those are
# logging.DEBUG currently to not show up when people are using `--genconf`
# and friends.
# TODO(cmaloney): Remove this and make the core bits of gen, code log at
# the proper info / warning / etc. level.
log = logging.getLogger()
original_log_level = log.getEffectiveLevel()
log.setLevel(logging.DEBUG)
provider_data = {}
providers = load_providers()
for name, module in sorted(providers.items()):
bootstrap_url = metadata['repository_url']
# If the particular provider has its own storage by the same name then
# Use the storage provider rather
if name in metadata['storage_urls']:
bootstrap_url = metadata['storage_urls'][name] + metadata['repository_path']
variant_arguments = dict()
for variant, variant_info in metadata['complete_dict'].items():
variant_arguments[variant] = copy.deepcopy({
'bootstrap_url': bootstrap_url,
'provider': name,
'bootstrap_id': variant_info['bootstrap'],
'bootstrap_variant': pkgpanda.util.variant_prefix(variant),
'package_ids': json.dumps(variant_info['packages'])
})
# Load additional default variant arguments out of gen_extra
if os.path.exists('gen_extra/calc.py'):
mod = importlib.machinery.SourceFileLoader('gen_extra.calc', 'gen_extra/calc.py').load_module()
variant_arguments[variant].update(mod.provider_template_defaults)
# Add templates for the default variant.
# Use keyword args to make not matching ordering a loud error around changes.
with logger.scope("Creating {} deploy tools".format(module.__name__)):
# TODO(cmaloney): Cleanup by just having this make and pass another source.
module_specific_variant_arguments = copy.deepcopy(variant_arguments)
for arg_dict in module_specific_variant_arguments.values():
if module.__name__ == 'gen.build_deploy.aws':
arg_dict['cloudformation_s3_url_full'] = metadata['cloudformation_s3_url_full']
elif module.__name__ == 'gen.build_deploy.azure':
arg_dict['azure_download_url'] = metadata['azure_download_url']
elif module.__name__ == 'gen.build_deploy.bash':
pass
else:
raise NotImplementedError("Unknown how to add args to deploy tool: {}".format(module.__name__))
for built_resource in module.do_create(
tag=metadata['tag'],
build_name=metadata['build_name'],
reproducible_artifact_path=metadata['reproducible_artifact_path'],
commit=metadata['commit'],
variant_arguments=module_specific_variant_arguments,
all_completes=metadata['all_completes']):
assert isinstance(built_resource, dict), built_resource
# Type switch
if 'packages' in built_resource:
for package in built_resource['packages']:
artifacts.append(get_gen_package_artifact(package))
else:
assert 'packages' not in built_resource
artifacts.append(built_resource)
# TODO(cmaloney): Check the provider artifacts adhere to the artifact template.
artifacts += provider_data.get('artifacts', list())
log.setLevel(original_log_level)
return artifacts
def make_channel_artifacts(metadata, provider_names):
artifacts = [{
'channel_path': 'version',
'local_content': DCOS_VERSION,
'content_type': 'text/plain; charset=utf-8',
}]
# Set logging to debug so we get gen error messages, since those are
# logging.DEBUG currently to not show up when people are using `--genconf`
# and friends.
# TODO(cmaloney): Remove this and make the core bits of gen, code log at
# the proper info / warning / etc. level.
log = logging.getLogger()
original_log_level = log.getEffectiveLevel()
log.setLevel(logging.DEBUG)
provider_data = {}
providers = load_providers(provider_names)
for name, module in sorted(providers.items()):
bootstrap_url = metadata['repository_url']
# If the particular provider has its own storage by the same name then
# Use the storage provider rather
if name in metadata['storage_urls']:
bootstrap_url = metadata['storage_urls'][name] + metadata[
'repository_path']
variant_arguments = dict()
for variant, variant_info in metadata['complete_dict'].items():
variant_arguments[variant] = copy.deepcopy({
'bootstrap_url':
bootstrap_url,
'provider':
name,
'bootstrap_id':
variant_info['bootstrap'],
'bootstrap_variant':
pkgpanda.util.variant_prefix(variant),
'package_ids':
json.dumps(variant_info['packages'])
})
# Load additional default variant arguments out of gen_extra
if os.path.exists('gen_extra/calc.py'):
mod = importlib.machinery.SourceFileLoader(
'gen_extra.calc', 'gen_extra/calc.py').load_module()
variant_arguments[variant].update(
mod.provider_template_defaults)
# Add templates for the default variant.
# Use keyword args to make not matching ordering a loud error around changes.
with logger.scope("Creating {} deploy tools".format(module.__name__)):
# TODO(cmaloney): Cleanup by just having this make and pass another source.
module_specific_variant_arguments = copy.deepcopy(
variant_arguments)
for arg_dict in module_specific_variant_arguments.values():
if module.__name__ == 'gen.build_deploy.aws':
arg_dict['cloudformation_s3_url_full'] = metadata[
'cloudformation_s3_url_full']
elif module.__name__ == 'gen.build_deploy.azure':
arg_dict['azure_download_url'] = metadata[
'azure_download_url']
elif module.__name__ == 'gen.build_deploy.bash':
pass
else:
raise NotImplementedError(
"Unknown how to add args to deploy tool: {}".format(
module.__name__))
for built_resource in module.do_create(
tag=metadata['tag'],
build_name=metadata['build_name'],
reproducible_artifact_path=metadata[
'reproducible_artifact_path'],
commit=metadata['commit'],
variant_arguments=module_specific_variant_arguments,
all_completes=metadata['all_completes']):
assert isinstance(built_resource, dict), built_resource
# Type switch
if 'packages' in built_resource:
for package in built_resource['packages']:
artifacts.append(get_gen_package_artifact(package))
else:
assert 'packages' not in built_resource
artifacts.append(built_resource)
# TODO(cmaloney): Check the provider artifacts adhere to the artifact template.
artifacts += provider_data.get('artifacts', list())
log.setLevel(original_log_level)
return artifacts
def _build(package_store, name, variant, clean_after_build, recursive):
assert isinstance(package_store, PackageStore)
tmpdir = tempfile.TemporaryDirectory(prefix="pkgpanda_repo")
repository = Repository(tmpdir.name)
package_dir = package_store.get_package_folder(name)
def src_abs(name):
return package_dir + '/' + name
def cache_abs(filename):
return package_store.get_package_cache_folder(name) + '/' + filename
# Build pkginfo over time, translating fields from buildinfo.
pkginfo = {}
# Build up the docker command arguments over time, translating fields as needed.
cmd = DockerCmd()
assert (name, variant) in package_store.packages, \
"Programming error: name, variant should have been validated to be valid before calling build()."
builder = IdBuilder(package_store.get_buildinfo(name, variant))
final_buildinfo = dict()
builder.add('name', name)
builder.add('variant', pkgpanda.util.variant_str(variant))
# Convert single_source -> sources
if builder.has('sources'):
if builder.has('single_source'):
raise BuildError('Both sources and single_source cannot be specified at the same time')
sources = builder.take('sources')
elif builder.has('single_source'):
sources = {name: builder.take('single_source')}
builder.replace('single_source', 'sources', sources)
else:
builder.add('sources', {})
sources = dict()
print("NOTICE: No sources specified")
final_buildinfo['sources'] = sources
# Construct the source fetchers, gather the checkout ids from them
checkout_ids = dict()
fetchers = dict()
try:
for src_name, src_info in sorted(sources.items()):
# TODO(cmaloney): Switch to a unified top level cache directory shared by all packages
cache_dir = package_store.get_package_cache_folder(name) + '/' + src_name
check_call(['mkdir', '-p', cache_dir])
fetcher = get_src_fetcher(src_info, cache_dir, package_dir)
fetchers[src_name] = fetcher
checkout_ids[src_name] = fetcher.get_id()
except ValidationError as ex:
raise BuildError("Validation error when fetching sources for package: {}".format(ex))
for src_name, checkout_id in checkout_ids.items():
# NOTE: single_source buildinfo was expanded above so the src_name is
# always correct here.
# Make sure we never accidentally overwrite something which might be
# important. Fields should match if specified (And that should be
# tested at some point). For now disallowing identical saves hassle.
assert_no_duplicate_keys(checkout_id, final_buildinfo['sources'][src_name])
final_buildinfo['sources'][src_name].update(checkout_id)
# Add the sha1 of the buildinfo.json + build file to the build ids
builder.update('sources', checkout_ids)
build_script = src_abs(builder.take('build_script'))
# TODO(cmaloney): Change dest name to build_script_sha1
builder.replace('build_script', 'build', pkgpanda.util.sha1(build_script))
builder.add('pkgpanda_version', pkgpanda.build.constants.version)
extra_dir = src_abs("extra")
# Add the "extra" folder inside the package as an additional source if it
# exists
if os.path.exists(extra_dir):
extra_id = hash_folder_abs(extra_dir, package_dir)
builder.add('extra_source', extra_id)
final_buildinfo['extra_source'] = extra_id
# Figure out the docker name.
docker_name = builder.take('docker')
cmd.container = docker_name
# Add the id of the docker build environment to the build_ids.
try:
docker_id = get_docker_id(docker_name)
except CalledProcessError:
# docker pull the container and try again
check_call(['docker', 'pull', docker_name])
docker_id = get_docker_id(docker_name)
builder.update('docker', docker_id)
# TODO(cmaloney): The environment variables should be generated during build
# not live in buildinfo.json.
pkginfo['environment'] = builder.take('environment')
# Whether pkgpanda should on the host make sure a `/var/lib` state directory is available
pkginfo['state_directory'] = builder.take('state_directory')
if pkginfo['state_directory'] not in [True, False]:
raise BuildError("state_directory in buildinfo.json must be a boolean `true` or `false`")
username = None
if builder.has('username'):
username = builder.take('username')
if not isinstance(username, str):
raise BuildError("username in buildinfo.json must be either not set (no user for this"
" package), or a user name string")
try:
pkgpanda.UserManagement.validate_username(username)
except ValidationError as ex:
raise BuildError("username in buildinfo.json didn't meet the validation rules. {}".format(ex))
pkginfo['username'] = username
group = None
if builder.has('group'):
group = builder.take('group')
if not isinstance(group, str):
raise BuildError("group in buildinfo.json must be either not set (use default group for this user)"
", or group must be a string")
try:
pkgpanda.UserManagement.validate_group_name(group)
except ValidationError as ex:
raise BuildError("group in buildinfo.json didn't meet the validation rules. {}".format(ex))
pkginfo['group'] = group
# Packages need directories inside the fake install root (otherwise docker
# will try making the directories on a readonly filesystem), so build the
# install root now, and make the package directories in it as we go.
install_dir = tempfile.mkdtemp(prefix="pkgpanda-")
active_packages = list()
active_package_ids = set()
active_package_variants = dict()
auto_deps = set()
# Final package has the same requires as the build.
requires = builder.take('requires')
pkginfo['requires'] = requires
if builder.has("sysctl"):
pkginfo["sysctl"] = builder.take("sysctl")
# TODO(cmaloney): Pull generating the full set of requires a function.
to_check = copy.deepcopy(requires)
if type(to_check) != list:
raise BuildError("`requires` in buildinfo.json must be an array of dependencies.")
while to_check:
requires_info = to_check.pop(0)
requires_name, requires_variant = expand_require(requires_info)
if requires_name in active_package_variants:
# TODO(cmaloney): If one package depends on the <default>
# variant of a package and 1+ others depends on a non-<default>
# variant then update the dependency to the non-default variant
# rather than erroring.
if requires_variant != active_package_variants[requires_name]:
# TODO(cmaloney): Make this contain the chains of
# dependencies which contain the conflicting packages.
# a -> b -> c -> d {foo}
# e {bar} -> d {baz}
raise BuildError(
"Dependncy on multiple variants of the same package {}. variants: {} {}".format(
requires_name,
requires_variant,
active_package_variants[requires_name]))
# The variant has package {requires_name, variant} already is a
# dependency, don't process it again / move on to the next.
continue
active_package_variants[requires_name] = requires_variant
# Figure out the last build of the dependency, add that as the
# fully expanded dependency.
requires_last_build = package_store.get_last_build_filename(requires_name, requires_variant)
if not os.path.exists(requires_last_build):
if recursive:
# Build the dependency
build(package_store, requires_name, requires_variant, clean_after_build, recursive)
else:
raise BuildError("No last build file found for dependency {} variant {}. Rebuild "
"the dependency".format(requires_name, requires_variant))
try:
pkg_id_str = load_string(requires_last_build)
auto_deps.add(pkg_id_str)
pkg_buildinfo = package_store.get_buildinfo(requires_name, requires_variant)
pkg_requires = pkg_buildinfo['requires']
pkg_path = repository.package_path(pkg_id_str)
pkg_tar = pkg_id_str + '.tar.xz'
if not os.path.exists(package_store.get_package_cache_folder(requires_name) + '/' + pkg_tar):
raise BuildError(
"The build tarball {} refered to by the last_build file of the dependency {} "
"variant {} doesn't exist. Rebuild the dependency.".format(
pkg_tar,
requires_name,
requires_variant))
active_package_ids.add(pkg_id_str)
# Mount the package into the docker container.
cmd.volumes[pkg_path] = "/opt/mesosphere/packages/{}:ro".format(pkg_id_str)
os.makedirs(os.path.join(install_dir, "packages/{}".format(pkg_id_str)))
# Add the dependencies of the package to the set which will be
# activated.
# TODO(cmaloney): All these 'transitive' dependencies shouldn't
# be available to the package being built, only what depends on
# them directly.
to_check += pkg_requires
except ValidationError as ex:
raise BuildError("validating package needed as dependency {0}: {1}".format(requires_name, ex)) from ex
except PackageError as ex:
raise BuildError("loading package needed as dependency {0}: {1}".format(requires_name, ex)) from ex
# Add requires to the package id, calculate the final package id.
# NOTE: active_packages isn't fully constructed here since we lazily load
# packages not already in the repository.
builder.update('requires', list(active_package_ids))
version_extra = None
if builder.has('version_extra'):
version_extra = builder.take('version_extra')
build_ids = builder.get_build_ids()
version_base = hash_checkout(build_ids)
version = None
if builder.has('version_extra'):
version = "{0}-{1}".format(version_extra, version_base)
else:
version = version_base
pkg_id = PackageId.from_parts(name, version)
# Everything must have been extracted by now. If it wasn't, then we just
# had a hard error that it was set but not used, as well as didn't include
# it in the caluclation of the PackageId.
builder = None
# Save the build_ids. Useful for verify exactly what went into the
# package build hash.
final_buildinfo['build_ids'] = build_ids
final_buildinfo['package_version'] = version
# Save the package name and variant. The variant is used when installing
# packages to validate dependencies.
final_buildinfo['name'] = name
final_buildinfo['variant'] = variant
# If the package is already built, don't do anything.
pkg_path = package_store.get_package_cache_folder(name) + '/{}.tar.xz'.format(pkg_id)
# Done if it exists locally
if exists(pkg_path):
print("Package up to date. Not re-building.")
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
write_string(package_store.get_last_build_filename(name, variant), str(pkg_id))
return pkg_path
# Try downloading.
dl_path = package_store.try_fetch_by_id(pkg_id)
if dl_path:
print("Package up to date. Not re-building. Downloaded from repository-url.")
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
write_string(package_store.get_last_build_filename(name, variant), str(pkg_id))
print(dl_path, pkg_path)
assert dl_path == pkg_path
return pkg_path
# Fall out and do the build since it couldn't be downloaded
print("Unable to download from cache. Proceeding to build")
print("Building package {} with buildinfo: {}".format(
pkg_id,
json.dumps(final_buildinfo, indent=2, sort_keys=True)))
# Clean out src, result so later steps can use them freely for building.
def clean():
# Run a docker container to remove src/ and result/
cmd = DockerCmd()
cmd.volumes = {
package_store.get_package_cache_folder(name): "/pkg/:rw",
}
cmd.container = "ubuntu:14.04.4"
cmd.run("package-cleaner", ["rm", "-rf", "/pkg/src", "/pkg/result"])
clean()
# Only fresh builds are allowed which don't overlap existing artifacts.
result_dir = cache_abs("result")
if exists(result_dir):
raise BuildError("result folder must not exist. It will be made when the package is "
"built. {}".format(result_dir))
# 'mkpanda add' all implicit dependencies since we actually need to build.
for dep in auto_deps:
print("Auto-adding dependency: {}".format(dep))
# NOTE: Not using the name pkg_id because that overrides the outer one.
id_obj = PackageId(dep)
add_package_file(repository, package_store.get_package_path(id_obj))
package = repository.load(dep)
active_packages.append(package)
# Checkout all the sources int their respective 'src/' folders.
try:
src_dir = cache_abs('src')
if os.path.exists(src_dir):
raise ValidationError(
"'src' directory already exists, did you have a previous build? " +
"Currently all builds must be from scratch. Support should be " +
"added for re-using a src directory when possible. src={}".format(src_dir))
os.mkdir(src_dir)
for src_name, fetcher in sorted(fetchers.items()):
root = cache_abs('src/' + src_name)
os.mkdir(root)
fetcher.checkout_to(root)
except ValidationError as ex:
raise BuildError("Validation error when fetching sources for package: {}".format(ex))
# Activate the packages so that we have a proper path, environment
# variables.
# TODO(cmaloney): RAII type thing for temproary directory so if we
# don't get all the way through things will be cleaned up?
install = Install(
root=install_dir,
config_dir=None,
rooted_systemd=True,
manage_systemd=False,
block_systemd=True,
fake_path=True,
manage_users=False,
manage_state_dir=False)
install.activate(active_packages)
# Rewrite all the symlinks inside the active path because we will
# be mounting the folder into a docker container, and the absolute
# paths to the packages will change.
# TODO(cmaloney): This isn't very clean, it would be much nicer to
# just run pkgpanda inside the package.
rewrite_symlinks(install_dir, repository.path, "/opt/mesosphere/packages/")
print("Building package in docker")
# TODO(cmaloney): Run as a specific non-root user, make it possible
# for non-root to cleanup afterwards.
# Run the build, prepping the environment as necessary.
mkdir(cache_abs("result"))
# Copy the build info to the resulting tarball
write_json(cache_abs("src/buildinfo.full.json"), final_buildinfo)
write_json(cache_abs("result/buildinfo.full.json"), final_buildinfo)
write_json(cache_abs("result/pkginfo.json"), pkginfo)
# Make the folder for the package we are building. If docker does it, it
# gets auto-created with root permissions and we can't actually delete it.
os.makedirs(os.path.join(install_dir, "packages", str(pkg_id)))
# TOOD(cmaloney): Disallow writing to well known files and directories?
# Source we checked out
cmd.volumes.update({
# TODO(cmaloney): src should be read only...
cache_abs("src"): "/pkg/src:rw",
# The build script
build_script: "/pkg/build:ro",
# Getting the result out
cache_abs("result"): "/opt/mesosphere/packages/{}:rw".format(pkg_id),
install_dir: "/opt/mesosphere:ro"
})
if os.path.exists(extra_dir):
cmd.volumes[extra_dir] = "/pkg/extra:ro"
cmd.environment = {
"PKG_VERSION": version,
"PKG_NAME": name,
"PKG_ID": pkg_id,
"PKG_PATH": "/opt/mesosphere/packages/{}".format(pkg_id),
"PKG_VARIANT": variant if variant is not None else "<default>",
"NUM_CORES": multiprocessing.cpu_count()
}
try:
# TODO(cmaloney): Run a wrapper which sources
# /opt/mesosphere/environment then runs a build. Also should fix
# ownership of /opt/mesosphere/packages/{pkg_id} post build.
cmd.run("package-builder", [
"/bin/bash",
"-o", "nounset",
"-o", "pipefail",
"-o", "errexit",
"/pkg/build"])
except CalledProcessError as ex:
raise BuildError("docker exited non-zero: {}\nCommand: {}".format(ex.returncode, ' '.join(ex.cmd)))
# Clean up the temporary install dir used for dependencies.
# TODO(cmaloney): Move to an RAII wrapper.
check_call(['rm', '-rf', install_dir])
with logger.scope("Build package tarball"):
# Check for forbidden services before packaging the tarball:
try:
check_forbidden_services(cache_abs("result"), RESERVED_UNIT_NAMES)
except ValidationError as ex:
raise BuildError("Package validation failed: {}".format(ex))
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
write_string(package_store.get_last_build_filename(name, variant), str(pkg_id))
# Bundle the artifacts into the pkgpanda package
tmp_name = pkg_path + "-tmp.tar.xz"
make_tar(tmp_name, cache_abs("result"))
os.rename(tmp_name, pkg_path)
print("Package built.")
if clean_after_build:
clean()
return pkg_path
def do_build_docker(name, path):
with logger.scope("dcos/dcos-builder ({})".format(name)):
return _do_build_docker(name, path)
def make_stable_artifacts(cache_repository_url, tree_variants):
metadata = {
"commit": util.dcos_image_commit,
"core_artifacts": [],
"packages": set()
}
# TODO(cmaloney): Rather than guessing / reverse-engineering all these paths
# have do_build_packages get them directly from pkgpanda
with logger.scope("Building packages"):
try:
all_completes = do_build_packages(cache_repository_url,
tree_variants)
except pkgpanda.build.BuildError as ex:
logger.error("Failure building package(s): {}".format(ex))
raise
# The installer and util are built bootstraps, but not a DC/OS variants. We use
# iteration over the complete_dict to enumerate all variants a whole lot,
# so explicity remove installer/util here so people don't accidentally hit it.
# TODO: make this into a tree option
complete_dict = dict()
for name, info in copy.copy(all_completes).items():
if name is not None and (name.endswith('installer')
or name.endswith('util')):
continue
complete_dict[name] = info
metadata["complete_dict"] = complete_dict
metadata["all_completes"] = all_completes
metadata["bootstrap_dict"] = {
k: v['bootstrap']
for k, v in complete_dict.items()
}
metadata["all_bootstraps"] = {
k: v['bootstrap']
for k, v in all_completes.items()
}
def add_file(info):
metadata["core_artifacts"].append(info)
def add_package(package_id):
if package_id in metadata['packages']:
return
metadata['packages'].add(package_id)
add_file(get_package_artifact(package_id))
# Add the bootstrap, active.json, packages as reproducible_path artifacts
# Add the <variant>.bootstrap.latest as a channel_path
for name, info in sorted(all_completes.items(),
key=lambda kv: pkgpanda.util.variant_str(kv[0])):
for file in make_bootstrap_artifacts(info['bootstrap'],
info['packages'], name,
'packages/cache'):
add_file(file)
# Add all the packages which haven't been added yet
for package_id in sorted(info['packages']):
add_package(package_id)
# Sets aren't json serializable, so transform to a list for future use.
metadata['packages'] = list(sorted(metadata['packages']))
return metadata
def verify_apps_state(self, dcos_api: DcosApiSession, dns_app: dict):
with logger.scope("verify apps state"):
# nested methods here so we can "close" over external state
def marathon_app_tasks_survive_upgrade():
# Verify that the tasks we started are still running.
tasks_end = {
app_id: sorted(self.app_task_ids(dcos_api, app_id))
for app_id in self.test_app_ids
}
self.log.debug('Test app tasks at end:\n' +
pprint.pformat(tasks_end))
if not self.tasks_start == tasks_end:
self.teamcity_msg.testFailed(
"test_upgrade_vpc.marathon_app_tasks_survive_upgrade",
details="expected: {}\nactual: {}".format(
self.tasks_start, tasks_end))
def test_mesos_task_state_remains_consistent():
# Verify that the "state" of the task does not change.
task_state_end = self.get_master_task_state(
dcos_api, self.tasks_start[0])
if not self.task_state_start == task_state_end:
self.teamcity_msg.testFailed(
"test_upgrade_vpc.test_mesos_task_state_remains_consistent",
details="expected: {}\nactual: {}".format(
self.task_state_start, task_state_end))
def test_app_dns_survive_upgrade():
# Verify DNS didn't fail.
marathon_framework_id = dcos_api.marathon.get(
'/v2/info').json()['frameworkId']
dns_app_task = dcos_api.marathon.get(
'/v2/apps' + dns_app['id'] + '/tasks').json()['tasks'][0]
dns_log = self.parse_dns_log(
dcos_api.mesos_sandbox_file(
dns_app_task['slaveId'],
marathon_framework_id,
dns_app_task['id'],
dns_app['env']['DNS_LOG_FILENAME'],
))
dns_failure_times = [
entry[0] for entry in dns_log if entry[1] != 'SUCCESS'
]
if len(dns_failure_times) > 0:
message = 'Failed to resolve Marathon app hostname {} at least once.'.format(
dns_app['env']['RESOLVE_NAME'])
err_msg = message + ' Hostname failed to resolve at these times:\n' + '\n'.join(
dns_failure_times)
self.log.debug(err_msg)
self.teamcity_msg.testFailed(
"test_upgrade_vpc.test_app_dns_survive_upgrade",
details=err_msg)
self.log_test(
"test_upgrade_vpc.marathon_app_tasks_survive_upgrade",
marathon_app_tasks_survive_upgrade)
self.log_test(
"test_upgrade_vpc.test_mesos_task_state_remains_consistent",
test_mesos_task_state_remains_consistent)
self.log_test("test_upgrade_vpc.test_app_dns_survive_upgrade",
test_app_dns_survive_upgrade)
def do_build_docker(name, path):
with logger.scope("dcos/dcos-builder ({})".format(name)):
return _do_build_docker(name, path)
def make_channel_artifacts(metadata):
artifacts = []
# Set logging to debug so we get gen error messages, since those are
# logging.DEBUG currently to not show up when people are using `--genconf`
# and friends.
# TODO(cmaloney): Remove this and make the core bits of gen, code log at
# the proper info / warning / etc. level.
log = logging.getLogger()
original_log_level = log.getEffectiveLevel()
log.setLevel(logging.DEBUG)
provider_data = {}
providers = load_providers()
for name, module in sorted(providers.items()):
bootstrap_url = metadata['repository_url']
# If the particular provider has its own storage by the same name then
# Use the storage provider rather
if name in metadata['storage_urls']:
bootstrap_url = metadata['storage_urls'][name] + metadata['repository_path']
variant_arguments = dict()
for bootstrap_name, bootstrap_id in metadata['bootstrap_dict'].items():
variant_arguments[bootstrap_name] = copy.deepcopy({
'bootstrap_url': bootstrap_url,
'provider': name,
'bootstrap_id': bootstrap_id,
'bootstrap_variant': pkgpanda.util.variant_prefix(bootstrap_name)
})
# Load additional default variant arguments out of gen_extra
if os.path.exists('gen_extra/calc.py'):
mod = importlib.machinery.SourceFileLoader('gen_extra.calc', 'gen_extra/calc.py').load_module()
variant_arguments[bootstrap_name].update(mod.provider_template_defaults)
# Add templates for the default variant.
# Use keyword args to make not matching ordering a loud error around changes.
with logger.scope("Creating {} deploy tools".format(module.__name__)):
for built_resource in module.do_create(
tag=metadata['tag'],
build_name=metadata['build_name'],
reproducible_artifact_path=metadata['reproducible_artifact_path'],
commit=metadata['commit'],
variant_arguments=variant_arguments,
all_bootstraps=metadata["all_bootstraps"]):
assert isinstance(built_resource, dict), built_resource
# Type switch
if 'packages' in built_resource:
for package in built_resource['packages']:
artifacts.append(get_gen_package_artifact(package))
else:
assert 'packages' not in built_resource
artifacts.append(built_resource)
# TODO(cmaloney): Check the provider artifacts adhere to the artifact template.
artifacts += provider_data.get('artifacts', list())
log.setLevel(original_log_level)
return artifacts
