def _get_package_list(package_list_id: str, repository_url: str) -> List[str]:
package_list_url = repository_url + '/package_lists/{}.package_list.json'.format(package_list_id)
with tempfile.NamedTemporaryFile() as f:
download(f.name, package_list_url, os.getcwd(), rm_on_error=False)
package_list = load_json(f.name)
if not isinstance(package_list, list):
raise ValidationError('{} should contain a JSON list of packages. Got a {}'.format(
package_list_url, type(package_list)
))
return package_list
def requests_fetcher(base_url, id_str, target, work_dir):
assert base_url
assert type(id_str) == str
id = PackageId(id_str)
# TODO(cmaloney): That file:// urls are allowed in base_url is likely a security hole.
# TODO(cmaloney): Switch to mesos-fetcher or aci or something so
# all the logic can go away, we gain integrity checking, etc.
base_url = base_url.rstrip('/')
url = base_url + "/packages/{0}/{1}.tar.xz".format(id.name, id_str)
# TODO(cmaloney): Use a private tmp directory so there is no chance of a user
# intercepting the tarball + other validation data locally.
with tempfile.NamedTemporaryFile(suffix=".tar.xz") as file:
download(file.name, url, work_dir)
extract_tarball(file.name, target)
def requests_fetcher(base_url, id_str, target, work_dir):
assert base_url
assert type(id_str) == str
id = PackageId(id_str)
# TODO(cmaloney): That file:// urls are allowed in base_url is likely a security hole.
# TODO(cmaloney): Switch to mesos-fetcher or aci or something so
# all the logic can go away, we gain integrity checking, etc.
base_url = base_url.rstrip('/')
url = base_url + "/packages/{0}/{1}.tar.xz".format(id.name, id_str)
# TODO(cmaloney): Use a private tmp directory so there is no chance of a user
# intercepting the tarball + other validation data locally.
with tempfile.NamedTemporaryFile(suffix=".tar.xz") as file:
download(file.name, url, work_dir, rm_on_error=False)
extract_tarball(file.name, target)
def checkout_to(self, directory):
# Download file to cache if it isn't already there
if not os.path.exists(self.cache_filename):
print("Downloading source tarball {}".format(self.url))
download(self.cache_filename, self.url, self.package_dir)
# Validate the sha1 of the source is given and matches the sha1
file_sha = sha1(self.cache_filename)
if self.sha != file_sha:
corrupt_filename = self.cache_filename + '.corrupt'
check_call(['mv', self.cache_filename, corrupt_filename])
raise ValidationError(
"Provided sha1 didn't match sha1 of downloaded file, corrupt download saved as {}. "
"Provided: {}, Download file's sha1: {}, Url: {}".format(
corrupt_filename, self.sha, file_sha, self.url))
if self.extract:
extract_archive(self.cache_filename, directory)
else:
# Copy the file(s) into src/
# TODO(cmaloney): Hardlink to save space?
shutil.copyfile(self.cache_filename, self._get_filename(directory))
def _do_bootstrap(install, repository):
# These files should be set by the environment which initially builds
# the host (cloud-init).
repository_url = if_exists(load_string, install.get_config_filename("setup-flags/repository-url"))
def fetcher(id, target):
if repository_url is None:
raise ValidationError("ERROR: Non-local package {} but no repository url given.".format(id))
return requests_fetcher(repository_url, id, target, os.getcwd())
setup_pkg_dir = install.get_config_filename("setup-packages")
if os.path.exists(setup_pkg_dir):
raise ValidationError(
"setup-packages is no longer supported. It's functionality has been replaced with late "
"binding packages. Found setup packages dir: {}".format(setup_pkg_dir))
setup_packages_to_activate = []
# If the host has late config values, build the late config package from them.
late_config = if_exists(load_yaml, install.get_config_filename("setup-flags/late-config.yaml"))
if late_config:
pkg_id_str = late_config['late_bound_package_id']
late_values = late_config['bound_values']
print("Binding late config to late package {}".format(pkg_id_str))
print("Bound values: {}".format(late_values))
if not PackageId.is_id(pkg_id_str):
raise ValidationError("Invalid late package id: {}".format(pkg_id_str))
pkg_id = PackageId(pkg_id_str)
if pkg_id.version != "setup":
raise ValidationError("Late package must have the version setup. Bad package: {}".format(pkg_id_str))
# Collect the late config package.
with tempfile.NamedTemporaryFile() as f:
download(
f.name,
repository_url + '/packages/{0}/{1}.dcos_config'.format(pkg_id.name, pkg_id_str),
os.getcwd(),
rm_on_error=False,
)
late_package = load_yaml(f.name)
# Resolve the late package using the bound late config values.
final_late_package = resolve_late_package(late_package, late_values)
# Render the package onto the filesystem and add it to the package
# repository.
with tempfile.NamedTemporaryFile() as f:
do_gen_package(final_late_package, f.name)
repository.add(lambda _, target: extract_tarball(f.name, target), pkg_id_str)
setup_packages_to_activate.append(pkg_id_str)
# If active.json is set on the host, use that as the set of packages to
# activate. Otherwise just use the set of currently active packages (those
# active in the bootstrap tarball)
to_activate = None
active_path = install.get_config_filename("setup-flags/active.json")
if os.path.exists(active_path):
print("Loaded active packages from", active_path)
to_activate = load_json(active_path)
# Ensure all packages are local
print("Ensuring all packages in active set {} are local".format(",".join(to_activate)))
for package in to_activate:
repository.add(fetcher, package)
else:
print("Calculated active packages from bootstrap tarball")
to_activate = list(install.get_active())
package_list_filename = install.get_config_filename("setup-flags/cluster-package-list")
print("Checking for cluster packages in:", package_list_filename)
package_list_id = if_exists(load_string, package_list_filename)
if package_list_id:
print("Cluster package list:", package_list_id)
cluster_packages = _get_package_list(package_list_id, repository_url)
print("Loading cluster-packages: {}".format(cluster_packages))
for package_id_str in cluster_packages:
# Validate the package ids
pkg_id = PackageId(package_id_str)
# Fetch the packages if not local
if not repository.has_package(package_id_str):
repository.add(fetcher, package_id_str)
# Add the package to the set to activate
setup_packages_to_activate.append(package_id_str)
else:
print("No cluster-packages specified")
# Calculate the full set of final packages (Explicit activations + setup packages).
# De-duplicate using a set.
to_activate = list(set(to_activate + setup_packages_to_activate))
print("Activating packages")
install.activate(repository.load_packages(to_activate))
def build(variant, package_dir, name, repository_url, clean_after_build):
print("Building package {} variant {}".format(name, variant
or "<default>"))
tmpdir = tempfile.TemporaryDirectory(prefix="pkgpanda_repo")
repository = Repository(tmpdir.name)
def pkg_abs(name):
return package_dir + '/' + name
# Build pkginfo over time, translating fields from buildinfo.
pkginfo = {}
# Build up the docker command arguments over time, translating fields as needed.
cmd = DockerCmd()
buildinfo = load_buildinfo(package_dir, variant)
if 'name' in buildinfo:
raise BuildError(
"'name' is not allowed in buildinfo.json, it is implicitly the name of the "
"folder containing the buildinfo.json")
# Make sure build_script is only set on variants
if 'build_script' in buildinfo and variant is None:
raise BuildError("build_script can only be set on package variants")
# Convert single_source -> sources
try:
sources = expand_single_source_alias(name, buildinfo)
except ValidationError as ex:
raise BuildError(
"Invalid buildinfo.json for package: {}".format(ex)) from ex
# Save the final sources back into buildinfo so it gets written into
# buildinfo.json. This also means buildinfo.json is always expanded form.
buildinfo['sources'] = sources
# Construct the source fetchers, gather the checkout ids from them
checkout_ids = dict()
fetchers = dict()
try:
for src_name, src_info in sorted(sources.items()):
if src_info[
'kind'] not in pkgpanda.build.src_fetchers.all_fetchers:
raise ValidationError(
"No known way to catch src with kind '{}'. Known kinds: {}"
.format(src_info['kind'],
pkgpanda.src_fetchers.all_fetchers.keys()))
cache_dir = pkg_abs("cache")
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
fetchers[src_name] = pkgpanda.build.src_fetchers.all_fetchers[
src_info['kind']](src_name, src_info, package_dir)
checkout_ids[src_name] = fetchers[src_name].get_id()
except ValidationError as ex:
raise BuildError(
"Validation error when fetching sources for package: {}".format(
ex))
for src_name, checkout_id in checkout_ids.items():
# NOTE: single_source buildinfo was expanded above so the src_name is
# always correct here.
# Make sure we never accidentally overwrite something which might be
# important. Fields should match if specified (And that should be
# tested at some point). For now disallowing identical saves hassle.
assert_no_duplicate_keys(checkout_id, buildinfo['sources'][src_name])
buildinfo['sources'][src_name].update(checkout_id)
# Add the sha1sum of the buildinfo.json + build file to the build ids
build_ids = {"sources": checkout_ids}
build_ids['build'] = pkgpanda.util.sha1(pkg_abs("build"))
build_ids['pkgpanda_version'] = pkgpanda.build.constants.version
build_ids['variant'] = '' if variant is None else variant
extra_dir = pkg_abs("extra")
# Add the "extra" folder inside the package as an additional source if it
# exists
if os.path.exists(extra_dir):
extra_id = hash_folder(extra_dir)
build_ids['extra_source'] = extra_id
buildinfo['extra_source'] = extra_id
# Figure out the docker name.
docker_name = buildinfo.get('docker', 'dcos-builder:latest')
cmd.container = docker_name
# Add the id of the docker build environment to the build_ids.
try:
docker_id = get_docker_id(docker_name)
except CalledProcessError:
# docker pull the container and try again
check_call(['docker', 'pull', docker_name])
docker_id = get_docker_id(docker_name)
build_ids['docker'] = docker_id
# TODO(cmaloney): The environment variables should be generated during build
# not live in buildinfo.json.
build_ids['environment'] = buildinfo.get('environment', {})
# Packages need directories inside the fake install root (otherwise docker
# will try making the directories on a readonly filesystem), so build the
# install root now, and make the package directories in it as we go.
install_dir = tempfile.mkdtemp(prefix="pkgpanda-")
active_packages = list()
active_package_ids = set()
active_package_variants = dict()
auto_deps = set()
# Verify all requires are in the repository.
if 'requires' in buildinfo:
# Final package has the same requires as the build.
pkginfo['requires'] = buildinfo['requires']
# TODO(cmaloney): Pull generating the full set of requires a function.
to_check = copy.deepcopy(buildinfo['requires'])
if type(to_check) != list:
raise BuildError(
"`requires` in buildinfo.json must be an array of dependencies."
)
while to_check:
requires_info = to_check.pop(0)
requires_name, requires_variant = expand_require(requires_info)
if requires_name in active_package_variants:
# TODO(cmaloney): If one package depends on the <default>
# variant of a package and 1+ others depends on a non-<default>
# variant then update the dependency to the non-default variant
# rather than erroring.
if requires_variant != active_package_variants[requires_name]:
# TODO(cmaloney): Make this contain the chains of
# dependencies which contain the conflicting packages.
# a -> b -> c -> d {foo}
# e {bar} -> d {baz}
raise BuildError(
"Dependncy on multiple variants of the same package {}. "
"variants: {} {}".format(
requires_name, requires_variant,
active_package_variants[requires_name]))
# The variant has package {requires_name, variant} already is a
# dependency, don't process it again / move on to the next.
continue
active_package_variants[requires_name] = requires_variant
# Figure out the last build of the dependency, add that as the
# fully expanded dependency.
require_package_dir = os.path.normpath(
pkg_abs('../' + requires_name))
last_build = require_package_dir + '/' + last_build_filename(
requires_variant)
if not os.path.exists(last_build):
raise BuildError(
"No last build file found for dependency {} variant {}. Rebuild "
"the dependency".format(requires_name, requires_variant))
try:
pkg_id_str = load_string(last_build)
auto_deps.add(pkg_id_str)
pkg_buildinfo = load_buildinfo(require_package_dir,
requires_variant)
pkg_requires = pkg_buildinfo.get('requires', list())
pkg_path = repository.package_path(pkg_id_str)
pkg_tar = pkg_id_str + '.tar.xz'
if not os.path.exists(require_package_dir + '/' + pkg_tar):
raise BuildError(
"The build tarball {} refered to by the last_build file of the "
"dependency {} variant {} doesn't exist. Rebuild the dependency."
.format(pkg_tar, requires_name, requires_variant))
active_package_ids.add(pkg_id_str)
# Mount the package into the docker container.
cmd.volumes[
pkg_path] = "/opt/mesosphere/packages/{}:ro".format(
pkg_id_str)
os.makedirs(
os.path.join(install_dir,
"packages/{}".format(pkg_id_str)))
# Add the dependencies of the package to the set which will be
# activated.
# TODO(cmaloney): All these 'transitive' dependencies shouldn't
# be available to the package being built, only what depends on
# them directly.
to_check += pkg_requires
except ValidationError as ex:
raise BuildError(
"validating package needed as dependency {0}: {1}".format(
requires_name, ex)) from ex
except PackageError as ex:
raise BuildError(
"loading package needed as dependency {0}: {1}".format(
requires_name, ex)) from ex
# Add requires to the package id, calculate the final package id.
# NOTE: active_packages isn't fully constructed here since we lazily load
# packages not already in the repository.
build_ids['requires'] = list(active_package_ids)
version_base = hash_checkout(build_ids)
version = None
if "version_extra" in buildinfo:
version = "{0}-{1}".format(buildinfo["version_extra"], version_base)
else:
version = version_base
pkg_id = PackageId.from_parts(name, version)
# Save the build_ids. Useful for verify exactly what went into the
# package build hash.
buildinfo['build_ids'] = build_ids
buildinfo['package_version'] = version
# Save the package name and variant. The variant is used when installing
# packages to validate dependencies.
buildinfo['name'] = name
buildinfo['variant'] = variant
# If the package is already built, don't do anything.
pkg_path = pkg_abs("{}.tar.xz".format(pkg_id))
# Done if it exists locally
if exists(pkg_path):
print("Package up to date. Not re-building.")
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
check_call(["mkdir", "-p", pkg_abs("cache")])
write_string(pkg_abs(last_build_filename(variant)), str(pkg_id))
return pkg_path
# Try downloading.
if repository_url:
tmp_filename = pkg_path + '.tmp'
try:
# Normalize to no trailing slash for repository_url
repository_url = repository_url.rstrip('/')
url = repository_url + '/packages/{0}/{1}.tar.xz'.format(
pkg_id.name, str(pkg_id))
print("Attempting to download", pkg_id, "from", url)
download(tmp_filename, url, package_dir)
os.rename(tmp_filename, pkg_path)
print(
"Package up to date. Not re-building. Downloaded from repository-url."
)
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
check_call(["mkdir", "-p", pkg_abs("cache")])
write_string(pkg_abs(last_build_filename(variant)), str(pkg_id))
return pkg_path
except FetchError:
try:
os.remove(tmp_filename)
except:
pass
# Fall out and do the build since the command errored.
print("Unable to download from cache. Proceeding to build")
print("Building package {} with buildinfo: {}".format(
pkg_id, json.dumps(buildinfo, indent=2, sort_keys=True)))
# Clean out src, result so later steps can use them freely for building.
clean(package_dir)
# Only fresh builds are allowed which don't overlap existing artifacts.
result_dir = pkg_abs("result")
if exists(result_dir):
raise BuildError(
"result folder must not exist. It will be made when the package is "
"built. {}".format(result_dir))
# 'mkpanda add' all implicit dependencies since we actually need to build.
for dep in auto_deps:
print("Auto-adding dependency: {}".format(dep))
# NOTE: Not using the name pkg_id because that overrides the outer one.
id_obj = PackageId(dep)
add_to_repository(
repository, pkg_abs('../{0}/{1}.tar.xz'.format(id_obj.name, dep)))
package = repository.load(dep)
active_packages.append(package)
# Checkout all the sources int their respective 'src/' folders.
try:
src_dir = pkg_abs('src')
if os.path.exists(src_dir):
raise ValidationError(
"'src' directory already exists, did you have a previous build? "
+
"Currently all builds must be from scratch. Support should be "
+ "added for re-using a src directory when possible. src={}".
format(src_dir))
os.mkdir(src_dir)
for src_name, fetcher in sorted(fetchers.items()):
root = pkg_abs('src/' + src_name)
os.mkdir(root)
fetcher.checkout_to(root)
except ValidationError as ex:
raise BuildError(
"Validation error when fetching sources for package: {}".format(
ex))
# Copy over environment settings
if 'environment' in buildinfo:
pkginfo['environment'] = buildinfo['environment']
# Activate the packages so that we have a proper path, environment
# variables.
# TODO(cmaloney): RAII type thing for temproary directory so if we
# don't get all the way through things will be cleaned up?
install = Install(install_dir, None, True, False, True, True)
install.activate(active_packages)
# Rewrite all the symlinks inside the active path because we will
# be mounting the folder into a docker container, and the absolute
# paths to the packages will change.
# TODO(cmaloney): This isn't very clean, it would be much nicer to
# just run pkgpanda inside the package.
rewrite_symlinks(install_dir, repository.path, "/opt/mesosphere/packages/")
print("Building package in docker")
# TODO(cmaloney): Run as a specific non-root user, make it possible
# for non-root to cleanup afterwards.
# Run the build, prepping the environment as necessary.
mkdir(pkg_abs("result"))
# Copy the build info to the resulting tarball
write_json(pkg_abs("src/buildinfo.full.json"), buildinfo)
write_json(pkg_abs("result/buildinfo.full.json"), buildinfo)
write_json(pkg_abs("result/pkginfo.json"), pkginfo)
# Make the folder for the package we are building. If docker does it, it
# gets auto-created with root permissions and we can't actually delete it.
os.makedirs(os.path.join(install_dir, "packages", str(pkg_id)))
# TOOD(cmaloney): Disallow writing to well known files and directories?
# Source we checked out
cmd.volumes.update({
# TODO(cmaloney): src should be read only...
pkg_abs("src"):
"/pkg/src:rw",
# The build script
pkg_abs(buildinfo.get('build_script', 'build')):
"/pkg/build:ro",
# Getting the result out
pkg_abs("result"):
"/opt/mesosphere/packages/{}:rw".format(pkg_id),
install_dir:
"/opt/mesosphere:ro"
})
if os.path.exists(extra_dir):
cmd.volumes[extra_dir] = "/pkg/extra:ro"
cmd.environment = {
"PKG_VERSION": version,
"PKG_NAME": name,
"PKG_ID": pkg_id,
"PKG_PATH": "/opt/mesosphere/packages/{}".format(pkg_id),
"PKG_VARIANT": variant if variant is not None else "<default>"
}
try:
# TODO(cmaloney): Run a wrapper which sources
# /opt/mesosphere/environment then runs a build. Also should fix
# ownership of /opt/mesosphere/packages/{pkg_id} post build.
cmd.run([
"/bin/bash", "-o", "nounset", "-o", "pipefail", "-o", "errexit",
"/pkg/build"
])
except CalledProcessError as ex:
raise BuildError("docker exited non-zero: {}\nCommand: {}".format(
ex.returncode, ' '.join(ex.cmd)))
# Clean up the temporary install dir used for dependencies.
# TODO(cmaloney): Move to an RAII wrapper.
check_call(['rm', '-rf', install_dir])
print("Building package tarball")
# Check for forbidden services before packaging the tarball:
try:
check_forbidden_services(pkg_abs("result"), RESERVED_UNIT_NAMES)
except ValidationError as ex:
raise BuildError("Package validation failed: {}".format(ex))
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
check_call(["mkdir", "-p", pkg_abs("cache")])
write_string(pkg_abs(last_build_filename(variant)), str(pkg_id))
# Bundle the artifacts into the pkgpanda package
tmp_name = pkg_path + "-tmp.tar.xz"
make_tar(tmp_name, pkg_abs("result"))
os.rename(tmp_name, pkg_path)
print("Package built.")
if clean_after_build:
clean(package_dir)
return pkg_path
def make_bootstrap_tarball(packages_dir, packages, variant, repository_url):
# Convert filenames to package ids
pkg_ids = list()
for pkg_path in packages:
# Get the package id from the given package path
filename = os.path.basename(pkg_path)
if not filename.endswith(".tar.xz"):
raise BuildError(
"Packages must be packaged / end with a .tar.xz. Got {}".
format(filename))
pkg_id = filename[:-len(".tar.xz")]
pkg_ids.append(pkg_id)
# Filename is output_name.<sha-1>.{active.json|.bootstrap.tar.xz}
bootstrap_id = hash_checkout(pkg_ids)
latest_name = "{}/{}bootstrap.latest".format(
packages_dir, pkgpanda.util.variant_prefix(variant))
output_name = packages_dir + '/' + bootstrap_id + '.'
# bootstrap tarball = <sha1 of packages in tarball>.bootstrap.tar.xz
bootstrap_name = "{}bootstrap.tar.xz".format(output_name)
active_name = "{}active.json".format(output_name)
def mark_latest():
# Ensure latest is always written
write_string(latest_name, bootstrap_id)
print("bootstrap: {}".format(bootstrap_name))
print("active: {}".format(active_name))
print("latest: {}".format(latest_name))
return bootstrap_name
if (os.path.exists(bootstrap_name)):
print("Bootstrap already up to date, not recreating")
return mark_latest()
# Try downloading.
if repository_url:
tmp_bootstrap = bootstrap_name + '.tmp'
tmp_active = active_name + '.tmp'
try:
repository_url = repository_url.rstrip('/')
bootstrap_url = repository_url + '/bootstrap/{}.bootstrap.tar.xz'.format(
bootstrap_id)
active_url = repository_url + '/bootstrap/{}.active.json'.format(
bootstrap_id)
print("Attempting to download", bootstrap_name, "from",
bootstrap_url)
# Normalize to no trailing slash for repository_url
download(tmp_bootstrap, bootstrap_url, packages_dir)
print("Attempting to download", active_name, "from", active_url)
download(tmp_active, active_url, packages_dir)
# Move into place
os.rename(tmp_bootstrap, bootstrap_name)
os.rename(tmp_active, active_name)
print(
"Bootstrap already up to date, Not recreating. Downloaded from repository-url."
)
return mark_latest()
except FetchError:
try:
os.remove(tmp_bootstrap)
except:
pass
try:
os.remove(tmp_active)
except:
pass
# Fall out and do the build since the command errored.
print("Unable to download from cache. Building.")
print("Creating bootstrap tarball for variant {}".format(variant))
work_dir = tempfile.mkdtemp(prefix='mkpanda_bootstrap_tmp')
def make_abs(path):
return os.path.join(work_dir, path)
pkgpanda_root = make_abs("opt/mesosphere")
repository = Repository(os.path.join(pkgpanda_root, "packages"))
# Fetch all the packages to the root
for pkg_path in packages:
filename = os.path.basename(pkg_path)
pkg_id = filename[:-len(".tar.xz")]
def local_fetcher(id, target):
shutil.unpack_archive(pkg_path, target, "gztar")
repository.add(local_fetcher, pkg_id, False)
# Activate the packages inside the repository.
# Do generate dcos.target.wants inside the root so that we don't
# try messing with /etc/systemd/system.
install = Install(pkgpanda_root, None, True, False, True, True, True)
install.activate(repository.load_packages(pkg_ids))
# Mark the tarball as a bootstrap tarball/filesystem so that
# dcos-setup.service will fire.
make_file(make_abs("opt/mesosphere/bootstrap"))
# Write out an active.json for the bootstrap tarball
write_json(active_name, pkg_ids)
# Rewrite all the symlinks to point to /opt/mesosphere
rewrite_symlinks(work_dir, work_dir, "/")
make_tar(bootstrap_name, pkgpanda_root)
shutil.rmtree(work_dir)
# Update latest last so that we don't ever use partially-built things.
write_string(latest_name, bootstrap_id)
print("Built bootstrap")
return mark_latest()
def build(variant, package_dir, name, repository_url, clean_after_build):
print("Building package {} variant {}".format(name, variant or "<default>"))
tmpdir = tempfile.TemporaryDirectory(prefix="pkgpanda_repo")
repository = Repository(tmpdir.name)
def pkg_abs(name):
return package_dir + '/' + name
# Build pkginfo over time, translating fields from buildinfo.
pkginfo = {}
# Build up the docker command arguments over time, translating fields as needed.
cmd = DockerCmd()
buildinfo = load_buildinfo(package_dir, variant)
if 'name' in buildinfo:
raise BuildError("'name' is not allowed in buildinfo.json, it is implicitly the name of the "
"folder containing the buildinfo.json")
# Make sure build_script is only set on variants
if 'build_script' in buildinfo and variant is None:
raise BuildError("build_script can only be set on package variants")
# Convert single_source -> sources
try:
sources = expand_single_source_alias(name, buildinfo)
except ValidationError as ex:
raise BuildError("Invalid buildinfo.json for package: {}".format(ex)) from ex
# Save the final sources back into buildinfo so it gets written into
# buildinfo.json. This also means buildinfo.json is always expanded form.
buildinfo['sources'] = sources
# Construct the source fetchers, gather the checkout ids from them
checkout_ids = dict()
fetchers = dict()
try:
for src_name, src_info in sorted(sources.items()):
if src_info['kind'] not in pkgpanda.build.src_fetchers.all_fetchers:
raise ValidationError("No known way to catch src with kind '{}'. Known kinds: {}".format(
src_info['kind'],
pkgpanda.src_fetchers.all_fetchers.keys()))
cache_dir = pkg_abs("cache")
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
fetchers[src_name] = pkgpanda.build.src_fetchers.all_fetchers[src_info['kind']](src_name,
src_info,
package_dir)
checkout_ids[src_name] = fetchers[src_name].get_id()
except ValidationError as ex:
raise BuildError("Validation error when fetching sources for package: {}".format(ex))
for src_name, checkout_id in checkout_ids.items():
# NOTE: single_source buildinfo was expanded above so the src_name is
# always correct here.
# Make sure we never accidentally overwrite something which might be
# important. Fields should match if specified (And that should be
# tested at some point). For now disallowing identical saves hassle.
assert_no_duplicate_keys(checkout_id, buildinfo['sources'][src_name])
buildinfo['sources'][src_name].update(checkout_id)
# Add the sha1sum of the buildinfo.json + build file to the build ids
build_ids = {"sources": checkout_ids}
build_ids['build'] = pkgpanda.util.sha1(pkg_abs("build"))
build_ids['pkgpanda_version'] = pkgpanda.build.constants.version
build_ids['variant'] = '' if variant is None else variant
extra_dir = pkg_abs("extra")
# Add the "extra" folder inside the package as an additional source if it
# exists
if os.path.exists(extra_dir):
extra_id = hash_folder(extra_dir)
build_ids['extra_source'] = extra_id
buildinfo['extra_source'] = extra_id
# Figure out the docker name.
docker_name = buildinfo.get('docker', 'dcos-builder:latest')
cmd.container = docker_name
# Add the id of the docker build environment to the build_ids.
try:
docker_id = get_docker_id(docker_name)
except CalledProcessError:
# docker pull the container and try again
check_call(['docker', 'pull', docker_name])
docker_id = get_docker_id(docker_name)
build_ids['docker'] = docker_id
# TODO(cmaloney): The environment variables should be generated during build
# not live in buildinfo.json.
build_ids['environment'] = buildinfo.get('environment', {})
# Packages need directories inside the fake install root (otherwise docker
# will try making the directories on a readonly filesystem), so build the
# install root now, and make the package directories in it as we go.
install_dir = tempfile.mkdtemp(prefix="pkgpanda-")
active_packages = list()
active_package_ids = set()
active_package_variants = dict()
auto_deps = set()
# Verify all requires are in the repository.
if 'requires' in buildinfo:
# Final package has the same requires as the build.
pkginfo['requires'] = buildinfo['requires']
# TODO(cmaloney): Pull generating the full set of requires a function.
to_check = copy.deepcopy(buildinfo['requires'])
if type(to_check) != list:
raise BuildError("`requires` in buildinfo.json must be an array of dependencies.")
while to_check:
requires_info = to_check.pop(0)
requires_name, requires_variant = expand_require(requires_info)
if requires_name in active_package_variants:
# TODO(cmaloney): If one package depends on the <default>
# variant of a package and 1+ others depends on a non-<default>
# variant then update the dependency to the non-default variant
# rather than erroring.
if requires_variant != active_package_variants[requires_name]:
# TODO(cmaloney): Make this contain the chains of
# dependencies which contain the conflicting packages.
# a -> b -> c -> d {foo}
# e {bar} -> d {baz}
raise BuildError("Dependncy on multiple variants of the same package {}. "
"variants: {} {}".format(
requires_name,
requires_variant,
active_package_variants[requires_name]))
# The variant has package {requires_name, variant} already is a
# dependency, don't process it again / move on to the next.
continue
active_package_variants[requires_name] = requires_variant
# Figure out the last build of the dependency, add that as the
# fully expanded dependency.
require_package_dir = os.path.normpath(pkg_abs('../' + requires_name))
last_build = require_package_dir + '/' + last_build_filename(requires_variant)
if not os.path.exists(last_build):
raise BuildError("No last build file found for dependency {} variant {}. Rebuild "
"the dependency".format(requires_name, requires_variant))
try:
pkg_id_str = load_string(last_build)
auto_deps.add(pkg_id_str)
pkg_buildinfo = load_buildinfo(require_package_dir, requires_variant)
pkg_requires = pkg_buildinfo.get('requires', list())
pkg_path = repository.package_path(pkg_id_str)
pkg_tar = pkg_id_str + '.tar.xz'
if not os.path.exists(require_package_dir + '/' + pkg_tar):
raise BuildError("The build tarball {} refered to by the last_build file of the "
"dependency {} variant {} doesn't exist. Rebuild the dependency.".format(
pkg_tar,
requires_name,
requires_variant))
active_package_ids.add(pkg_id_str)
# Mount the package into the docker container.
cmd.volumes[pkg_path] = "/opt/mesosphere/packages/{}:ro".format(pkg_id_str)
os.makedirs(os.path.join(install_dir, "packages/{}".format(pkg_id_str)))
# Add the dependencies of the package to the set which will be
# activated.
# TODO(cmaloney): All these 'transitive' dependencies shouldn't
# be available to the package being built, only what depends on
# them directly.
to_check += pkg_requires
except ValidationError as ex:
raise BuildError("validating package needed as dependency {0}: {1}".format(requires_name, ex)) from ex
except PackageError as ex:
raise BuildError("loading package needed as dependency {0}: {1}".format(requires_name, ex)) from ex
# Add requires to the package id, calculate the final package id.
# NOTE: active_packages isn't fully constructed here since we lazily load
# packages not already in the repository.
build_ids['requires'] = list(active_package_ids)
version_base = hash_checkout(build_ids)
version = None
if "version_extra" in buildinfo:
version = "{0}-{1}".format(buildinfo["version_extra"], version_base)
else:
version = version_base
pkg_id = PackageId.from_parts(name, version)
# Save the build_ids. Useful for verify exactly what went into the
# package build hash.
buildinfo['build_ids'] = build_ids
buildinfo['package_version'] = version
# Save the package name and variant. The variant is used when installing
# packages to validate dependencies.
buildinfo['name'] = name
buildinfo['variant'] = variant
# If the package is already built, don't do anything.
pkg_path = pkg_abs("{}.tar.xz".format(pkg_id))
# Done if it exists locally
if exists(pkg_path):
print("Package up to date. Not re-building.")
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
check_call(["mkdir", "-p", pkg_abs("cache")])
write_string(pkg_abs(last_build_filename(variant)), str(pkg_id))
return pkg_path
# Try downloading.
if repository_url:
tmp_filename = pkg_path + '.tmp'
try:
# Normalize to no trailing slash for repository_url
repository_url = repository_url.rstrip('/')
url = repository_url + '/packages/{0}/{1}.tar.xz'.format(pkg_id.name, str(pkg_id))
print("Attempting to download", pkg_id, "from", url)
download(tmp_filename, url, package_dir)
os.rename(tmp_filename, pkg_path)
print("Package up to date. Not re-building. Downloaded from repository-url.")
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
check_call(["mkdir", "-p", pkg_abs("cache")])
write_string(pkg_abs(last_build_filename(variant)), str(pkg_id))
return pkg_path
except FetchError:
try:
os.remove(tmp_filename)
except:
pass
# Fall out and do the build since the command errored.
print("Unable to download from cache. Proceeding to build")
print("Building package {} with buildinfo: {}".format(
pkg_id,
json.dumps(buildinfo, indent=2, sort_keys=True)))
# Clean out src, result so later steps can use them freely for building.
clean(package_dir)
# Only fresh builds are allowed which don't overlap existing artifacts.
result_dir = pkg_abs("result")
if exists(result_dir):
raise BuildError("result folder must not exist. It will be made when the package is "
"built. {}".format(result_dir))
# 'mkpanda add' all implicit dependencies since we actually need to build.
for dep in auto_deps:
print("Auto-adding dependency: {}".format(dep))
# NOTE: Not using the name pkg_id because that overrides the outer one.
id_obj = PackageId(dep)
add_to_repository(repository, pkg_abs('../{0}/{1}.tar.xz'.format(id_obj.name, dep)))
package = repository.load(dep)
active_packages.append(package)
# Checkout all the sources int their respective 'src/' folders.
try:
src_dir = pkg_abs('src')
if os.path.exists(src_dir):
raise ValidationError(
"'src' directory already exists, did you have a previous build? " +
"Currently all builds must be from scratch. Support should be " +
"added for re-using a src directory when possible. src={}".format(src_dir))
os.mkdir(src_dir)
for src_name, fetcher in sorted(fetchers.items()):
root = pkg_abs('src/' + src_name)
os.mkdir(root)
fetcher.checkout_to(root)
except ValidationError as ex:
raise BuildError("Validation error when fetching sources for package: {}".format(ex))
# Copy over environment settings
if 'environment' in buildinfo:
pkginfo['environment'] = buildinfo['environment']
# Activate the packages so that we have a proper path, environment
# variables.
# TODO(cmaloney): RAII type thing for temproary directory so if we
# don't get all the way through things will be cleaned up?
install = Install(install_dir, None, True, False, True, True)
install.activate(active_packages)
# Rewrite all the symlinks inside the active path because we will
# be mounting the folder into a docker container, and the absolute
# paths to the packages will change.
# TODO(cmaloney): This isn't very clean, it would be much nicer to
# just run pkgpanda inside the package.
rewrite_symlinks(install_dir, repository.path, "/opt/mesosphere/packages/")
print("Building package in docker")
# TODO(cmaloney): Run as a specific non-root user, make it possible
# for non-root to cleanup afterwards.
# Run the build, prepping the environment as necessary.
mkdir(pkg_abs("result"))
# Copy the build info to the resulting tarball
write_json(pkg_abs("src/buildinfo.full.json"), buildinfo)
write_json(pkg_abs("result/buildinfo.full.json"), buildinfo)
write_json(pkg_abs("result/pkginfo.json"), pkginfo)
# Make the folder for the package we are building. If docker does it, it
# gets auto-created with root permissions and we can't actually delete it.
os.makedirs(os.path.join(install_dir, "packages", str(pkg_id)))
# TOOD(cmaloney): Disallow writing to well known files and directories?
# Source we checked out
cmd.volumes.update({
# TODO(cmaloney): src should be read only...
pkg_abs("src"): "/pkg/src:rw",
# The build script
pkg_abs(buildinfo.get('build_script', 'build')): "/pkg/build:ro",
# Getting the result out
pkg_abs("result"): "/opt/mesosphere/packages/{}:rw".format(pkg_id),
install_dir: "/opt/mesosphere:ro"
})
if os.path.exists(extra_dir):
cmd.volumes[extra_dir] = "/pkg/extra:ro"
cmd.environment = {
"PKG_VERSION": version,
"PKG_NAME": name,
"PKG_ID": pkg_id,
"PKG_PATH": "/opt/mesosphere/packages/{}".format(pkg_id),
"PKG_VARIANT": variant if variant is not None else "<default>"
}
try:
# TODO(cmaloney): Run a wrapper which sources
# /opt/mesosphere/environment then runs a build. Also should fix
# ownership of /opt/mesosphere/packages/{pkg_id} post build.
cmd.run([
"/bin/bash",
"-o", "nounset",
"-o", "pipefail",
"-o", "errexit",
"/pkg/build"])
except CalledProcessError as ex:
raise BuildError("docker exited non-zero: {}\nCommand: {}".format(ex.returncode, ' '.join(ex.cmd)))
# Clean up the temporary install dir used for dependencies.
# TODO(cmaloney): Move to an RAII wrapper.
check_call(['rm', '-rf', install_dir])
print("Building package tarball")
# Check for forbidden services before packaging the tarball:
try:
check_forbidden_services(pkg_abs("result"), RESERVED_UNIT_NAMES)
except ValidationError as ex:
raise BuildError("Package validation failed: {}".format(ex))
# TODO(cmaloney): Updating / filling last_build should be moved out of
# the build function.
check_call(["mkdir", "-p", pkg_abs("cache")])
write_string(pkg_abs(last_build_filename(variant)), str(pkg_id))
# Bundle the artifacts into the pkgpanda package
tmp_name = pkg_path + "-tmp.tar.xz"
make_tar(tmp_name, pkg_abs("result"))
os.rename(tmp_name, pkg_path)
print("Package built.")
if clean_after_build:
clean(package_dir)
return pkg_path
def make_bootstrap_tarball(packages_dir, packages, variant, repository_url):
# Convert filenames to package ids
pkg_ids = list()
for pkg_path in packages:
# Get the package id from the given package path
filename = os.path.basename(pkg_path)
if not filename.endswith(".tar.xz"):
raise BuildError("Packages must be packaged / end with a .tar.xz. Got {}".format(filename))
pkg_id = filename[:-len(".tar.xz")]
pkg_ids.append(pkg_id)
# Filename is output_name.<sha-1>.{active.json|.bootstrap.tar.xz}
bootstrap_id = hash_checkout(pkg_ids)
latest_name = "{}/{}bootstrap.latest".format(packages_dir, pkgpanda.util.variant_prefix(variant))
output_name = packages_dir + '/' + bootstrap_id + '.'
# bootstrap tarball = <sha1 of packages in tarball>.bootstrap.tar.xz
bootstrap_name = "{}bootstrap.tar.xz".format(output_name)
active_name = "{}active.json".format(output_name)
def mark_latest():
# Ensure latest is always written
write_string(latest_name, bootstrap_id)
print("bootstrap: {}".format(bootstrap_name))
print("active: {}".format(active_name))
print("latest: {}".format(latest_name))
return bootstrap_name
if (os.path.exists(bootstrap_name)):
print("Bootstrap already up to date, not recreating")
return mark_latest()
# Try downloading.
if repository_url:
tmp_bootstrap = bootstrap_name + '.tmp'
tmp_active = active_name + '.tmp'
try:
repository_url = repository_url.rstrip('/')
bootstrap_url = repository_url + '/bootstrap/{}.bootstrap.tar.xz'.format(bootstrap_id)
active_url = repository_url + '/bootstrap/{}.active.json'.format(bootstrap_id)
print("Attempting to download", bootstrap_name, "from", bootstrap_url)
# Normalize to no trailing slash for repository_url
download(tmp_bootstrap, bootstrap_url, packages_dir)
print("Attempting to download", active_name, "from", active_url)
download(tmp_active, active_url, packages_dir)
# Move into place
os.rename(tmp_bootstrap, bootstrap_name)
os.rename(tmp_active, active_name)
print("Bootstrap already up to date, Not recreating. Downloaded from repository-url.")
return mark_latest()
except FetchError:
try:
os.remove(tmp_bootstrap)
except:
pass
try:
os.remove(tmp_active)
except:
pass
# Fall out and do the build since the command errored.
print("Unable to download from cache. Building.")
print("Creating bootstrap tarball for variant {}".format(variant))
work_dir = tempfile.mkdtemp(prefix='mkpanda_bootstrap_tmp')
def make_abs(path):
return os.path.join(work_dir, path)
pkgpanda_root = make_abs("opt/mesosphere")
repository = Repository(os.path.join(pkgpanda_root, "packages"))
# Fetch all the packages to the root
for pkg_path in packages:
filename = os.path.basename(pkg_path)
pkg_id = filename[:-len(".tar.xz")]
def local_fetcher(id, target):
shutil.unpack_archive(pkg_path, target, "gztar")
repository.add(local_fetcher, pkg_id, False)
# Activate the packages inside the repository.
# Do generate dcos.target.wants inside the root so that we don't
# try messing with /etc/systemd/system.
install = Install(pkgpanda_root, None, True, False, True, True, True)
install.activate(repository.load_packages(pkg_ids))
# Mark the tarball as a bootstrap tarball/filesystem so that
# dcos-setup.service will fire.
make_file(make_abs("opt/mesosphere/bootstrap"))
# Write out an active.json for the bootstrap tarball
write_json(active_name, pkg_ids)
# Rewrite all the symlinks to point to /opt/mesosphere
rewrite_symlinks(work_dir, work_dir, "/")
make_tar(bootstrap_name, pkgpanda_root)
shutil.rmtree(work_dir)
# Update latest last so that we don't ever use partially-built things.
write_string(latest_name, bootstrap_id)
print("Built bootstrap")
return mark_latest()
