def requests_fetcher(base_url, id_str, target, work_dir):
assert base_url
assert type(id_str) == str
id = PackageId(id_str)
# TODO(cmaloney): That file:// urls are allowed in base_url is likely a security hole.
# TODO(cmaloney): Switch to mesos-fetcher or aci or something so
# all the logic can go away, we gain integrity checking, etc.
base_url = base_url.rstrip('/')
url = base_url + "/packages/{0}/{1}.tar.xz".format(id.name, id_str)
# TODO(cmaloney): Use a private tmp directory so there is no chance of a user
# intercepting the tarball + other validation data locally.
with tempfile.NamedTemporaryFile(suffix=".tar.xz") as file:
download(file.name, url, work_dir, rm_on_error=False)
extract_tarball(file.name, target)
def requests_fetcher(base_url, id_str, target, work_dir):
assert base_url
assert type(id_str) == str
id = PackageId(id_str)
# TODO(cmaloney): That file:// urls are allowed in base_url is likely a security hole.
# TODO(cmaloney): Switch to mesos-fetcher or aci or something so
# all the logic can go away, we gain integrity checking, etc.
base_url = base_url.rstrip('/')
url = base_url + "/packages/{0}/{1}.tar.xz".format(id.name, id_str)
# TODO(cmaloney): Use a private tmp directory so there is no chance of a user
# intercepting the tarball + other validation data locally.
with tempfile.NamedTemporaryFile(suffix=".tar.xz") as file:
download(file.name, url, work_dir)
extract_tarball(file.name, target)
def _do_bootstrap(install, repository):
# These files should be set by the environment which initially builds
# the host (cloud-init).
repository_url = if_exists(load_string, install.get_config_filename("setup-flags/repository-url"))
def fetcher(id, target):
if repository_url is None:
raise ValidationError("ERROR: Non-local package {} but no repository url given.".format(id))
return requests_fetcher(repository_url, id, target, os.getcwd())
setup_pkg_dir = install.get_config_filename("setup-packages")
if os.path.exists(setup_pkg_dir):
raise ValidationError(
"setup-packages is no longer supported. It's functionality has been replaced with late "
"binding packages. Found setup packages dir: {}".format(setup_pkg_dir))
setup_packages_to_activate = []
# If the host has late config values, build the late config package from them.
late_config = if_exists(load_yaml, install.get_config_filename("setup-flags/late-config.yaml"))
if late_config:
pkg_id_str = late_config['late_bound_package_id']
late_values = late_config['bound_values']
print("Binding late config to late package {}".format(pkg_id_str))
print("Bound values: {}".format(late_values))
if not PackageId.is_id(pkg_id_str):
raise ValidationError("Invalid late package id: {}".format(pkg_id_str))
pkg_id = PackageId(pkg_id_str)
if pkg_id.version != "setup":
raise ValidationError("Late package must have the version setup. Bad package: {}".format(pkg_id_str))
# Collect the late config package.
with tempfile.NamedTemporaryFile() as f:
download(
f.name,
repository_url + '/packages/{0}/{1}.dcos_config'.format(pkg_id.name, pkg_id_str),
os.getcwd(),
rm_on_error=False,
)
late_package = load_yaml(f.name)
# Resolve the late package using the bound late config values.
final_late_package = resolve_late_package(late_package, late_values)
# Render the package onto the filesystem and add it to the package
# repository.
with tempfile.NamedTemporaryFile() as f:
do_gen_package(final_late_package, f.name)
repository.add(lambda _, target: extract_tarball(f.name, target), pkg_id_str)
setup_packages_to_activate.append(pkg_id_str)
# If active.json is set on the host, use that as the set of packages to
# activate. Otherwise just use the set of currently active packages (those
# active in the bootstrap tarball)
to_activate = None
active_path = install.get_config_filename("setup-flags/active.json")
if os.path.exists(active_path):
print("Loaded active packages from", active_path)
to_activate = load_json(active_path)
# Ensure all packages are local
print("Ensuring all packages in active set {} are local".format(",".join(to_activate)))
for package in to_activate:
repository.add(fetcher, package)
else:
print("Calculated active packages from bootstrap tarball")
to_activate = list(install.get_active())
package_list_filename = install.get_config_filename("setup-flags/cluster-package-list")
print("Checking for cluster packages in:", package_list_filename)
package_list_id = if_exists(load_string, package_list_filename)
if package_list_id:
print("Cluster package list:", package_list_id)
cluster_packages = _get_package_list(package_list_id, repository_url)
print("Loading cluster-packages: {}".format(cluster_packages))
for package_id_str in cluster_packages:
# Validate the package ids
pkg_id = PackageId(package_id_str)
# Fetch the packages if not local
if not repository.has_package(package_id_str):
repository.add(fetcher, package_id_str)
# Add the package to the set to activate
setup_packages_to_activate.append(package_id_str)
else:
print("No cluster-packages specified")
# Calculate the full set of final packages (Explicit activations + setup packages).
# De-duplicate using a set.
to_activate = list(set(to_activate + setup_packages_to_activate))
print("Activating packages")
install.activate(repository.load_packages(to_activate))
def fetch(_, target):
extract_tarball(package_filename, target)
def fetch(_, target):
extract_tarball(path, target)
def fetch(_, target):
extract_tarball(path, target)
