def requests_fetcher(base_url, id_str, target, work_dir): assert base_url assert type(id_str) == str id = PackageId(id_str) # TODO(cmaloney): That file:// urls are allowed in base_url is likely a security hole. # TODO(cmaloney): Switch to mesos-fetcher or aci or something so # all the logic can go away, we gain integrity checking, etc. base_url = base_url.rstrip('/') url = base_url + "/packages/{0}/{1}.tar.xz".format(id.name, id_str) # TODO(cmaloney): Use a private tmp directory so there is no chance of a user # intercepting the tarball + other validation data locally. with tempfile.NamedTemporaryFile(suffix=".tar.xz") as file: download(file.name, url, work_dir, rm_on_error=False) extract_tarball(file.name, target)
def requests_fetcher(base_url, id_str, target, work_dir): assert base_url assert type(id_str) == str id = PackageId(id_str) # TODO(cmaloney): That file:// urls are allowed in base_url is likely a security hole. # TODO(cmaloney): Switch to mesos-fetcher or aci or something so # all the logic can go away, we gain integrity checking, etc. base_url = base_url.rstrip('/') url = base_url + "/packages/{0}/{1}.tar.xz".format(id.name, id_str) # TODO(cmaloney): Use a private tmp directory so there is no chance of a user # intercepting the tarball + other validation data locally. with tempfile.NamedTemporaryFile(suffix=".tar.xz") as file: download(file.name, url, work_dir) extract_tarball(file.name, target)
def _do_bootstrap(install, repository): # These files should be set by the environment which initially builds # the host (cloud-init). repository_url = if_exists(load_string, install.get_config_filename("setup-flags/repository-url")) def fetcher(id, target): if repository_url is None: raise ValidationError("ERROR: Non-local package {} but no repository url given.".format(id)) return requests_fetcher(repository_url, id, target, os.getcwd()) setup_pkg_dir = install.get_config_filename("setup-packages") if os.path.exists(setup_pkg_dir): raise ValidationError( "setup-packages is no longer supported. It's functionality has been replaced with late " "binding packages. Found setup packages dir: {}".format(setup_pkg_dir)) setup_packages_to_activate = [] # If the host has late config values, build the late config package from them. late_config = if_exists(load_yaml, install.get_config_filename("setup-flags/late-config.yaml")) if late_config: pkg_id_str = late_config['late_bound_package_id'] late_values = late_config['bound_values'] print("Binding late config to late package {}".format(pkg_id_str)) print("Bound values: {}".format(late_values)) if not PackageId.is_id(pkg_id_str): raise ValidationError("Invalid late package id: {}".format(pkg_id_str)) pkg_id = PackageId(pkg_id_str) if pkg_id.version != "setup": raise ValidationError("Late package must have the version setup. Bad package: {}".format(pkg_id_str)) # Collect the late config package. with tempfile.NamedTemporaryFile() as f: download( f.name, repository_url + '/packages/{0}/{1}.dcos_config'.format(pkg_id.name, pkg_id_str), os.getcwd(), rm_on_error=False, ) late_package = load_yaml(f.name) # Resolve the late package using the bound late config values. final_late_package = resolve_late_package(late_package, late_values) # Render the package onto the filesystem and add it to the package # repository. with tempfile.NamedTemporaryFile() as f: do_gen_package(final_late_package, f.name) repository.add(lambda _, target: extract_tarball(f.name, target), pkg_id_str) setup_packages_to_activate.append(pkg_id_str) # If active.json is set on the host, use that as the set of packages to # activate. Otherwise just use the set of currently active packages (those # active in the bootstrap tarball) to_activate = None active_path = install.get_config_filename("setup-flags/active.json") if os.path.exists(active_path): print("Loaded active packages from", active_path) to_activate = load_json(active_path) # Ensure all packages are local print("Ensuring all packages in active set {} are local".format(",".join(to_activate))) for package in to_activate: repository.add(fetcher, package) else: print("Calculated active packages from bootstrap tarball") to_activate = list(install.get_active()) package_list_filename = install.get_config_filename("setup-flags/cluster-package-list") print("Checking for cluster packages in:", package_list_filename) package_list_id = if_exists(load_string, package_list_filename) if package_list_id: print("Cluster package list:", package_list_id) cluster_packages = _get_package_list(package_list_id, repository_url) print("Loading cluster-packages: {}".format(cluster_packages)) for package_id_str in cluster_packages: # Validate the package ids pkg_id = PackageId(package_id_str) # Fetch the packages if not local if not repository.has_package(package_id_str): repository.add(fetcher, package_id_str) # Add the package to the set to activate setup_packages_to_activate.append(package_id_str) else: print("No cluster-packages specified") # Calculate the full set of final packages (Explicit activations + setup packages). # De-duplicate using a set. to_activate = list(set(to_activate + setup_packages_to_activate)) print("Activating packages") install.activate(repository.load_packages(to_activate))
def fetch(_, target): extract_tarball(package_filename, target)
def fetch(_, target): extract_tarball(path, target)