def extract_tarball(path, target): """Extract the tarball into target. If there are any errors, delete the folder being extracted to. """ # TODO(cmaloney): Validate extraction will pass before unpacking as much as possible. # TODO(cmaloney): Unpack into a temporary directory then move into place to # prevent partial extraction from ever laying around on the filesystem. try: assert os.path.exists( path), "Path doesn't exist but should: {}".format(path) make_directory(target) # TODO(tweidner): https://jira.mesosphere.com/browse/DCOS-48220 # Make this cross-platform via Python's tarfile module once # https://bugs.python.org/issue21872 is fixed. if is_windows: subprocess.check_call(['bsdtar', '-xf', path, '-C', target]) else: subprocess.check_call(['tar', '-xf', path, '-C', target]) except: # If there are errors, we can't really cope since we are already in an error state. rmtree(target, ignore_errors=True) raise
def remove_directory(path): """recursively removes a directory tree. fails silently if the tree does not exist""" if is_windows: # python library on Windows does not like symbolic links in directories # so calling out to the cmd prompt to do this fixes that. path = path.replace('/', '\\') if os.path.exists(path): subprocess.call(['cmd.exe', '/c', 'rmdir', '/s', '/q', path]) else: subprocess.check_call(['rm', '-rf', path])
def copy_file(src_path, dst_path): """copy a single directory item from one location to another""" if is_windows: # To make sure the copy works we are using cmd version as python # libraries may not handle symbolic links and other things that are # thrown at it. src = src_path.replace('/', '\\') dst = dst_path.replace('/', '\\') subprocess.check_call(['cmd.exe', '/c', 'copy', src, dst]) else: subprocess.check_call(['cp', src_path, dst_path])
def copy_directory(src_path, dst_path): """copy recursively a directory tree from one location to another""" if is_windows: # To make sure the copy works we are using cmd version as python # libraries may not handle symbolic links and other things that are # thrown at it. src = src_path.replace('/', '\\') dst = dst_path.replace('/', '\\') subprocess.check_call( ['cmd.exe', '/c', 'xcopy', src, dst, '/E', '/B', '/I']) else: subprocess.check_call(['cp', '-r', src_path, dst_path])
def checkout_to(self, directory): # Clone into `src/`. check_call(["git", "clone", "-q", self.src_repo_path, directory]) # Make sure we got the right commit as head assert get_git_sha1(directory + "/.git", "HEAD") == self.commit # Checkout from the bare repo in the cache folder at the specific sha1 check_call([ "git", "--git-dir", directory + "/.git", "--work-tree", directory, "checkout", "-f", "-q", self.commit ])
def _do_build_docker(name, path): path_sha = pkgpanda.build.hash_folder_abs(path, os.path.dirname(path)) container_name = 'dcos/dcos-builder:{}_dockerdir-{}'.format(name, path_sha) log.debug("Attempting to pull docker: %s", container_name) pulled = False try: # TODO(cmaloney): Rather than pushing / pulling from Docker Hub upload as a build artifact. # the exported tarball. subprocess.check_call(['docker', 'pull', container_name]) pulled = True # TODO(cmaloney): Differentiate different failures of running the process better here except subprocess.CalledProcessError: pulled = False if not pulled: log.debug("Pull failed, building instead: %s", container_name) # Pull failed, build it subprocess.check_call(['docker', 'build', '-t', container_name, path]) # TODO(cmaloney): Push the built docker image on successful package build to both # 1) commit-<commit_id> # 2) Dockerfile-<file_contents_sha1> # 3) bootstrap-<bootstrap_id> # So we can track back the builder id for a given commit or bootstrap id, and reproduce whatever # we need. The Dockerfile-<sha1> is useful for making sure we don't rebuild more than # necessary. try: subprocess.check_call(['docker', 'push', container_name]) except subprocess.CalledProcessError: logger.warning( "docker push of dcos-builder failed. This means it will be very difficult " "for this build to be reproduced (others will have a different / non-identical " "base docker for most packages.") pass # mark as latest so it will be used when building packages # extract the docker client version string try: docker_version = subprocess.check_output( ['docker', 'version', '-f', '{{.Client.Version}}']).decode() except subprocess.CalledProcessError: # If the above command fails then we know we have an older version of docker # Older versions of docker spit out an entirely different format docker_version = subprocess.check_output( ['docker', 'version']).decode().split("\n")[0].split()[2] # only use force tag if using docker version 1.9 or earlier container_name_t = 'dcos/dcos-builder:{}_dockerdir-latest'.format(name) if LooseVersion(docker_version) < LooseVersion('1.10'): args = ['docker', 'tag', '-f', container_name, container_name_t] else: args = ['docker', 'tag', container_name, container_name_t] subprocess.check_call(args)
def checkout_to(self, directory): # fetch into a bare repository so if we're on a host which has a cache we can # only get the new commits. fetch_git(self.bare_folder, self.url) # Warn if the ref_origin is set and gives a different sha1 than the # current ref. try: origin_commit = get_git_sha1(self.bare_folder, self.ref_origin) except Exception as ex: raise ValidationError( "Unable to find sha1 of ref_origin {}: {}".format( self.ref_origin, ex)) if self.ref != origin_commit: logger.warning( "Current ref doesn't match the ref origin. " "Package ref should probably be updated to pick up " "new changes to the code:" + " Current: {}, Origin: {}".format(self.ref, origin_commit)) # Clone into `src/`. if is_windows: # Note: Mesos requires autocrlf to be set on Windows otherwise it does not build. check_call([ "git", "clone", "-q", "--config", "core.autocrlf=true", self.bare_folder, directory ]) else: check_call(["git", "clone", "-q", self.bare_folder, directory]) # Checkout from the bare repo in the cache folder at the specific sha1 check_call([ "git", "--git-dir", directory + "/.git", "--work-tree", directory, "checkout", "-f", "-q", self.ref ])
def stop_all(self): if not self.__active: log.warning("Do not stop services") return if not os.path.exists(self.__unit_directory): log.warning("Do not stop services. %s does not exist", self.__unit_directory) return names = list(filter( lambda n: os.path.isfile(os.path.join(self.__unit_directory, n)), os.listdir(self.__unit_directory))) try: cmd = ["systemctl", "stop"] + names if not self.__block: cmd.append("--no-block") check_call(cmd) except CalledProcessError as ex: # If the service doesn't exist, don't error. This happens when a # bootstrap tarball has just been extracted but nothing started # yet during first activation. log.warning(ex) if ex.returncode != 5: raise
def extract_archive(archive, dst_dir): archive_type = _identify_archive_type(archive) if archive_type == 'tar': if is_windows: check_call(["bsdtar", "-xf", archive, "-C", dst_dir]) else: check_call( ["tar", "-xf", archive, "--strip-components=1", "-C", dst_dir]) elif archive_type == 'zip': if is_windows: check_call([ "powershell.exe", "-command", "expand-archive", "-path", archive, "-destinationpath", dst_dir ]) else: check_call(["unzip", "-x", archive, "-d", dst_dir]) # unzip binary does not support '--strip-components=1', _strip_first_path_component(dst_dir) else: raise ValidationError("Unsupported archive: {}".format( os.path.basename(archive)))
def fetch_git(bare_folder, git_uri): # Do a git clone if the cache folder doesn't exist yet, otherwise # do a git pull of everything. if not os.path.exists(bare_folder): check_call( ["git", "clone", "--mirror", "--progress", git_uri, bare_folder]) else: check_call([ "git", "--git-dir", bare_folder, "remote", "set-url", "origin", git_uri ]) check_call( ["git", "--git-dir", bare_folder, "remote", "update", "origin"]) return bare_folder
def make_installer_docker(variant, variant_info, installer_info): bootstrap_id = variant_info['bootstrap'] assert len(bootstrap_id) > 0 image_version = util.dcos_image_commit[:18] + '-' + bootstrap_id[:18] genconf_tar = "dcos-genconf." + image_version + ".tar" installer_filename = "packages/cache/dcos_generate_config." + pkgpanda.util.variant_prefix( variant) + "sh" bootstrap_filename = bootstrap_id + ".bootstrap.tar.xz" bootstrap_active_filename = bootstrap_id + ".active.json" installer_bootstrap_filename = installer_info[ 'bootstrap'] + '.bootstrap.tar.xz' bootstrap_latest_filename = pkgpanda.util.variant_prefix( variant) + 'bootstrap.latest' latest_complete_filename = pkgpanda.util.variant_prefix( variant) + 'complete.latest.json' packages_dir = 'packages' docker_image_name = 'mesosphere/dcos-genconf:' + image_version # TODO(cmaloney): All of this should use package_resources with tempfile.TemporaryDirectory() as build_dir: assert build_dir[-1] != '/' print("Setting up build environment") def dest_path(filename): return build_dir + '/' + filename def copy_to_build(src_prefix, filename): dest_filename = dest_path(filename) os.makedirs(os.path.dirname(dest_filename), exist_ok=True) copy_file(os.getcwd() + '/' + src_prefix + '/' + filename, dest_filename) def fill_template(base_name, format_args): pkgpanda.util.write_string( dest_path(base_name), pkg_resources.resource_string( __name__, 'bash/' + base_name + '.in').decode().format(**format_args)) fill_template( 'Dockerfile', { 'installer_bootstrap_filename': installer_bootstrap_filename, 'bootstrap_filename': bootstrap_filename, 'bootstrap_active_filename': bootstrap_active_filename, 'bootstrap_latest_filename': bootstrap_latest_filename, 'latest_complete_filename': latest_complete_filename, 'packages_dir': packages_dir }) fill_template( 'installer_internal_wrapper', { 'variant': pkgpanda.util.variant_str(variant), 'bootstrap_id': bootstrap_id, 'dcos_image_commit': util.dcos_image_commit }) subprocess.check_call( ['chmod', '+x', dest_path('installer_internal_wrapper')]) # TODO(cmaloney) make this use make_bootstrap_artifacts / that set # rather than manually keeping everything in sync copy_to_build('packages/cache/bootstrap', bootstrap_filename) copy_to_build('packages/cache/bootstrap', installer_bootstrap_filename) copy_to_build('packages/cache/bootstrap', bootstrap_active_filename) copy_to_build('packages/cache/bootstrap', bootstrap_latest_filename) copy_to_build('packages/cache/complete', latest_complete_filename) for package_id in variant_info['packages']: package_name = pkgpanda.PackageId(package_id).name copy_to_build( 'packages/cache/', packages_dir + '/' + package_name + '/' + package_id + '.tar.xz') # Copy across gen_extra if it exists if os.path.exists('gen_extra'): copy_directory('gen_extra', dest_path('gen_extra')) else: make_directory(dest_path('gen_extra')) print("Building docker container in " + build_dir) subprocess.check_call( ['docker', 'build', '-t', docker_image_name, build_dir]) print("Building", installer_filename) pkgpanda.util.write_string( installer_filename, pkg_resources.resource_string( __name__, 'bash/dcos_generate_config.sh.in').decode().format( genconf_tar=genconf_tar, docker_image_name=docker_image_name, variant=variant) + '\n#EOF#\n') subprocess.check_call(['docker', 'save', docker_image_name], stdout=open(genconf_tar, 'w')) subprocess.check_call(['tar', 'cvf', '-', genconf_tar], stdout=open(installer_filename, 'a')) subprocess.check_call(['chmod', '+x', installer_filename]) # Cleanup subprocess.check_call(['rm', genconf_tar]) return installer_filename
def activate(self, packages): # Ensure the new set is reasonable. validate_compatible(packages, self.__roles) # Build the absolute paths for the running config, new config location, # and where to archive the config. active_names = self.get_active_names() active_dirs = list( map(self._make_abs, self.__well_known_dirs + ["active"])) new_names = [name + ".new" for name in active_names] new_dirs = [name + ".new" for name in active_dirs] old_names = [name + ".old" for name in active_names] log.info("Remove all pre-existing new and old directories") for name in chain(new_names, old_names): if os.path.exists(name): if os.path.isdir(name): remove_directory(name) else: os.remove(name) log.info( "Remove unit files staged for an activation that didn't occur.") if not self.__skip_systemd_dirs: self.systemd.remove_staged_unit_files() log.debug("Make the directories for the new config: " + ", ".join(new_dirs)) for name in new_dirs: os.makedirs(name) def symlink_all(src, dest): if not os.path.isdir(src): return symlink_tree(src, dest) log.info("Set the new LD_LIBRARY_PATH, PATH.") env_contents = env_header.format( "/opt/mesosphere" if self.__fake_path else self.__root) env_export_contents = env_export_header.format( "/opt/mesosphere" if self.__fake_path else self.__root) active_buildinfo_full = {} dcos_service_configuration = self._get_dcos_configuration_template() log.info("Building up the set of users.") sysusers = UserManagement(self.__manage_users, self.__add_users) def _get_service_files(_dir): service_files = [] for root, directories, filenames in os.walk(_dir): for filename in filter(lambda name: name.endswith(".service"), filenames): service_files.append(os.path.join(root, filename)) return service_files def _get_service_names(_dir): service_files = list( map(os.path.basename, _get_service_files(_dir))) if not service_files: return [] return list( map(lambda name: os.path.splitext(name)[0], service_files)) # Add the folders, config in each package. for package in packages: # Package folders # NOTE: Since active is at the end of the folder list it will be # removed by the zip. This is the desired behavior, since it will be # populated later. # Do the basename since some well known dirs are full paths (dcos.target.wants) # while inside the packages they are always top level directories. for new, dir_name in zip(new_dirs, self.__well_known_dirs): dir_name = os.path.basename(dir_name) pkg_dir = os.path.join(package.path, dir_name) assert os.path.isabs(new) assert os.path.isabs(pkg_dir) try: symlink_all(pkg_dir, new) # Symlink all applicable role-based config for role in self.__roles: role_dir = os.path.join( package.path, "{0}_{1}".format(dir_name, role)) symlink_all(role_dir, new) except ConflictingFile as ex: raise ValidationError( "Two packages are trying to install the same file {0} or " "two roles in the set of roles {1} are causing a package " "to try activating multiple versions of the same file. " "One of the package files is {2}.".format( ex.dest, self.__roles, ex.src)) log.info("Add %s to the active folder", package.name) os.symlink( package.path, os.path.join(self._make_abs("active.new"), package.name)) # Add to the environment and environment.export contents env_contents += "# package: {0}\n".format(package.id) env_export_contents += "# package: {0}\n".format(package.id) for k, v in package.environment.items(): env_contents += "{0}={1}\n".format(k, v) env_export_contents += "export {0}={1}\n".format(k, v) env_contents += "\n" env_export_contents += "\n" # Add to the buildinfo try: active_buildinfo_full[package.name] = load_json( os.path.join(package.path, "buildinfo.full.json")) except FileNotFoundError: # TODO(cmaloney): These only come from setup-packages. Should update # setup-packages to add a buildinfo.full for those packages active_buildinfo_full[package.name] = None # NOTE: It is critical the state dir, the package name and the user name are all the # same. Otherwise on upgrades we might remove access to a files by changing their chown # to something incompatible. We survive the first upgrade because everything goes from # root to specific users, and root can access all user files. if package.username is not None: sysusers.add_user(package.username, package.group) # Ensure the state directory exists # TODO(cmaloney): On upgrade take a snapshot? if self.__manage_state_dir: state_dir_path = self.__state_dir_root + '/' + package.name if package.state_directory: make_directory(state_dir_path) if package.username and not is_windows: uid = sysusers.get_uid(package.username) check_call(['chown', '-R', str(uid), state_dir_path]) if package.sysctl: service_names = _get_service_names(package.path) if not service_names: raise ValueError( "service name required for sysctl could not be determined for {package}" .format(package=package.id)) for service in service_names: if service in package.sysctl: dcos_service_configuration["sysctl"][ service] = package.sysctl[service] log.info("Prepare new systemd units for activation.") if not self.__skip_systemd_dirs: new_wants_dir = self._make_abs(self.__systemd_dir + ".new") if os.path.exists(new_wants_dir): self.systemd.stage_new_units(new_wants_dir) dcos_service_configuration_file = os.path.join( self._make_abs("etc.new"), DCOS_SERVICE_CONFIGURATION_FILE) write_json(dcos_service_configuration_file, dcos_service_configuration) log.info("Write out the new environment file.") new_env = self._make_abs("environment.new") write_string(new_env, env_contents) log.info("Write out the new environment.export file") new_env_export = self._make_abs("environment.export.new") write_string(new_env_export, env_export_contents) log.info("Write out the buildinfo of every active package") new_buildinfo_meta = self._make_abs("active.buildinfo.full.json.new") write_json(new_buildinfo_meta, active_buildinfo_full) self.swap_active(".new")