def apply(self, action, auth_config): """ Write a set of files to disk, returning a list of the files that have changed. Arguments: action -- a dict of pathname to attributes, such as owner, group, mode, content, and encoding auth_config -- an AuthenticationConfig object for managing authenticated downloads Exceptions: ToolError -- on expected failures """ files_changed = [] if not action.keys(): log.debug("No files specified") return files_changed for (filename, attribs) in sorted(action.iteritems(), key=lambda pair: pair[0]): if not os.path.isabs(filename): raise ToolError('File specified with non-absolute path: %s' % filename) # The only difference between a file and a symlink is hidden in the mode file_is_link = "mode" in attribs and stat.S_ISLNK(int(attribs["mode"], 8)) if file_is_link: if "content" not in attribs: raise ToolError("Symbolic link specified without a destination") elif os.path.exists(filename) and FileTool.is_same_file(os.path.realpath(filename), attribs["content"]): log.info("Symbolic link %s already exists", filename) continue parent = os.path.dirname(filename) if not os.path.isdir(parent): if not os.path.exists(parent): log.debug("Parent directory %s does not exist, creating", parent) os.makedirs(parent) else: raise ToolError("Parent directory %s exists and is a file" % parent) with self.backup(filename, files_changed): if file_is_link: log.debug("%s is specified as a symbolic link to %s", filename, attribs['content']) os.symlink(attribs["content"], filename) else: file_is_text = 'content' in attribs and not self._is_base64(attribs) with file(filename, 'w' + ('' if file_is_text else 'b')) as f: log.debug("Writing content to %s", filename) self._write_file(f, attribs, auth_config) if "mode" in attribs: log.debug("Setting mode for %s to %s", filename, attribs["mode"]) os.chmod(filename, stat.S_IMODE(int(attribs["mode"], 8))) else: log.debug("No mode specified for %s", filename) security.set_owner_and_group(filename, attribs.get("owner"), attribs.get("group")) return files_changed
def create_group(group_name, gid=None): """Create a group in the OS, returning True if one is created""" try: group_record = grp.getgrnam(group_name) if gid and str(group_record[2]) != gid: raise ToolError("Group %s exists with gid %s, but gid %s was requested" % (group_name, group_record[2], gid)) log.debug("Group %s already exists", group_name) return False except KeyError: pass cmd = ['/usr/sbin/groupadd', '-r'] if gid: cmd.extend(['-g', gid]) cmd.append(group_name) result = ProcessHelper(cmd).call() if result.returncode: log.error("Failed to create group %s", group_name) log.debug("Groupadd output: %s", result.stdout) raise ToolError("Failed to create group %s" % group_name) else: log.info("Created group %s successfully", group_name) return True
def apply(self, action, auth_config=None): """ Install a set of packages via APT, returning the packages actually installed or updated. Arguments: action -- a dict of package name to version; version can be empty, a single string or a list of strings Exceptions: ToolError -- on expected failures (such as a non-zero exit code) """ pkgs_changed = [] if not action: log.debug("No packages specified for APT") return pkgs_changed cache_result = ProcessHelper(['apt-cache', '-q', 'gencaches']).call() if cache_result.returncode: log.error("APT gencache failed. Output: %s", cache_result.stdout) raise ToolError("Could not create apt cache", cache_result.returncode) pkg_specs = [] for pkg_name in action: if action[pkg_name]: if isinstance(action[pkg_name], basestring): pkg_keys = ['%s=%s' % (pkg_name, action[pkg_name])] else: pkg_keys = ['%s=%s' % (pkg_name, ver) if ver else pkg_name for ver in action[pkg_name]] else: pkg_keys = [pkg_name] pkgs_filtered = [pkg_key for pkg_key in pkg_keys if self._pkg_filter(pkg_key, pkg_name)] if pkgs_filtered: pkg_specs.extend(pkgs_filtered) pkgs_changed.append(pkg_name) if not pkg_specs: log.info("All APT packages were already installed") return [] log.info("Attempting to install %s via APT", pkg_specs) env = dict(os.environ) env['DEBIAN_FRONTEND'] = 'noninteractive' result = ProcessHelper(['apt-get', '-q', '-y', 'install'] + pkg_specs, env=env).call() if result.returncode: log.error("apt-get failed. Output: %s", result.stdout) raise ToolError("Could not successfully install APT packages", result.returncode) log.info("APT installed %s", pkgs_changed) log.debug("APT output: %s", result.stdout) return pkgs_changed
def apply(self, action, auth_config): """ Extract archives to their corresponding destination directories, returning directories which were updated. Arguments: action -- a dict of directory to archive location, which can be either a path or URL auth_config -- an AuthenticationConfig object for managing authenticated downloads Exceptions: ToolError -- on expected failures """ dirs_changed = [] if not action: log.debug("No sources specified") return dirs_changed for (path, archive) in sorted(action.iteritems(), key=lambda pair: pair[0]): if SourcesTool._remote_pattern.match(archive): try: archive_file = self._archive_from_url(archive, auth_config) except IOError, e: raise ToolError("Failed to retrieve %s: %s" % (archive, e.strerror)) else: if not os.path.isfile(archive): raise ToolError("%s does not exist" % archive) archive_file = file(archive, 'rb') if TarWrapper.is_compatible(archive_file): log.debug("Treating %s as a tarball", archive) archive_wrapper = TarWrapper(archive_file) elif ZipWrapper.is_compatible(archive_file): log.debug("Treating %s as a zip archive", archive) archive_wrapper = ZipWrapper(archive_file) else: raise ToolError( "Unsupported source file (not zip or tarball): %s" % archive) log.debug( "Checking to ensure that all archive members fall under path %s" % path) self._check_all_members_in_path(path, archive_wrapper) if SourcesTool._github_pattern.match(archive.lower()): log.debug( "Attempting to magically strip GitHub parent directory from archive" ) archive_wrapper = self._perform_github_magic(archive_wrapper) log.debug("Expanding %s into %s", archive, path) archive_wrapper.extract_all(path) dirs_changed.append(path)
def _render_template(self, content, context): if not _templates_supported: raise ToolError("Pystache must be installed in order to render files as Mustache templates") log.debug('Rendering as Mustache template') try: return Renderer(string_encoding='utf-8', file_encoding='utf-8').render(content, context) except Exception, e: raise ToolError("Failed to render content as a Mustache template: %s" % e.message)
def apply(self, action, auth_config): """ Install a set of MSI packages Arguments: action -- a dict of package name to path, which is a string Exceptions: ToolError -- on expected failures (such as a non-zero exit code) """ pkgs_changed = [] if not action.keys(): log.debug("No packages installed for MSI") return pkgs_changed if not _msi_supported: raise ToolError("MSI support is only available under Windows") pkgs = {} tmp_pkgs = [] installer_db = Installer() try: for name, loc in action.iteritems(): if MsiTool._remote_pattern.match(loc): try: msi_file = self._msi_from_url(loc, auth_config) except IOError, e: raise ToolError("Failed to retrieve %s: %s" % (loc, e.strerror)) tmp_pkgs.append(msi_file) else: msi_file = loc if installer_db.isMsiInstalled(msi_file): log.info("%s is already installed; skipping", name) else: pkgs[name] = msi_file if not pkgs: log.info("All MSI packages already installed") return for name, pkg in pkgs.iteritems(): log.debug("Installing %s via MSI", name) installer_db.installProduct(pkg) log.info("Installed %s successfully", name) pkgs_changed.append(pkg) return pkgs_changed
def _install_gem(self, pkg, ver=None): """Install a gem if the version is not already installed; return True if installed, False if skipped.""" if self._gem_is_installed(pkg, ver): log.info("%s-%s is already installed, skipping.", pkg, ver) return False else: log.info("Installing %s version %s via gem", pkg, ver) install_command = [ 'gem', 'install', '-b', '--no-ri', '--no-rdoc', pkg ] if ver: install_command.extend(['-v', '= %s' % ver]) result = ProcessHelper(install_command).call() if result.returncode: log.error("Gem failed. Output: %s", result.stdout) raise ToolError("Failed to install gem: %s-%s" % (pkg, ver), result.returncode) else: log.info("Gem installed: %s-%s", pkg, ver) log.debug("Gem output: %s", result.stdout) return True
def _backup_file(self, source, dest): try: log.debug("Moving %s to %s", source, dest) os.rename(source, dest) except OSError, e: log.error("Could not move %s to %s", source, dest) raise ToolError("Could not rename %s: %s" % (source, str(e)))
def _write_file(self, dest_fileobj, attribs, auth_config): content = attribs.get("content", "") if content: self._write_inline_content(dest_fileobj, content, self._is_base64(attribs), attribs.get('context')) else: source = attribs.get("source", "") if not source: raise ToolError("File specified without source or content") log.debug("Retrieving contents from %s", source) try: self._write_remote_file(source, auth_config.get_auth(attribs.get('authentication', None)), dest_fileobj, attribs.get('context')) except IOError, e: raise ToolError("Failed to retrieve %s: %s" % (source, e.strerror))
def _get_gids(groups): gids = [] for group_name in groups: try: gids.append(str(grp.getgrnam(group_name)[2])) except KeyError: raise ToolError("%s is not a valid group name" % group_name) return gids
def set_service_enabled(self, service, enabled=True): if not os.path.exists(self._executable): raise ToolError("Cannot find chkconfig") result = ProcessHelper( [self._executable, service, 'on' if enabled else 'off']).call() if result.returncode: log.error("chkconfig failed with error %s. Output: %s", result.returncode, result.stdout) raise ToolError( "Could not %s service %s" % ("enable" if enabled else "disable", service), result.returncode) else: log.info("%s service %s", "enabled" if enabled else "disabled", service)
def _pkg_filter(self, pkg, pkg_name): if self._pkg_installed(pkg, pkg_name): log.debug("%s will not be installed as it is already present", pkg) return False elif not self._pkg_available(pkg): log.error("%s is not available to be installed", pkg) raise ToolError("APT does not have %s available for installation" % pkg) else: return True
def set_owner_and_group(filename, owner_name, group_name): owner_id = -1 group_id = -1 if owner_name: try: owner_id = pwd.getpwnam(owner_name)[2] except KeyError: raise ToolError("%s is not a valid user name" % owner_name) if group_name: try: group_id = grp.getgrnam(group_name)[2] except KeyError: raise ToolError("%s is not a valid group name" % group_name) if group_id != -1 or owner_id != -1: logging.debug("Setting owner %s and group %s for %s", owner_id, group_id, filename) os.lchown(filename, owner_id, group_id)
def create_or_modify_user(user_name, groups=[], homedir=None, uid=None): """Create or modify a user in the OS, returning True if action was taken""" try: user_record = pwd.getpwnam(user_name) if uid and str(user_record[2]) != uid: raise ToolError("User %s exists with uid %s, but uid %s was requested" % (user_name, user_record[2], uid)) return _modify_user(user_name, groups, homedir) except KeyError: _create_user(user_name, groups, homedir, uid) return True
def _stop_service(self, service): cmd = self._get_service_executable(service) cmd.append("stop") result = ProcessHelper(cmd).call() if result.returncode: log.error("Could not stop service %s; return code was %s", service, result.returncode) log.debug("Service output: %s", result.stdout) raise ToolError("Could not stop %s" % service) else: log.info("Stopped %s successfully", service)
def _write_inline_content(self, dest, content, is_base64): if not isinstance(content, basestring): log.debug('Content will be serialized as a JSON structure') json.dump(content, dest) return if is_base64: try: log.debug("Decoding base64 content") content = base64.b64decode(content.strip()) except TypeError: raise ToolError("Malformed base64: %s" % content) dest.write(content)
def _check_all_members_in_path(self, path, archive): """ This does a best-effort test to make sure absolute paths or ../../../../ nonsense in archives makes files "escape" their destination """ normalized_parent = os.path.normcase(os.path.abspath(path)) for member in archive.files(): if os.path.isabs(member): prefix = os.path.commonprefix([os.path.normcase(os.path.normpath(member)), normalized_parent]) else: prefix = os.path.commonprefix([os.path.normcase(os.path.normpath(os.path.join(normalized_parent, member))), normalized_parent]) if prefix != normalized_parent: raise ToolError("%s is not a sub-path of %s" % (member, path))
def apply(self, action, auth_config=None): """ Install a set of packages via RPM, returning the packages actually installed or updated. Arguments: action -- a dict of package name to version; version can be empty, a single string or a list of strings Exceptions: ToolError -- on expected failures (such as a non-zero exit code) """ pkgs_changed = [] if not action.keys(): log.debug("No packages installed for RPM") return pkgs_changed pkgs = [] for pkg_name, loc in action.iteritems(): pkgs_to_process = ([loc] if isinstance(loc, basestring) else loc) pkgs_filtered = [ pkg_key for pkg_key in pkgs_to_process if self._package_filter(pkg_key) ] if pkgs_filtered: pkgs.extend(pkgs_filtered) pkgs_changed.append(pkg_name) if not pkgs: log.info("All RPMs were already installed") return [] log.debug("Installing %s via RPM", pkgs) result = ProcessHelper( ['rpm', '-U', '--quiet', '--nosignature', '--replacepkgs'] + pkgs).call() if result.returncode: log.error("RPM failed. Output: %s", result.stdout) raise ToolError("Could not successfully install rpm packages", result.returncode) else: log.debug("RPM output: %s", result.stdout) return pkgs_changed
def apply(self, action, auth_config=None): """ Install a set of packages via easy_install, returning the packages actually installed or updated. Arguments: action -- a dict of package name to version; version can be empty, a single string or a list of strings Exceptions: ToolError -- on expected failures (such as a non-zero exit code) """ pkgs_changed = [] if not action.keys(): log.debug("No packages specified for python") return pkgs_changed pkgs = [] for pkg in action: if not action[pkg] or isinstance(action[pkg], basestring): pkgs.append(PythonTool._pkg_spec(pkg, action[pkg])) else: pkgs.extend( PythonTool._pkg_spec(pkg, ver) for ver in action[pkg]) pkgs_changed.append(pkg) log.info("Attempting to install %s via easy_install", pkgs) result = ProcessHelper(['easy_install'] + pkgs).call() if result.returncode: log.error("easy_install failed. Output: %s", result.stdout) raise ToolError("Could not successfully install python packages", result.returncode) else: log.info("easy_install installed %s", pkgs) log.debug("easy_install output: %s", result.stdout) return pkgs_changed
def _modify_user(user_name, groups=[], homedir=None): """ Modify a user and return True, else return False """ if not homedir and not groups: log.info("No homedir or groups specified; not modifying %s", user_name) return False cmd = ['/usr/sbin/usermod'] if groups: gids = _get_gids(groups) current_gids = _gids_for_user(user_name) if frozenset(gids) ^ frozenset(current_gids): cmd.extend(['-G', ','.join(gids)]) else: log.debug("Groups have not changed for %s", user_name) if homedir: if homedir != _get_user_homedir(user_name): cmd.extend(['-d', homedir]) else: log.debug("Homedir has not changed for %s", user_name) if len(cmd) == 1: log.debug("User %s does not need modification", user_name) return False cmd.append(user_name) result = ProcessHelper(cmd).call() if result.returncode: log.error("Failed to modify user %s", user_name) log.debug("Usermod output: %s", result.stdout) raise ToolError("Failed to modify user %s" % user_name) else: log.info("Modified user %s successfully", user_name) return True
def _create_user(user_name, groups=[], homedir=None, uid=None): gids = _get_gids(groups) cmd = ['/usr/sbin/useradd', '-M', '-r', '--shell', '/sbin/nologin'] if homedir: cmd.extend(['-d', homedir]) if uid: cmd.extend(['-u', uid]) if gids: cmd.extend(['-G', ','.join(gids)]) cmd.append(user_name) result = ProcessHelper(cmd).call() if result.returncode: log.error("Failed to add user %s", user_name) log.debug("Useradd output: %s", result.stdout) raise ToolError("Failed to add user %s" % user_name) else: log.info("Added user %s successfully", user_name)
def apply(self, action, changes=collections.defaultdict(list)): """ Takes a dict of service name to dict. Keys we look for are: - "enabled" (setting a service to "Automatic") - "ensureRunning" (actually start the service) """ if not action.keys(): log.debug("No Windows services specified") return if not _windows_supported: raise ToolError("Cannot modify windows services without pywin32") manager = win32service.OpenSCManager( None, None, win32service.SC_MANAGER_ALL_ACCESS) try: for service, serviceProperties in action.iteritems(): handle = win32service.OpenService( manager, service, win32service.SERVICE_ALL_ACCESS) try: if "enabled" in serviceProperties: start_type = win32service.SERVICE_AUTO_START if util.interpret_boolean( serviceProperties["enabled"] ) else win32service.SERVICE_DEMAND_START self._set_service_startup_type(handle, start_type) else: log.debug("Not modifying enabled state of service %s", service) if self._detect_required_restart(serviceProperties, changes): log.debug( "Restarting %s due to change detected in dependency", service) win32serviceutil.RestartService(service) elif "ensureRunning" in serviceProperties: ensureRunning = util.interpret_boolean( serviceProperties["ensureRunning"]) status = win32service.QueryServiceStatus(handle)[1] isRunning = status & win32service.SERVICE_RUNNING or status & win32service.SERVICE_START_PENDING if ensureRunning and not isRunning: log.debug( "Starting service %s as it is not running", service) win32service.StartService(handle, None) elif not ensureRunning and isRunning: log.debug("Stopping service %s as it is running", service) win32service.ControlService( handle, win32service.SERVICE_CONTROL_STOP) else: log.debug( "No need to modify running state of service %s", service) else: log.debug("Not modifying running state of service %s", service) finally: win32service.CloseServiceHandle(handle) finally: win32service.CloseServiceHandle(manager)
def apply(self, action): """ Execute a set of commands, returning a list of commands that were executed. Arguments: action -- a dict of command to attributes, where attributes has keys of: command: the command to run (a string or list) cwd: working directory (a string) env: a dictionary of environment variables test: a commmand to run; if it returns zero, the command will run ignoreErrors: if true, ignore errors waitAfterCompletion: # of seconds to wait after completion (or "forever") defaults: a command to run; the stdout will be used to provide defaults Exceptions: ToolError -- on expected failures """ commands_run = [] if not action: log.debug("No commands specified") return commands_run for name in sorted(action.keys()): log.debug(u"Running command %s", name) attributes = action[name] if "defaults" in attributes: log.debug(u"Generating defaults for command %s", name) defaultsResult = ProcessHelper(attributes['defaults'], stderr=subprocess.PIPE).call() log.debug(u"Defaults script for %s output: %s", name, defaultsResult.stdout.decode('utf-8')) if defaultsResult.returncode: log.error(u"Defaults script failed for %s: %s", name, defaultsResult.stderr.decode('utf-8')) raise ToolError(u"Defaults script for command %s failed" % name) default_attrs = attributes default_env = default_attrs.get("env",{}) attributes = json.loads(defaultsResult.stdout) user_env = attributes.get("env",{}) user_env.update(default_env) attributes.update(default_attrs) attributes["env"] = user_env if not "command" in attributes: log.error(u"No command specified for %s", name) raise ToolError(u"%s does not specify the 'command' attribute, which is required" % name) cwd = os.path.expanduser(attributes["cwd"]) if "cwd" in attributes else None env = attributes.get("env", None) if "test" in attributes: log.debug(u"Running test for command %s", name) test = attributes["test"] testResult = LoggingProcessHelper(test, name=u'Test for Command %s' % name, env=env, cwd=cwd).call() log.debug(u"Test command output: %s", testResult.stdout.decode('utf-8')) if testResult.returncode: log.info(u"Test failed with code %s", testResult.returncode) continue else: log.debug(u"Test for command %s passed", name) else: log.debug(u"No test for command %s", name) cmd_to_run = attributes["command"] if "runas" in attributes: if os.name == 'nt': raise ToolError(u'Command %s specified "runas", which is not supported on Windows' % name) if isinstance(cmd_to_run, basestring): cmd_to_run = u'su %s -c %s' % (attributes['runas'], cmd_to_run) else: cmd_to_run = ['su', attributes['runas'], '-c'] + cmd_to_run commandResult = LoggingProcessHelper(cmd_to_run, name=u'Command %s' % name, env=env, cwd=cwd).call() if commandResult.returncode: log.error(u"Command %s (%s) failed", name, attributes["command"]) log.debug(u"Command %s output: %s", name, commandResult.stdout.decode('utf-8')) if interpret_boolean(attributes.get("ignoreErrors")): log.info("ignoreErrors set to true, continuing build") commands_run.append(name) else: raise ToolError(u"Command %s failed" % name) else: log.info(u"Command %s succeeded", name) log.debug(u"Command %s output: %s", name, commandResult.stdout.decode('utf-8')) commands_run.append(name) return commands_run
def apply(self, action, auth_config=None): """ Install a set of packages via yum, returning the packages actually installed or updated. Arguments: action -- a dict of package name to version; version can be empty, a single string or a list of strings Exceptions: ToolError -- on expected failures (such as a non-zero exit code) """ pkgs_changed = [] if not action.keys(): log.debug("No packages specified for yum") return pkgs_changed cache_result = ProcessHelper(['yum', '-y', 'makecache']).call() if cache_result.returncode: log.error("Yum makecache failed. Output: %s", cache_result.stdout) raise ToolError("Could not create yum cache", cache_result.returncode) pkg_specs_to_upgrade = [] pkg_specs_to_downgrade = [] for pkg_name in action: if action[pkg_name]: if isinstance(action[pkg_name], basestring): pkg_ver = action[pkg_name] else: # Yum only cares about one version anyway... so take the max specified version in the list pkg_ver = RpmTool.max_version(action[pkg_name]) else: pkg_ver = None pkg_spec = '%s-%s' % (pkg_name, pkg_ver) if pkg_ver else pkg_name if self._pkg_installed(pkg_spec): # If the EXACT requested spec is installed, don't do anything log.debug("%s will not be installed as it is already present", pkg_spec) elif not self._pkg_available(pkg_spec): # If the requested spec is not available, blow up log.error("%s is not available to be installed", pkg_spec) raise ToolError( "Yum does not have %s available for installation" % pkg_spec) elif not pkg_ver: # If they didn't request a specific version, always upgrade pkg_specs_to_upgrade.append(pkg_spec) pkgs_changed.append(pkg_name) else: # They've requested a specific version that's available but not installed. # Figure out if it's an upgrade or a downgrade installed_version = RpmTool.get_package_version( pkg_name, False)[1] if self._should_upgrade(pkg_ver, installed_version): pkg_specs_to_upgrade.append(pkg_spec) pkgs_changed.append(pkg_name) else: log.debug("Downgrading to %s from installed version %s", pkg_spec, installed_version) pkg_specs_to_downgrade.append(pkg_spec) pkgs_changed.append(pkg_name) if not pkgs_changed: log.debug("All yum packages were already installed") return [] if pkg_specs_to_upgrade: log.debug("Installing/updating %s via yum", pkg_specs_to_upgrade) result = ProcessHelper(['yum', '-y', 'install'] + pkg_specs_to_upgrade).call() if result.returncode: log.error("Yum failed. Output: %s", result.stdout) raise ToolError( "Could not successfully install/update yum packages", result.returncode) if pkg_specs_to_downgrade: log.debug("Downgrading %s via yum", pkg_specs_to_downgrade) result = ProcessHelper(['yum', '-y', 'downgrade'] + pkg_specs_to_downgrade).call() if result.returncode: log.error("Yum failed. Output: %s", result.stdout) raise ToolError( "Could not successfully downgrade yum packages", result.returncode) log.info("Yum installed %s", pkgs_changed) return pkgs_changed