def call_process(self, cmd, stdout=False): """ Execute a shell command. Args: cmd: (:obj:`list`) Command to execute. stdout: (:obj:`bool`) Switch to control whether to return stdout. (*Default*: ``False``) Returns: :obj:`None` or :obj:`bytes`: ``None`` unless ``stdout`` is ``True``. In that case, the stdout is returned as a bytes object. """ ret = None stdout_ret = b'' stderr_ret = b'' # pylint: disable=unused-variable if not isinstance(cmd, list): msg = 'Command is not a list: {0}'.format(cmd) self.log.critical(msg) raise WatchmakerException(msg) self.log.debug('Running command: %s', cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: stdout_future = executor.submit(self._pipe_logger, process.stdout, self.log.debug, 'Command stdout: ', stdout) stderr_future = executor.submit(self._pipe_logger, process.stderr, self.log.error, 'Command stderr: ') stdout_ret = stdout_future.result() stderr_ret = stderr_future.result() # noqa: F841,E501 # pylint: disable=unused-variable returncode = process.wait() if returncode != 0: msg = 'Command failed! Exit code={0}, cmd={1}'.format( process.returncode, cmd) self.log.critical(msg) raise WatchmakerException(msg) if stdout: # Return stdout ret = stdout_ret return ret
def get_dist_info(self): """Validate the Linux distro and return info about the distribution.""" dist = None version = None el_version = None # Read first line from /etc/system-release try: with open(name='/etc/system-release', mode='rb') as fh_: release = fh_.readline().strip() except Exception: self.log.critical( 'Failed to read /etc/system-release. Cannot determine system ' 'distribution!' ) raise # Search the release file for a match against _supported_dists matched = self.DIST_PATTERN.search(release.lower()) if matched is None: # Release not supported, exit with error msg = ( 'Unsupported OS distribution. OS must be one of: {0}' .format(', '.join(self.SUPPORTED_DISTS)) ) self.log.critical(msg) raise WatchmakerException(msg) # Assign dist,version from the match groups tuple, removing any spaces dist, version = (x.replace(' ', '') for x in matched.groups()) # Determine el_version if dist == 'amazon': el_version = self._get_amazon_el_version(version) else: el_version = version.split('.')[0] if el_version is None: msg = ( 'Unsupported OS version! dist = {0}, version = {1}.' .format(dist, version) ) self.log.critical(msg) raise WatchmakerException(msg) dist_info = { 'dist': dist, 'el_version': el_version } self.log.debug('dist_info=%s', dist_info) return dist_info
def _validate_config(self): """Validate the config is properly formed.""" if not self.yumrepomap: self.log.warning('`yumrepomap` did not exist or was empty.') elif not isinstance(self.yumrepomap, list): msg = '`yumrepomap` must be a list!' self.log.critical(msg) raise WatchmakerException(msg)
def process_states(self, states, exclude): """ Apply salt states but exclude certain states. Args: states: (:obj:`str`) Comma-separated string of salt states to execute. Accepts two special keywords (case-insensitive): - ``none``: Do not apply any salt states. - ``highstate``: Apply the salt "highstate". exclude: (:obj:`str`) Comma-separated string of states to exclude from execution. """ if not states: self.log.info( 'No States were specified. Will not apply any salt states.' ) else: cmd = self.salt_state_args if states.lower() == 'highstate': self.log.info( 'Applying the salt "highstate", states=%s', states ) cmd.extend(['state.highstate']) else: self.log.info( 'Applying the user-defined list of states, states=%s', states ) cmd.extend(['state.sls', states]) if exclude: cmd.extend(['exclude={0}'.format(exclude)]) ret = self.run_salt(cmd, log_pipe='stderr', raise_error=False) if ret['retcode'] != 0: failed_states = self._get_failed_states( ast.literal_eval(ret['stdout'].decode('utf-8'))) if failed_states: raise WatchmakerException( yaml.safe_dump( { 'Salt state execution failed': failed_states }, default_flow_style=False, indent=4 ) ) self.log.info('Salt states all applied successfully!')
def _set_system_params(self): """Set OS-specific attributes.""" if 'linux' in self.system: self.system_drive = '/' self.workers_manager = LinuxWorkersManager self.system_params = self._get_linux_system_params() elif 'windows' in self.system: self.system_drive = os.environ['SYSTEMDRIVE'] self.workers_manager = WindowsWorkersManager self.system_params = self._get_windows_system_params() else: msg = 'System, {0}, is not recognized?'.format(self.system) self.log.critical(msg) raise WatchmakerException(msg) if self.log_dir: self.system_params['logdir'] = self.log_dir
def _get_config(self): """ Read and validate configuration data for installation. Returns: :obj:`collections.OrderedDict`: Returns the data from the the YAML configuration file, scoped to the value of ``self.system`` and merged with the value of the ``"All"`` key. """ if not self.config_path: self.log.warning( 'User did not supply a config. Using the default config.') self.config_path = self.default_config else: self.log.info('User supplied config being used.') # Convert a local config path to a URI self.config_path = watchmaker.utils.uri_from_filepath(self.config_path) # Get the raw config data data = '' try: data = watchmaker.utils.urlopen_retry(self.config_path).read() except (ValueError, urllib.error.URLError): msg = ('Could not read config file from the provided value "{0}"! ' 'Check that the config is available.'.format( self.config_path)) self.log.critical(msg) raise config_full = yaml.safe_load(data) try: config_all = config_full.get('all', []) config_system = config_full.get(self.system, []) except AttributeError: msg = 'Malformed config file. Must be a dictionary.' self.log.critical(msg) raise # If both config and config_system are empty, raise if not config_system and not config_all: msg = 'Malformed config file. No workers for this system.' self.log.critical(msg) raise WatchmakerException(msg) # Merge the config data, preserving the listed order of workers config = collections.OrderedDict() for worker in config_system + config_all: try: # worker is a single-key dict, where the key is the name of the # worker and the value is the worker parameters. we need to # test if the worker is already in the config, but a dict is # is not hashable so cannot be tested directly with # `if worker not in config`. this bit of ugliness extracts the # key and its value so we can use them directly. worker_name, worker_config = list(worker.items())[0] if worker_name not in config: # Add worker to config config[worker_name] = {'config': worker_config} self.log.debug('%s config: %s', worker_name, worker_config) else: # Worker present in both config_system and config_all config[worker_name]['config'].update(worker_config) self.log.debug('%s extra config: %s', worker_name, worker_config) # Need to (re)merge cli worker args so they override config[worker_name]['__merged'] = False if not config[worker_name].get('__merged'): # Merge worker_args into config params config[worker_name]['config'].update(self.worker_args) config[worker_name]['__merged'] = True except Exception: msg = ( 'Failed to merge worker config; worker={0}'.format(worker)) self.log.critical(msg) raise self.log.debug('Command-line arguments merged into worker configs: %s', self.worker_args) return config
def extract_contents(self, filepath, to_directory, create_dir=False): """ Extract a compressed archive to the specified directory. Args: filepath: (:obj:`str`) Path to the compressed file. Supported file extensions: - `.zip` - `.tar.gz` - `.tgz` - `.tar.bz2` - `.tbz` to_directory: (:obj:`str`) Path to the target directory create_dir: (:obj:`bool`) Switch to control the creation of a subdirectory within ``to_directory`` named for the filename of the compressed file. (*Default*: ``False``) """ if filepath.endswith('.zip'): self.log.debug('File Type: zip') opener, mode = zipfile.ZipFile, 'r' elif filepath.endswith('.tar.gz') or filepath.endswith('.tgz'): self.log.debug('File Type: GZip Tar') opener, mode = tarfile.open, 'r:gz' elif filepath.endswith('.tar.bz2') or filepath.endswith('.tbz'): self.log.debug('File Type: Bzip Tar') opener, mode = tarfile.open, 'r:bz2' else: msg = ( 'Could not extract "{0}" as no appropriate extractor is found.' .format(filepath)) self.log.critical(msg) raise WatchmakerException(msg) if create_dir: to_directory = os.sep.join( (to_directory, '.'.join(filepath.split(os.sep)[-1].split('.')[:-1]))) try: os.makedirs(to_directory) except OSError: if not os.path.isdir(to_directory): msg = 'Unable create directory - {0}'.format(to_directory) self.log.critical(msg) raise cwd = os.getcwd() os.chdir(to_directory) try: openfile = opener(filepath, mode) try: openfile.extractall() finally: openfile.close() finally: os.chdir(cwd) self.log.info('Extracted file. source=%s, dest=%s', filepath, to_directory)
def _build_salt_formula(self, extract_dir): if self.salt_content: salt_content_filename = watchmaker.utils.basename_from_uri( self.salt_content ) salt_content_file = os.sep.join(( self.working_dir, salt_content_filename )) self.retrieve_file(self.salt_content, salt_content_file) if not self.salt_content_path: self.extract_contents( filepath=salt_content_file, to_directory=extract_dir ) else: self.log.debug( 'Using salt content path: %s', self.salt_content_path ) temp_extract_dir = os.sep.join(( self.working_dir, 'salt-archive')) self.extract_contents( filepath=salt_content_file, to_directory=temp_extract_dir ) salt_content_src = os.sep.join(( temp_extract_dir, self.salt_content_path)) salt_content_glob = glob.glob(salt_content_src) self.log.debug('salt_content_glob: %s', salt_content_glob) if len(salt_content_glob) > 1: msg = 'Found multiple paths matching' \ ' \'{0}\' in {1}'.format( self.salt_content_path, self.salt_content) self.log.critical(msg) raise WatchmakerException(msg) try: salt_files_dir = salt_content_glob[0] except IndexError: msg = 'Salt content glob path \'{0}\' not' \ ' found in {1}'.format( self.salt_content_path, self.salt_content) self.log.critical(msg) raise WatchmakerException(msg) watchmaker.utils.copy_subdirectories( salt_files_dir, extract_dir, self.log) bundled_content = os.sep.join( (static.__path__[0], 'salt', 'content') ) watchmaker.utils.copy_subdirectories( bundled_content, extract_dir, self.log) with codecs.open( os.path.join(self.salt_conf_path, 'minion'), 'r+', encoding="utf-8" ) as fh_: salt_conf = yaml.safe_load(fh_) salt_conf.update(self.salt_file_roots) fh_.seek(0) yaml.safe_dump(salt_conf, fh_, default_flow_style=False)
def call_process(self, cmd, log_pipe='all', raise_error=True): """ Execute a shell command. Args: cmd: (:obj:`list`) Command to execute. log_pipe: (:obj:`str`) Controls what to log from the command output. Supports three values: ``stdout``, ``stderr``, ``all``. (*Default*: ``all``) raise_error: (:obj:`bool`) Switch to control whether to raise if the command return code is non-zero. (*Default*: ``True``) Returns: :obj:`dict`: Dictionary containing three keys: ``retcode`` (:obj:`int`), ``stdout`` (:obj:`bytes`), and ``stderr`` (:obj:`bytes`). """ ret = {'retcode': 0, 'stdout': b'', 'stderr': b''} if not isinstance(cmd, list): msg = 'Command is not a list: {0}'.format(cmd) self.log.critical(msg) raise WatchmakerException(msg) self.log.debug('Command: %s', ' '.join(cmd)) # If running as a standalone, PyInstaller will have modified the # LD_LIBRARY_PATH to point to standalone libraries. If there were a # value at runtime, PyInstaller will create LD_LIBRARY_PATH_ORIG. In # order for salt to run correctly, LD_LIBRARY_PATH has to be fixed. kwargs = {} env = dict(os.environ) lib_path_key = 'LD_LIBRARY_PATH' if env.get(lib_path_key) is not None: lib_path_orig_value = env.get(lib_path_key + '_ORIG') if lib_path_orig_value is None: # you can have lib_path and no orig, if: # 1. none was set and pyinstaller set one, or # 2. one was set and we're not in standalone package env.pop(lib_path_key, None) else: # put original lib_path back env[lib_path_key] = lib_path_orig_value kwargs['env'] = env process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: stdout_future = executor.submit( self._pipe_handler, process.stdout, self.log.debug if log_pipe in ['stdout', 'all'] else None, 'Command stdout: ') stderr_future = executor.submit( self._pipe_handler, process.stderr, self.log.error if log_pipe in ['stderr', 'all'] else None, 'Command stderr: ') ret['stdout'] = stdout_future.result() ret['stderr'] = stderr_future.result() ret['retcode'] = process.wait() self.log.debug('Command retcode: %s', ret['retcode']) if raise_error and ret['retcode'] != 0: msg = 'Command failed! Exit code={0}, cmd={1}'.format( ret['retcode'], ' '.join(cmd)) self.log.critical(msg) raise WatchmakerException(msg) return ret
def call_process(self, cmd, log_pipe='all', raise_error=True): """ Execute a shell command. Args: cmd: (:obj:`list`) Command to execute. log_pipe: (:obj:`str`) Controls what to log from the command output. Supports three values: ``stdout``, ``stderr``, ``all``. (*Default*: ``all``) raise_error: (:obj:`bool`) Switch to control whether to raise if the command return code is non-zero. (*Default*: ``True``) Returns: :obj:`dict`: Dictionary containing three keys: ``retcode`` (:obj:`int`), ``stdout`` (:obj:`bytes`), and ``stderr`` (:obj:`bytes`). """ ret = {'retcode': 0, 'stdout': b'', 'stderr': b''} if not isinstance(cmd, list): msg = 'Command is not a list: {0}'.format(cmd) self.log.critical(msg) raise WatchmakerException(msg) self.log.debug('Command: %s', ' '.join(cmd)) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: stdout_future = executor.submit( self._pipe_handler, process.stdout, self.log.debug if log_pipe in ['stdout', 'all'] else None, 'Command stdout: ') stderr_future = executor.submit( self._pipe_handler, process.stderr, self.log.error if log_pipe in ['stderr', 'all'] else None, 'Command stderr: ') ret['stdout'] = stdout_future.result() ret['stderr'] = stderr_future.result() ret['retcode'] = process.wait() self.log.debug('Command retcode: %s', ret['retcode']) if raise_error and ret['retcode'] != 0: msg = 'Command failed! Exit code={0}, cmd={1}'.format( ret['retcode'], ' '.join(cmd)) self.log.critical(msg) raise WatchmakerException(msg) return ret