def call_process(self, cmd, stdout=False):
"""
Execute a shell command.
Args:
cmd: (:obj:`list`)
Command to execute.
stdout: (:obj:`bool`)
Switch to control whether to return stdout.
(*Default*: ``False``)
Returns:
:obj:`None` or :obj:`bytes`:
``None`` unless ``stdout`` is ``True``. In that case, the
stdout is returned as a bytes object.
"""
ret = None
stdout_ret = b''
stderr_ret = b'' # pylint: disable=unused-variable
if not isinstance(cmd, list):
msg = 'Command is not a list: {0}'.format(cmd)
self.log.critical(msg)
raise WatchmakerException(msg)
self.log.debug('Running command: %s', cmd)
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
stdout_future = executor.submit(self._pipe_logger, process.stdout,
self.log.debug, 'Command stdout: ',
stdout)
stderr_future = executor.submit(self._pipe_logger, process.stderr,
self.log.error, 'Command stderr: ')
stdout_ret = stdout_future.result()
stderr_ret = stderr_future.result() # noqa: F841,E501 # pylint: disable=unused-variable
returncode = process.wait()
if returncode != 0:
msg = 'Command failed! Exit code={0}, cmd={1}'.format(
process.returncode, cmd)
self.log.critical(msg)
raise WatchmakerException(msg)
if stdout:
# Return stdout
ret = stdout_ret
return ret
def get_dist_info(self):
"""Validate the Linux distro and return info about the distribution."""
dist = None
version = None
el_version = None
# Read first line from /etc/system-release
try:
with open(name='/etc/system-release', mode='rb') as fh_:
release = fh_.readline().strip()
except Exception:
self.log.critical(
'Failed to read /etc/system-release. Cannot determine system '
'distribution!'
)
raise
# Search the release file for a match against _supported_dists
matched = self.DIST_PATTERN.search(release.lower())
if matched is None:
# Release not supported, exit with error
msg = (
'Unsupported OS distribution. OS must be one of: {0}'
.format(', '.join(self.SUPPORTED_DISTS))
)
self.log.critical(msg)
raise WatchmakerException(msg)
# Assign dist,version from the match groups tuple, removing any spaces
dist, version = (x.replace(' ', '') for x in matched.groups())
# Determine el_version
if dist == 'amazon':
el_version = self._get_amazon_el_version(version)
else:
el_version = version.split('.')[0]
if el_version is None:
msg = (
'Unsupported OS version! dist = {0}, version = {1}.'
.format(dist, version)
)
self.log.critical(msg)
raise WatchmakerException(msg)
dist_info = {
'dist': dist,
'el_version': el_version
}
self.log.debug('dist_info=%s', dist_info)
return dist_info
def _validate_config(self):
"""Validate the config is properly formed."""
if not self.yumrepomap:
self.log.warning('`yumrepomap` did not exist or was empty.')
elif not isinstance(self.yumrepomap, list):
msg = '`yumrepomap` must be a list!'
self.log.critical(msg)
raise WatchmakerException(msg)
def process_states(self, states, exclude):
"""
Apply salt states but exclude certain states.
Args:
states: (:obj:`str`)
Comma-separated string of salt states to execute. Accepts two
special keywords (case-insensitive):
- ``none``: Do not apply any salt states.
- ``highstate``: Apply the salt "highstate".
exclude: (:obj:`str`)
Comma-separated string of states to exclude from execution.
"""
if not states:
self.log.info(
'No States were specified. Will not apply any salt states.'
)
else:
cmd = self.salt_state_args
if states.lower() == 'highstate':
self.log.info(
'Applying the salt "highstate", states=%s',
states
)
cmd.extend(['state.highstate'])
else:
self.log.info(
'Applying the user-defined list of states, states=%s',
states
)
cmd.extend(['state.sls', states])
if exclude:
cmd.extend(['exclude={0}'.format(exclude)])
ret = self.run_salt(cmd, log_pipe='stderr', raise_error=False)
if ret['retcode'] != 0:
failed_states = self._get_failed_states(
ast.literal_eval(ret['stdout'].decode('utf-8')))
if failed_states:
raise WatchmakerException(
yaml.safe_dump(
{
'Salt state execution failed':
failed_states
},
default_flow_style=False,
indent=4
)
)
self.log.info('Salt states all applied successfully!')
def _set_system_params(self):
"""Set OS-specific attributes."""
if 'linux' in self.system:
self.system_drive = '/'
self.workers_manager = LinuxWorkersManager
self.system_params = self._get_linux_system_params()
elif 'windows' in self.system:
self.system_drive = os.environ['SYSTEMDRIVE']
self.workers_manager = WindowsWorkersManager
self.system_params = self._get_windows_system_params()
else:
msg = 'System, {0}, is not recognized?'.format(self.system)
self.log.critical(msg)
raise WatchmakerException(msg)
if self.log_dir:
self.system_params['logdir'] = self.log_dir
def _get_config(self):
"""
Read and validate configuration data for installation.
Returns:
:obj:`collections.OrderedDict`: Returns the data from the the YAML
configuration file, scoped to the value of ``self.system`` and
merged with the value of the ``"All"`` key.
"""
if not self.config_path:
self.log.warning(
'User did not supply a config. Using the default config.')
self.config_path = self.default_config
else:
self.log.info('User supplied config being used.')
# Convert a local config path to a URI
self.config_path = watchmaker.utils.uri_from_filepath(self.config_path)
# Get the raw config data
data = ''
try:
data = watchmaker.utils.urlopen_retry(self.config_path).read()
except (ValueError, urllib.error.URLError):
msg = ('Could not read config file from the provided value "{0}"! '
'Check that the config is available.'.format(
self.config_path))
self.log.critical(msg)
raise
config_full = yaml.safe_load(data)
try:
config_all = config_full.get('all', [])
config_system = config_full.get(self.system, [])
except AttributeError:
msg = 'Malformed config file. Must be a dictionary.'
self.log.critical(msg)
raise
# If both config and config_system are empty, raise
if not config_system and not config_all:
msg = 'Malformed config file. No workers for this system.'
self.log.critical(msg)
raise WatchmakerException(msg)
# Merge the config data, preserving the listed order of workers
config = collections.OrderedDict()
for worker in config_system + config_all:
try:
# worker is a single-key dict, where the key is the name of the
# worker and the value is the worker parameters. we need to
# test if the worker is already in the config, but a dict is
# is not hashable so cannot be tested directly with
# `if worker not in config`. this bit of ugliness extracts the
# key and its value so we can use them directly.
worker_name, worker_config = list(worker.items())[0]
if worker_name not in config:
# Add worker to config
config[worker_name] = {'config': worker_config}
self.log.debug('%s config: %s', worker_name, worker_config)
else:
# Worker present in both config_system and config_all
config[worker_name]['config'].update(worker_config)
self.log.debug('%s extra config: %s', worker_name,
worker_config)
# Need to (re)merge cli worker args so they override
config[worker_name]['__merged'] = False
if not config[worker_name].get('__merged'):
# Merge worker_args into config params
config[worker_name]['config'].update(self.worker_args)
config[worker_name]['__merged'] = True
except Exception:
msg = (
'Failed to merge worker config; worker={0}'.format(worker))
self.log.critical(msg)
raise
self.log.debug('Command-line arguments merged into worker configs: %s',
self.worker_args)
return config
def extract_contents(self, filepath, to_directory, create_dir=False):
"""
Extract a compressed archive to the specified directory.
Args:
filepath: (:obj:`str`)
Path to the compressed file. Supported file extensions:
- `.zip`
- `.tar.gz`
- `.tgz`
- `.tar.bz2`
- `.tbz`
to_directory: (:obj:`str`)
Path to the target directory
create_dir: (:obj:`bool`)
Switch to control the creation of a subdirectory within
``to_directory`` named for the filename of the compressed file.
(*Default*: ``False``)
"""
if filepath.endswith('.zip'):
self.log.debug('File Type: zip')
opener, mode = zipfile.ZipFile, 'r'
elif filepath.endswith('.tar.gz') or filepath.endswith('.tgz'):
self.log.debug('File Type: GZip Tar')
opener, mode = tarfile.open, 'r:gz'
elif filepath.endswith('.tar.bz2') or filepath.endswith('.tbz'):
self.log.debug('File Type: Bzip Tar')
opener, mode = tarfile.open, 'r:bz2'
else:
msg = (
'Could not extract "{0}" as no appropriate extractor is found.'
.format(filepath))
self.log.critical(msg)
raise WatchmakerException(msg)
if create_dir:
to_directory = os.sep.join(
(to_directory,
'.'.join(filepath.split(os.sep)[-1].split('.')[:-1])))
try:
os.makedirs(to_directory)
except OSError:
if not os.path.isdir(to_directory):
msg = 'Unable create directory - {0}'.format(to_directory)
self.log.critical(msg)
raise
cwd = os.getcwd()
os.chdir(to_directory)
try:
openfile = opener(filepath, mode)
try:
openfile.extractall()
finally:
openfile.close()
finally:
os.chdir(cwd)
self.log.info('Extracted file. source=%s, dest=%s', filepath,
to_directory)
def _build_salt_formula(self, extract_dir):
if self.salt_content:
salt_content_filename = watchmaker.utils.basename_from_uri(
self.salt_content
)
salt_content_file = os.sep.join((
self.working_dir,
salt_content_filename
))
self.retrieve_file(self.salt_content, salt_content_file)
if not self.salt_content_path:
self.extract_contents(
filepath=salt_content_file,
to_directory=extract_dir
)
else:
self.log.debug(
'Using salt content path: %s',
self.salt_content_path
)
temp_extract_dir = os.sep.join((
self.working_dir, 'salt-archive'))
self.extract_contents(
filepath=salt_content_file,
to_directory=temp_extract_dir
)
salt_content_src = os.sep.join((
temp_extract_dir, self.salt_content_path))
salt_content_glob = glob.glob(salt_content_src)
self.log.debug('salt_content_glob: %s', salt_content_glob)
if len(salt_content_glob) > 1:
msg = 'Found multiple paths matching' \
' \'{0}\' in {1}'.format(
self.salt_content_path,
self.salt_content)
self.log.critical(msg)
raise WatchmakerException(msg)
try:
salt_files_dir = salt_content_glob[0]
except IndexError:
msg = 'Salt content glob path \'{0}\' not' \
' found in {1}'.format(
self.salt_content_path,
self.salt_content)
self.log.critical(msg)
raise WatchmakerException(msg)
watchmaker.utils.copy_subdirectories(
salt_files_dir, extract_dir, self.log)
bundled_content = os.sep.join(
(static.__path__[0], 'salt', 'content')
)
watchmaker.utils.copy_subdirectories(
bundled_content, extract_dir, self.log)
with codecs.open(
os.path.join(self.salt_conf_path, 'minion'),
'r+',
encoding="utf-8"
) as fh_:
salt_conf = yaml.safe_load(fh_)
salt_conf.update(self.salt_file_roots)
fh_.seek(0)
yaml.safe_dump(salt_conf, fh_, default_flow_style=False)
def call_process(self, cmd, log_pipe='all', raise_error=True):
"""
Execute a shell command.
Args:
cmd: (:obj:`list`)
Command to execute.
log_pipe: (:obj:`str`)
Controls what to log from the command output. Supports three
values: ``stdout``, ``stderr``, ``all``.
(*Default*: ``all``)
raise_error: (:obj:`bool`)
Switch to control whether to raise if the command return code
is non-zero.
(*Default*: ``True``)
Returns:
:obj:`dict`:
Dictionary containing three keys: ``retcode`` (:obj:`int`),
``stdout`` (:obj:`bytes`), and ``stderr`` (:obj:`bytes`).
"""
ret = {'retcode': 0, 'stdout': b'', 'stderr': b''}
if not isinstance(cmd, list):
msg = 'Command is not a list: {0}'.format(cmd)
self.log.critical(msg)
raise WatchmakerException(msg)
self.log.debug('Command: %s', ' '.join(cmd))
# If running as a standalone, PyInstaller will have modified the
# LD_LIBRARY_PATH to point to standalone libraries. If there were a
# value at runtime, PyInstaller will create LD_LIBRARY_PATH_ORIG. In
# order for salt to run correctly, LD_LIBRARY_PATH has to be fixed.
kwargs = {}
env = dict(os.environ)
lib_path_key = 'LD_LIBRARY_PATH'
if env.get(lib_path_key) is not None:
lib_path_orig_value = env.get(lib_path_key + '_ORIG')
if lib_path_orig_value is None:
# you can have lib_path and no orig, if:
# 1. none was set and pyinstaller set one, or
# 2. one was set and we're not in standalone package
env.pop(lib_path_key, None)
else:
# put original lib_path back
env[lib_path_key] = lib_path_orig_value
kwargs['env'] = env
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**kwargs)
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
stdout_future = executor.submit(
self._pipe_handler, process.stdout,
self.log.debug if log_pipe in ['stdout', 'all'] else None,
'Command stdout: ')
stderr_future = executor.submit(
self._pipe_handler, process.stderr,
self.log.error if log_pipe in ['stderr', 'all'] else None,
'Command stderr: ')
ret['stdout'] = stdout_future.result()
ret['stderr'] = stderr_future.result()
ret['retcode'] = process.wait()
self.log.debug('Command retcode: %s', ret['retcode'])
if raise_error and ret['retcode'] != 0:
msg = 'Command failed! Exit code={0}, cmd={1}'.format(
ret['retcode'], ' '.join(cmd))
self.log.critical(msg)
raise WatchmakerException(msg)
return ret
def call_process(self, cmd, log_pipe='all', raise_error=True):
"""
Execute a shell command.
Args:
cmd: (:obj:`list`)
Command to execute.
log_pipe: (:obj:`str`)
Controls what to log from the command output. Supports three
values: ``stdout``, ``stderr``, ``all``.
(*Default*: ``all``)
raise_error: (:obj:`bool`)
Switch to control whether to raise if the command return code
is non-zero.
(*Default*: ``True``)
Returns:
:obj:`dict`:
Dictionary containing three keys: ``retcode`` (:obj:`int`),
``stdout`` (:obj:`bytes`), and ``stderr`` (:obj:`bytes`).
"""
ret = {'retcode': 0, 'stdout': b'', 'stderr': b''}
if not isinstance(cmd, list):
msg = 'Command is not a list: {0}'.format(cmd)
self.log.critical(msg)
raise WatchmakerException(msg)
self.log.debug('Command: %s', ' '.join(cmd))
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
stdout_future = executor.submit(
self._pipe_handler, process.stdout,
self.log.debug if log_pipe in ['stdout', 'all'] else None,
'Command stdout: ')
stderr_future = executor.submit(
self._pipe_handler, process.stderr,
self.log.error if log_pipe in ['stderr', 'all'] else None,
'Command stderr: ')
ret['stdout'] = stdout_future.result()
ret['stderr'] = stderr_future.result()
ret['retcode'] = process.wait()
self.log.debug('Command retcode: %s', ret['retcode'])
if raise_error and ret['retcode'] != 0:
msg = 'Command failed! Exit code={0}, cmd={1}'.format(
ret['retcode'], ' '.join(cmd))
self.log.critical(msg)
raise WatchmakerException(msg)
return ret
