class MapMeta(six.with_metaclass(Prepareable, type)):
"""
This is the metaclass for our Map class, used for building data maps based
off of grain data.
"""
@classmethod
def __prepare__(metacls, name, bases):
return OrderedDict()
def __new__(cls, name, bases, attrs):
c = type.__new__(cls, name, bases, attrs)
c.__ordered_attrs__ = attrs.keys()
return c
def __init__(cls, name, bases, nmspc):
cls.__set_attributes__() # pylint: disable=no-value-for-parameter
super(MapMeta, cls).__init__(name, bases, nmspc)
def __set_attributes__(cls):
match_info = []
grain_targets = set()
# find all of our filters
for item in cls.__ordered_attrs__:
if item[0] == "_":
continue
filt = cls.__dict__[item]
# only process classes
if not inspect.isclass(filt):
continue
# which grain are we filtering on
grain = getattr(filt, "__grain__", "os_family")
grain_targets.add(grain)
# does the object pointed to have a __match__ attribute?
# if so use it, otherwise use the name of the object
# this is so that you can match complex values, which the python
# class name syntax does not allow
match = getattr(filt, "__match__", item)
match_attrs = {}
for name in filt.__dict__:
if name[0] != "_":
match_attrs[name] = filt.__dict__[name]
match_info.append((grain, match, match_attrs))
# Reorder based on priority
try:
if not hasattr(cls.priority, "__iter__"):
log.error("pyobjects: priority must be an iterable")
else:
new_match_info = []
for grain in cls.priority:
# Using list() here because we will be modifying
# match_info during iteration
for index, item in list(enumerate(match_info)):
try:
if item[0] == grain:
# Add item to new list
new_match_info.append(item)
# Clear item from old list
match_info[index] = None
except TypeError:
# Already moved this item to new list
pass
# Add in any remaining items not defined in priority
new_match_info.extend([x for x in match_info if x is not None])
# Save reordered list as the match_info list
match_info = new_match_info
except AttributeError:
pass
# Check for matches and update the attrs dict accordingly
attrs = {}
if match_info:
grain_vals = Map.__salt__["grains.item"](*grain_targets)
for grain, match, match_attrs in match_info:
if grain not in grain_vals:
continue
if grain_vals[grain] == match:
attrs.update(match_attrs)
if hasattr(cls, "merge"):
pillar = Map.__salt__["pillar.get"](cls.merge)
if pillar:
attrs.update(pillar)
for name in attrs:
setattr(cls, name, attrs[name])
class Map(six.with_metaclass(MapMeta, object)): # pylint: disable=W0232
__salt__ = {"grains.filter_by": need_salt, "pillar.get": need_salt}
class TestProgram(six.with_metaclass(TestProgramMeta, object)):
'''
Set up an arbitrary executable to run.
:attribute dirtree: An iterable of directories to be created
'''
empty_config = ''
config_file = ''
config_attrs = set([
'name',
'test_dir',
'config_dirs',
])
config_vals = {}
config_base = ''
config_dir = os.path.join('etc')
configs = {}
config_types = (
str,
six.string_types,
)
dirtree = [
'&config_dirs',
]
@staticmethod
def config_caster(cfg):
return str(cfg)
def __init__(self,
program=None,
name=None,
env=None,
shell=False,
parent_dir=None,
clean_on_exit=True,
**kwargs):
self.program = program or getattr(self, 'program', None)
self.name = name or getattr(self, 'name', '')
self.env = env or {}
self.shell = shell
self._parent_dir = parent_dir or None
self.clean_on_exit = clean_on_exit
self._root_dir = kwargs.pop('root_dir', self.name)
self.config_dir = kwargs.pop('config_dir', copy.copy(self.config_dir))
config_attrs = copy.copy(self.config_attrs)
config_attrs.update(kwargs.pop('config_attrs', set()))
self.config_attrs = config_attrs
config_vals = copy.copy(self.config_vals)
config_vals.update(kwargs.pop('config_vals', {}))
self.config_vals = config_vals
config_base = copy.deepcopy(self.config_base)
config_base = self.config_merge(
config_base, kwargs.pop('config_base', self.config_types[0]()))
self.config_base = config_base
configs = copy.deepcopy(self.configs)
for cname, cinfo in kwargs.pop('configs', {}).items():
target = configs.setdefault(cname, {})
if 'path' in cinfo:
target['path'] = cinfo['path']
if 'map' in cinfo:
target_map = target.setdefault('map', self.config_types[0]())
target_map = self.config_merge(target_map, cinfo['map'])
target['map'] = target_map
self.configs = configs
if not self.name:
if not self.program:
raise ValueError(
'"{0}" object must specify "program" parameter'.format(
self.__class__.__name__))
self.name = os.path.basename(self.program)
self.process = None
self.created_parent_dir = False
self._setup_done = False
dirtree = set(self.dirtree)
dirtree.update(kwargs.pop('dirtree', []))
self.dirtree = dirtree
# Register the exit clean-up before making anything needing clean-up
atexit.register(self.cleanup)
def __enter__(self):
pass
def __exit__(self, typ, value, traceback):
pass
@property
def test_dir(self):
'''Directory that will contains all of the static and dynamic files for the daemon'''
return os.path.join(self.parent_dir, self._root_dir)
def config_file_get(self, config):
'''Get the filename (viz. path) to the configuration file'''
cfgf = self.configs[config].get('path')
if cfgf:
cfgf.format(**self.config_subs())
else:
cfgf = os.path.join(self.config_dir, config)
return cfgf
def config_dir_get(self, config):
'''Get the parent directory for the configuration file'''
return os.path.dirname(self.config_file_get(config))
@property
def config_dirs(self):
'''Return a list of configuration directories'''
cdirs = [self.config_dir_get(config) for config in self.configs.keys()]
return cdirs
def abs_path(self, path):
'''Absolute path of file including the test_dir'''
return os.path.join(self.test_dir, path)
@property
def start_pid(self):
'''PID of the called script prior to deamonizing.'''
return self.process.pid if self.process else None
@property
def parent_dir(self):
'''
Directory that contains everything generated for running scripts - possibly
for multiple scripts.
'''
if self._parent_dir is None:
self.created_parent_dir = True
self._parent_dir = tempfile.mkdtemp(prefix='salt-testdaemon-')
else:
self._parent_dir = os.path.abspath(
os.path.normpath(self._parent_dir))
if not os.path.exists(self._parent_dir):
self.created_parent_dir = True
os.makedirs(self._parent_dir)
elif not os.path.isdir(self._parent_dir):
raise ValueError(
'Parent path "{0}" exists but is not a directory'.format(
self._parent_dir))
return self._parent_dir
def config_write(self, config):
'''Write out the config to a file'''
if not config:
return
cpath = self.abs_path(self.config_file_get(config))
with open(cpath, 'w') as cfo:
cfg = self.config_stringify(config)
log.debug('Writing configuration for {0} to {1}:\n{2}'.format(
self.name, cpath, cfg))
cfo.write(cfg)
cfo.flush()
def configs_write(self):
'''Write all configuration files'''
for config in self.configs:
self.config_write(config)
def config_type(self, config):
'''Check if a configuration is an acceptable type.'''
return isinstance(config, self.config_types)
def config_cast(self, config):
'''Cast a configuration to the internal expected type.'''
if not self.config_type(config):
config = self.config_caster(config)
return config
def config_subs(self):
'''Get the substitution values for use to generate the config'''
subs = dict([(attr, getattr(self, attr, None))
for attr in self.config_attrs])
for key, val in self.config_vals.items():
subs[key] = val.format(**subs)
return subs
def config_stringify(self, config):
'''Get the configuration as a string'''
cfg = self.config_get(config)
cfg.format(**self.config_subs())
return cfg
def config_merge(self, base, overrides):
'''Merge two configuration hunks'''
base = self.config_cast(base)
overrides = self.config_cast(overrides)
return ''.join([base, overrides])
def config_get(self, config):
'''Get the configuration data'''
return self.configs[config]
def config_set(self, config, val):
'''Set the configuration data'''
self.configs[config] = val
def make_dirtree(self):
'''Create directory structure.'''
subdirs = []
for branch in self.dirtree:
log.debug('checking dirtree: {0}'.format(branch))
if not branch:
continue
if isinstance(branch, six.string_types) and branch[0] == '&':
log.debug('Looking up dirtree branch "{0}"'.format(branch))
try:
dirattr = getattr(self, branch[1:], None)
log.debug('dirtree "{0}" => "{1}"'.format(branch, dirattr))
except AttributeError:
raise ValueError(
'Unable to find dirtree attribute "{0}" on object "{1}.name = {2}: {3}"'
.format(
branch,
self.__class__.__name__,
self.name,
dir(self),
))
if not dirattr:
continue
if isinstance(dirattr, six.string_types):
subdirs.append(dirattr)
elif hasattr(dirattr, '__iter__'):
subdirs.extend(dirattr)
else:
raise TypeError(
"Branch type of {0} in dirtree is unhandled".format(
branch))
elif isinstance(branch, six.string_types):
subdirs.append(branch)
else:
raise TypeError(
"Branch type of {0} in dirtree is unhandled".format(
branch))
for subdir in subdirs:
path = self.abs_path(subdir)
if not os.path.exists(path):
log.debug('make_dirtree: {0}'.format(path))
os.makedirs(path)
def setup(self, *args, **kwargs):
'''Create any scaffolding for run-time'''
# unused
_ = args, kwargs
if not self._setup_done:
self.make_dirtree()
self.configs_write()
self._setup_done = True
def cleanup(self, *args, **kwargs):
''' Clean out scaffolding of setup() and any run-time generated files.'''
# Unused for now
_ = (args, kwargs)
if self.process:
try:
self.process.kill()
self.process.wait()
except OSError:
pass
if os.path.exists(self.test_dir):
shutil.rmtree(self.test_dir)
if self.created_parent_dir and os.path.exists(self.parent_dir):
shutil.rmtree(self.parent_dir)
def run(
self,
args=None,
catch_stderr=False,
with_retcode=False,
timeout=None,
raw=False,
env=None,
verbatim_args=False,
verbatim_env=False,
):
'''
Execute a command possibly using a supplied environment.
:param args:
A command string or a command sequence of arguments for the program.
:param catch_stderr: A boolean whether to capture and return stderr.
:param with_retcode: A boolean whether to return the exit code.
:param timeout: A float of how long to wait for the process to
complete before it is killed.
:param raw: A boolean whether to return buffer strings for stdout and
stderr or sequences of output lines.
:param env: A dictionary of environment key/value settings for the
command.
:param verbatim_args: A boolean whether to automatically add inferred arguments.
:param verbatim_env: A boolean whether to automatically add inferred
environment values.
:return list: (stdout [,stderr] [,retcode])
'''
# unused for now
_ = verbatim_args
self.setup()
if args is None:
args = []
if env is None:
env = {}
env_delta = {}
env_delta.update(self.env)
env_delta.update(env)
if not verbatim_env:
env_pypath = env_delta.get('PYTHONPATH',
os.environ.get('PYTHONPATH'))
if not env_pypath:
env_pypath = sys.path
else:
env_pypath = env_pypath.split(':')
for path in sys.path:
if path not in env_pypath:
env_pypath.append(path)
# Always ensure that the test tree is searched first for python modules
if integration.CODE_DIR != env_pypath[0]:
env_pypath.insert(0, integration.CODE_DIR)
env_delta['PYTHONPATH'] = ':'.join(env_pypath)
cmd_env = dict(os.environ)
cmd_env.update(env_delta)
popen_kwargs = {
'shell': self.shell,
'stdout': subprocess.PIPE,
'env': cmd_env,
}
if catch_stderr is True:
popen_kwargs['stderr'] = subprocess.PIPE
if not sys.platform.lower().startswith('win'):
popen_kwargs['close_fds'] = True
def detach_from_parent_group():
'''
A utility function that prevents child process from getting parent signals.
'''
os.setpgrp()
popen_kwargs['preexec_fn'] = detach_from_parent_group
elif sys.platform.lower().startswith('win') and timeout is not None:
raise RuntimeError('Timeout is not supported under windows')
argv = [self.program]
argv.extend(args)
log.debug('TestProgram.run: {0} Environment {1}'.format(
argv, env_delta))
process = subprocess.Popen(argv, **popen_kwargs)
self.process = process
if timeout is not None:
stop_at = datetime.now() + timedelta(seconds=timeout)
term_sent = False
while True:
process.poll()
if datetime.now() > stop_at:
if term_sent is False:
# Kill the process group since sending the term signal
# would only terminate the shell, not the command
# executed in the shell
os.killpg(os.getpgid(process.pid), signal.SIGINT)
term_sent = True
continue
try:
# As a last resort, kill the process group
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
process.wait()
except OSError as exc:
if exc.errno != errno.ESRCH:
raise
out = process.stdout.read().splitlines()
out.extend([
'Process took more than {0} seconds to complete. '
'Process Killed!'.format(timeout)
])
if catch_stderr:
err = process.stderr.read().splitlines()
if with_retcode:
return out, err, process.returncode
else:
return out, err
if with_retcode:
return out, process.returncode
else:
return out
if process.returncode is not None:
break
if catch_stderr:
if sys.version_info < (2, 7):
# On python 2.6, the subprocess'es communicate() method uses
# select which, is limited by the OS to 1024 file descriptors
# We need more available descriptors to run the tests which
# need the stderr output.
# So instead of .communicate() we wait for the process to
# finish, but, as the python docs state "This will deadlock
# when using stdout=PIPE and/or stderr=PIPE and the child
# process generates enough output to a pipe such that it
# blocks waiting for the OS pipe buffer to accept more data.
# Use communicate() to avoid that." <- a catch, catch situation
#
# Use this work around were it's needed only, python 2.6
process.wait()
out = process.stdout.read()
err = process.stderr.read()
else:
out, err = process.communicate()
# Force closing stderr/stdout to release file descriptors
if process.stdout is not None:
process.stdout.close()
if process.stderr is not None:
process.stderr.close()
# pylint: disable=maybe-no-member
try:
if with_retcode:
if out is not None and err is not None:
if not raw:
return out.splitlines(), err.splitlines(
), process.returncode
else:
return out, err, process.returncode
return out.splitlines(), [], process.returncode
else:
if out is not None and err is not None:
if not raw:
return out.splitlines(), err.splitlines()
else:
return out, err
if not raw:
return out.splitlines(), []
else:
return out, []
finally:
try:
process.terminate()
except OSError as err:
# process already terminated
pass
# pylint: enable=maybe-no-member
data = process.communicate()
process.stdout.close()
try:
if with_retcode:
if not raw:
return data[0].splitlines(), process.returncode
else:
return data[0], process.returncode
else:
if not raw:
return data[0].splitlines()
else:
return data[0]
finally:
try:
process.terminate()
except OSError as err:
# process already terminated
pass
class TestSaltDaemon(
six.with_metaclass(TestSaltProgramMeta, TestDaemon, TestSaltProgram)):
'''
A class to run arbitrary salt daemons (master, minion, syndic, etc.)
'''
pass
class SaltLoggingClass(
six.with_metaclass(LoggingMixInMeta, LOGGING_LOGGER_CLASS,
NewStyleClassMixIn)):
def __new__(cls, *args): # pylint: disable=W0613, E1002
'''
We override `__new__` in our logging logger class in order to provide
some additional features like expand the module name padding if length
is being used, and also some Unicode fixes.
This code overhead will only be executed when the class is
instantiated, i.e.:
logging.getLogger(__name__)
'''
instance = super(SaltLoggingClass, cls).__new__(cls)
try:
max_logger_length = len(
max(list(logging.Logger.manager.loggerDict), key=len))
for handler in logging.root.handlers:
if handler in (LOGGING_NULL_HANDLER, LOGGING_STORE_HANDLER,
LOGGING_TEMP_HANDLER):
continue
formatter = handler.formatter
if not formatter:
continue
if not handler.lock:
handler.createLock()
handler.acquire()
fmt = formatter._fmt.replace('%', '%%')
match = MODNAME_PATTERN.search(fmt)
if not match:
# Not matched. Release handler and return.
handler.release()
return instance
if 'digits' not in match.groupdict():
# No digits group. Release handler and return.
handler.release()
return instance
digits = match.group('digits')
if not digits or not (digits and digits.isdigit()):
# No valid digits. Release handler and return.
handler.release()
return instance
if int(digits) < max_logger_length:
# Formatter digits value is lower than current max, update.
fmt = fmt.replace(match.group('name'), '%%(name)-%ds')
formatter = logging.Formatter(fmt % max_logger_length,
datefmt=formatter.datefmt)
handler.setFormatter(formatter)
handler.release()
except ValueError:
# There are no registered loggers yet
pass
return instance
def _log(
self,
level,
msg,
args,
exc_info=None,
extra=None, # pylint: disable=arguments-differ
exc_info_on_loglevel=None):
# If both exc_info and exc_info_on_loglevel are both passed, let's fail
if exc_info and exc_info_on_loglevel:
raise RuntimeError(
'Only one of \'exc_info\' and \'exc_info_on_loglevel\' is '
'permitted')
if exc_info_on_loglevel is not None:
if isinstance(exc_info_on_loglevel, six.string_types):
exc_info_on_loglevel = LOG_LEVELS.get(exc_info_on_loglevel,
logging.ERROR)
elif not isinstance(exc_info_on_loglevel, int):
raise RuntimeError(
'The value of \'exc_info_on_loglevel\' needs to be a '
'logging level or a logging level name, not \'{0}\''.
format(exc_info_on_loglevel))
if extra is None:
extra = {'exc_info_on_loglevel': exc_info_on_loglevel}
else:
extra['exc_info_on_loglevel'] = exc_info_on_loglevel
LOGGING_LOGGER_CLASS._log(self,
level,
msg,
args,
exc_info=exc_info,
extra=extra)
# pylint: disable=C0103
# pylint: disable=W0221
def makeRecord(self,
name,
level,
fn,
lno,
msg,
args,
exc_info,
func=None,
extra=None,
sinfo=None):
# Let's remove exc_info_on_loglevel from extra
exc_info_on_loglevel = extra.pop('exc_info_on_loglevel')
if not extra:
# If nothing else is in extra, make it None
extra = None
# Let's try to make every logging message unicode
if isinstance(msg, six.string_types) \
and not isinstance(msg, six.text_type):
salt_system_encoding = __salt_system_encoding__
if salt_system_encoding == 'ascii':
# Encoding detection most likely failed, let's use the utf-8
# value which we defaulted before __salt_system_encoding__ was
# implemented
salt_system_encoding = 'utf-8'
try:
_msg = msg.decode(salt_system_encoding, 'replace')
except UnicodeDecodeError:
_msg = msg.decode(salt_system_encoding, 'ignore')
else:
_msg = msg
if six.PY3:
logrecord = _LOG_RECORD_FACTORY(name, level, fn, lno, _msg, args,
exc_info, func, sinfo)
else:
logrecord = _LOG_RECORD_FACTORY(name, level, fn, lno, _msg, args,
exc_info, func)
if extra is not None:
for key in extra:
if (key in ['message', 'asctime']) or (key
in logrecord.__dict__):
raise KeyError(
'Attempt to overwrite \'{0}\' in LogRecord'.format(
key))
logrecord.__dict__[key] = extra[key]
if exc_info_on_loglevel is not None:
# Let's add some custom attributes to the LogRecord class in order
# to include the exc_info on a per handler basis. This will allow
# showing tracebacks on logfiles but not on console if the logfile
# handler is enabled for the log level "exc_info_on_loglevel" and
# console handler is not.
logrecord.exc_info_on_loglevel_instance = sys.exc_info()
logrecord.exc_info_on_loglevel_formatted = None
logrecord.exc_info_on_loglevel = exc_info_on_loglevel
return logrecord
class TestSaltProgram(six.with_metaclass(TestSaltProgramMeta, TestProgram)):
'''
This is like TestProgram but with some functions to run a salt-specific
auxiliary program.
'''
config_types = (dict, )
config_attrs = set([
'log_dir',
'script_dir',
])
config_base = {
'root_dir': '{test_dir}',
}
configs = {}
config_dir = os.path.join('etc', 'salt')
log_dir = os.path.join('var', 'log', 'salt')
dirtree = [
'&log_dir',
'&script_dir',
]
script = ''
script_dir = 'bin'
@staticmethod
def config_caster(cfg):
return yaml.safe_load(cfg)
def __init__(self, *args, **kwargs):
if len(args) < 2 and 'program' not in kwargs:
# This is effectively a place-holder - it gets set correctly after super()
kwargs['program'] = self.script
super(TestSaltProgram, self).__init__(*args, **kwargs)
self.program = self.abs_path(os.path.join(self.script_dir,
self.script))
path = self.env.get('PATH', os.getenv('PATH'))
self.env['PATH'] = ':'.join([self.abs_path(self.script_dir), path])
def config_merge(self, base, overrides):
_base = self.config_cast(copy.deepcopy(base))
_overrides = self.config_cast(overrides)
# NOTE: this simple update will not work for deep dictionaries
_base.update(copy.deepcopy(_overrides))
return _base
def config_get(self, config):
cfg_base = {}
for key, val in self.config_base.items():
_val = val
if val and isinstance(val, six.string_types) and val[0] == '&':
_val = getattr(self, val[1:], None)
if _val is None:
continue
cfg_base[key] = _val
if config in self.configs:
cfg = {}
for key, val in self.configs.get(config, {}).get('map',
{}).items():
_val = val
if val and isinstance(val, six.string_types) and val[0] == '&':
_val = getattr(self, val[1:], None)
if _val is None:
continue
cfg[key] = _val
cfg = self.config_merge(cfg_base, cfg)
log.debug('Generated config => {0}'.format(cfg))
return cfg
def config_stringify(self, config):
'''Transform the configuration data into a string (suitable to write to a file)'''
subs = self.config_subs()
cfg = {}
for key, val in self.config_get(config).items():
if isinstance(val, six.string_types):
cfg[key] = val.format(**subs)
else:
cfg[key] = val
scfg = yaml.safe_dump(cfg, default_flow_style=False)
return scfg
def setup(self, *args, **kwargs):
super(TestSaltProgram, self).setup(*args, **kwargs)
self.install_script()
def install_script(self):
'''Generate the script file that calls python objects and libraries.'''
lines = []
script_source = os.path.join(integration.CODE_DIR, 'scripts',
self.script)
with open(script_source, 'r') as sso:
lines.extend(sso.readlines())
if lines[0].startswith('#!'):
lines.pop(0)
lines.insert(0, '#!{0}\n'.format(sys.executable))
script_path = self.abs_path(os.path.join(self.script_dir, self.script))
log.debug('Installing "{0}" to "{1}"'.format(script_source,
script_path))
with open(script_path, 'w') as sdo:
sdo.write(''.join(lines))
sdo.flush()
os.chmod(script_path, 0o755)
def run(self, **kwargs):
if not kwargs.get('verbatim_args'):
args = kwargs.setdefault('args', [])
if '-c' not in args and '--config-dir' not in args:
args.extend(['--config-dir', self.abs_path(self.config_dir)])
return super(TestSaltProgram, self).run(**kwargs)
class SqlBaseExtPillar(six.with_metaclass(abc.ABCMeta, object)):
'''
This class receives and processes the database rows in a database
agnostic way.
'''
result = None
focus = None
field_names = None
num_fields = 0
depth = 0
as_list = False
with_lists = None
ignore_null = False
def __init__(self):
self.result = self.focus = {}
@classmethod
@abc.abstractmethod
def _db_name(cls):
'''
Return a friendly name for the database, e.g. 'MySQL' or 'SQLite'.
Used in logging output.
'''
pass
@abc.abstractmethod
def _get_cursor(self):
'''
Yield a PEP 249 compliant Cursor as a context manager.
'''
pass
def extract_queries(self, args, kwargs):
'''
This function normalizes the config block into a set of queries we
can use. The return is a list of consistently laid out dicts.
'''
# Please note the function signature is NOT an error. Neither args, nor
# kwargs should have asterisks. We are passing in a list and dict,
# rather than receiving variable args. Adding asterisks WILL BREAK the
# function completely.
# First, this is the query buffer. Contains lists of [base,sql]
qbuffer = []
# Add on the non-keywords...
qbuffer.extend([[None, s] for s in args])
# And then the keywords...
# They aren't in definition order, but they can't conflict each other.
klist = list(kwargs.keys())
klist.sort()
qbuffer.extend([[k, kwargs[k]] for k in klist])
# Filter out values that don't have queries.
qbuffer = [
x for x in qbuffer
if ((isinstance(x[1], six.string_types) and len(x[1])) or (
isinstance(x[1], (list, tuple)) and (len(x[1]) > 0) and x[1][0]
) or (isinstance(x[1], dict) and 'query' in x[1]
and len(x[1]['query'])))
]
# Next, turn the whole buffer into full dicts.
for qb in qbuffer:
defaults = {
'query': '',
'depth': 0,
'as_list': False,
'with_lists': None,
'ignore_null': False
}
if isinstance(qb[1], six.string_types):
defaults['query'] = qb[1]
elif isinstance(qb[1], (list, tuple)):
defaults['query'] = qb[1][0]
if len(qb[1]) > 1:
defaults['depth'] = qb[1][1]
# May set 'as_list' from qb[1][2].
else:
defaults.update(qb[1])
if defaults['with_lists'] and isinstance(
defaults['with_lists'], six.string_types):
defaults['with_lists'] = [
int(i) for i in defaults['with_lists'].split(',')
]
qb[1] = defaults
return qbuffer
def enter_root(self, root):
'''
Set self.focus for kwarg queries
'''
# There is no collision protection on root name isolation
if root:
self.result[root] = self.focus = {}
else:
self.focus = self.result
def process_fields(self, field_names, depth):
'''
The primary purpose of this function is to store the sql field list
and the depth to which we process.
'''
# List of field names in correct order.
self.field_names = field_names
# number of fields.
self.num_fields = len(field_names)
# Constrain depth.
if (depth == 0) or (depth >= self.num_fields):
self.depth = self.num_fields - 1
else:
self.depth = depth
def process_results(self, rows):
'''
This function takes a list of database results and iterates over,
merging them into a dict form.
'''
listify = OrderedDict()
listify_dicts = OrderedDict()
for ret in rows:
# crd is the Current Return Data level, to make this non-recursive.
crd = self.focus
# Walk and create dicts above the final layer
for i in range(0, self.depth - 1):
# At the end we'll use listify to find values to make a list of
if i + 1 in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if ret[i] not in listify[id(crd)]:
listify[id(crd)].append(ret[i])
if ret[i] not in crd:
# Key missing
crd[ret[i]] = {}
crd = crd[ret[i]]
else:
# Check type of collision
ty = type(crd[ret[i]])
if ty is list:
# Already made list
temp = {}
crd[ret[i]].append(temp)
crd = temp
elif ty is not dict:
# Not a list, not a dict
if self.as_list:
# Make list
temp = {}
crd[ret[i]] = [crd[ret[i]], temp]
crd = temp
else:
# Overwrite
crd[ret[i]] = {}
crd = crd[ret[i]]
else:
# dict, descend.
crd = crd[ret[i]]
# If this test is true, the penultimate field is the key
if self.depth == self.num_fields - 1:
nk = self.num_fields - 2 # Aka, self.depth-1
# Should we and will we have a list at the end?
if ((self.as_list and (ret[nk] in crd))
or (nk + 1 in self.with_lists)):
if ret[nk] in crd:
if not isinstance(crd[ret[nk]], list):
crd[ret[nk]] = [crd[ret[nk]]]
# if it's already a list, do nothing
else:
crd[ret[nk]] = []
crd[ret[nk]].append(ret[self.num_fields - 1])
else:
if not self.ignore_null or ret[self.num_fields -
1] is not None:
crd[ret[nk]] = ret[self.num_fields - 1]
else:
# Otherwise, the field name is the key but we have a spare.
# The spare results because of {c: d} vs {c: {"d": d, "e": e }}
# So, make that last dict
if ret[self.depth - 1] not in crd:
crd[ret[self.depth - 1]] = {}
# This bit doesn't escape listify
if self.depth in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if ret[self.depth - 1] not in listify[id(crd)]:
listify[id(crd)].append(ret[self.depth - 1])
crd = crd[ret[self.depth - 1]]
# Now for the remaining keys, we put them into the dict
for i in range(self.depth, self.num_fields):
nk = self.field_names[i]
# Listify
if i + 1 in self.with_lists:
if id(crd) not in listify:
listify[id(crd)] = []
listify_dicts[id(crd)] = crd
if nk not in listify[id(crd)]:
listify[id(crd)].append(nk)
# Collision detection
if self.as_list and (nk in crd):
# Same as before...
if isinstance(crd[nk], list):
crd[nk].append(ret[i])
else:
crd[nk] = [crd[nk], ret[i]]
else:
if not self.ignore_null or ret[i] is not None:
crd[nk] = ret[i]
# Get key list and work backwards. This is inner-out processing
ks = list(listify_dicts.keys())
ks.reverse()
for i in ks:
d = listify_dicts[i]
for k in listify[i]:
if isinstance(d[k], dict):
d[k] = list(d[k].values())
elif isinstance(d[k], list):
d[k] = [d[k]]
def fetch(
self,
minion_id,
pillar, # pylint: disable=W0613
*args,
**kwargs):
'''
Execute queries, merge and return as a dict.
'''
db_name = self._db_name()
log.info('Querying {0} for information for {1}'.format(
db_name, minion_id))
#
# log.debug('ext_pillar {0} args: {1}'.format(db_name, args))
# log.debug('ext_pillar {0} kwargs: {1}'.format(db_name, kwargs))
#
# Most of the heavy lifting is in this class for ease of testing.
qbuffer = self.extract_queries(args, kwargs)
with self._get_cursor() as cursor:
for root, details in qbuffer:
# Run the query
cursor.execute(details['query'], (minion_id, ))
# Extract the field names the db has returned and process them
self.process_fields([row[0] for row in cursor.description],
details['depth'])
self.enter_root(root)
self.as_list = details['as_list']
if details['with_lists']:
self.with_lists = details['with_lists']
else:
self.with_lists = []
self.ignore_null = details['ignore_null']
self.process_results(cursor.fetchall())
log.debug('ext_pillar {0}: Return data: {1}'.format(
db_name, self))
return self.result
class SaltStandaloneProxyOptionParser(
six.with_metaclass(
salt.utils.parsers.OptionParserMeta,
salt.utils.parsers.OptionParser,
salt.utils.parsers.ConfigDirMixIn,
salt.utils.parsers.MergeConfigMixIn,
salt.utils.parsers.TimeoutMixIn,
salt.utils.parsers.LogLevelMixIn,
salt.utils.parsers.HardCrashMixin,
salt.utils.parsers.SaltfileMixIn,
salt.utils.parsers.TargetOptionsMixIn,
salt.utils.parsers.OutputOptionsMixIn,
salt.utils.parsers.ArgsStdinMixIn,
salt.utils.parsers.ProfilingPMixIn,
salt.utils.parsers.EAuthMixIn,
salt.utils.parsers.NoParseMixin,
)
):
default_timeout = 1
description = (
'''
___ _ _ ___ ___
/ __| __ _ | | | |_ / __| | _ \ _ _ ___ __ __ _ _
\__ \ / _` | | | | _| \__ \ | _/ | '_| / _ \ \ \ / | || |
|___/ \__,_| |_| \__| |___/ |_| |_| \___/ /_\_\ \_, |
|__/
\n
'''
'salt-sproxy is a tool to invoke arbitrary Salt functions on a group\n'
'of (network) devices connecting through a Salt Proxy Minion, without\n'
'having the Proxy Minion services up and running (or the Master).\n'
)
VERSION = salt_sproxy.version.__version__
usage = '%prog [options] <target> <function> [arguments]'
epilog = (
'You can find additional help about %prog at '
'https://salt-sproxy.readthedocs.io/en/latest/'
)
# ConfigDirMixIn config filename attribute
_config_filename_ = 'master'
# LogLevelMixIn attributes
_default_logging_level_ = config.DEFAULT_MASTER_OPTS['log_level']
_default_logging_logfile_ = config.DEFAULT_MASTER_OPTS['log_file']
def format_description(self, formatter):
return self.description
def _mixin_setup(self):
self.add_option(
'-r', '--roster', default=False, help='The name of the Salt Roster to use.'
)
self.add_option(
'--roster-file',
dest='roster_file',
help='Absolute path to the Roster file to use.',
)
self.add_option(
'--sync',
default=False,
action='store_true',
help=(
'Return the replies from the devices immediately they are '
'received, or everything at once.'
),
)
self.add_option(
'--cache-grains',
default=False,
action='store_true',
help=(
'Cache the collected Grains. This is going to override the '
'existing cached Grains.'
),
)
self.add_option(
'--cache-pillar',
default=False,
action='store_true',
help=(
'Cache the compiled Pillar. This is going to override the '
'existing cached Pillar.'
),
)
self.add_option(
'--no-cached-grains',
default=False,
action='store_true',
help='Do not use the available cached Grains (if any).',
)
self.add_option(
'--no-cached-pillar',
default=False,
action='store_true',
help='Do not use the available cached Pillar (if any)',
)
self.add_option(
'--no-grains',
default=False,
action='store_true',
help=(
'Do not attempt to collect Grains at all. Use with care, it '
'may lead to unexpected results.'
),
)
self.add_option(
'--no-pillar',
default=False,
action='store_true',
help=(
'Do not compile Pillar at all. Use with care, it may lead to '
'unexpected results.'
),
)
self.add_option(
'-b',
'--batch',
'--batch-size',
default=CPU_COUNT,
dest='batch_size',
help=(
'The number of devices to connect to in parallel. '
'Default: {}'.format(CPU_COUNT)
),
)
self.add_option(
'--preview-target',
dest='preview_target',
action='store_true',
help='Show the devices expected to match the target.',
)
self.add_option(
'--sync-grains',
dest='sync_grains',
action='store_true',
default=False,
help=(
'Re-sync the Grains modules. Useful if you have custom Grains '
'modules in your own environment.'
),
)
self.add_option(
'--sync-modules',
dest='sync_modules',
action='store_true',
help=('Load the salt-sproxy Execution modules.'),
)
self.add_option(
'--sync-roster',
dest='sync_roster',
action='store_true',
help=(
'Synchronise the Roster modules (both salt-sproxy native '
'and provided by the user in their own environment).'
),
)
self.add_option(
'--events',
dest='events',
action='store_true',
help=(
'Whether should put the events on the Salt bus (mostly '
'useful when having a Master running).'
),
)
self.add_option(
'--use-proxy',
'--use-existing-proxy',
'--use-existing-minion',
dest='use_existing_proxy',
action='store_true',
help=(
'Use the existing Proxy Minions to execute the commands, '
'whenever available.'
),
)
self.add_option(
'--file-roots',
'--display-file-roots',
dest='display_file_roots',
action='store_true',
help=(
'Display the file_roots option you would need to configure '
'in order to use the salt-sproxy extension modules directly, '
'and, implicitly, leverage the event-driven methodology and '
'the Salt REST API.'
),
)
self.add_option(
'--save-file-roots',
dest='save_file_roots',
action='store_true',
help=(
'Saves the file_roots configuration so you can start '
'leveraging the event-driven automation and the Salt REST API.'
),
)
self.add_option(
'--no-connect',
dest='no_connect',
action='store_true',
default=False,
help=(
'Do not initiate the connection with the device, only use '
'cached data to compile data and execute Salt functions that '
'do not require the actual connection with the device.'
),
)
self.add_option(
'--test-ping',
dest='test_ping',
action='store_true',
default=False,
help=(
'When using together with --use-existing-proxy, this option can'
' help to ensure the existing Proxy Minion is responsive (not '
'only up and running, by executing a ping test.'
),
)
self.add_option(
'--no-target-cache',
dest='no_target_cache',
action='store_true',
default=False,
help=('Do no load the list of targets from the cache.'),
)
group = self.output_options_group = optparse.OptionGroup(
self, 'Output Options', 'Configure your preferred output format.'
)
self.add_option_group(group)
group.add_option(
'-q',
'--quiet',
default=False,
action='store_true',
help='Do not display the results of the run.',
)
# Everything else that follows here is verbatim copy from
# https://github.com/saltstack/salt/blob/develop/salt/utils/parsers.py
def _mixin_after_parsed(self):
if self.options.display_file_roots or self.options.save_file_roots:
# Insert dummy arg when displaying the file_roots
self.args.append('not_a_valid_target')
self.args.append('not_a_valid_command')
if self.options.list:
try:
if ',' in self.args[0]:
self.config['tgt'] = self.args[0].replace(' ', '').split(',')
else:
self.config['tgt'] = self.args[0].split()
except IndexError:
self.exit(42, '\nCannot execute command without defining a target.\n\n')
else:
try:
self.config['tgt'] = self.args[0]
except IndexError:
self.exit(42, '\nCannot execute command without defining a target.\n\n')
if self.options.preview_target:
# Insert dummy arg which won't be used
self.args.append('not_a_valid_command')
# Detect compound command and set up the data for it
if self.args:
try:
if ',' in self.args[1]:
self.config['fun'] = self.args[1].split(',')
self.config['arg'] = [[]]
cmd_index = 0
if (
self.args[2:].count(self.options.args_separator)
== len(self.config['fun']) - 1
):
# new style parsing: standalone argument separator
for arg in self.args[2:]:
if arg == self.options.args_separator:
cmd_index += 1
self.config['arg'].append([])
else:
self.config['arg'][cmd_index].append(arg)
else:
# old style parsing: argument separator can be inside args
for arg in self.args[2:]:
if self.options.args_separator in arg:
sub_args = arg.split(self.options.args_separator)
for sub_arg_index, sub_arg in enumerate(sub_args):
if sub_arg:
self.config['arg'][cmd_index].append(sub_arg)
if sub_arg_index != len(sub_args) - 1:
cmd_index += 1
self.config['arg'].append([])
else:
self.config['arg'][cmd_index].append(arg)
if len(self.config['fun']) > len(self.config['arg']):
self.exit(
42,
'Cannot execute compound command without '
'defining all arguments.\n',
)
elif len(self.config['fun']) < len(self.config['arg']):
self.exit(
42,
'Cannot execute compound command with more '
'arguments than commands.\n',
)
# parse the args and kwargs before sending to the publish
# interface
for i in range(len(self.config['arg'])):
self.config['arg'][i] = salt.utils.args.parse_input(
self.config['arg'][i], no_parse=self.options.no_parse
)
else:
self.config['fun'] = self.args[1]
self.config['arg'] = self.args[2:]
# parse the args and kwargs before sending to the publish
# interface
self.config['arg'] = salt.utils.args.parse_input(
self.config['arg'], no_parse=self.options.no_parse
)
except IndexError:
self.exit(42, '\nIncomplete options passed.\n\n')
def setup_config(self):
return config.client_config(self.get_config_file_path())
class LoaderModuleMockMixin(
six.with_metaclass(_FixLoaderModuleMockMixinMroOrder, object)):
'''
This class will setup salt loader dunders.
Please check `set_up_loader_mocks` above
'''
# Define our setUp function decorator
@staticmethod
def __setup_loader_modules_mocks__(setup_func):
@functools.wraps(setup_func)
def wrapper(self):
if NO_MOCK:
self.skipTest(NO_MOCK_REASON)
loader_modules_configs = self.setup_loader_modules()
if not isinstance(loader_modules_configs, dict):
raise RuntimeError(
'{}.setup_loader_modules() must return a dictionary where the keys are the '
'modules that require loader mocking setup and the values, the global module '
'variables for each of the module being mocked. For example \'__salt__\', '
'\'__opts__\', etc.'.format(self.__class__.__name__))
salt_dunders = (
'__opts__',
'__salt__',
'__runner__',
'__context__',
'__utils__',
'__ext_pillar__',
'__thorium__',
'__states__',
'__serializers__',
'__ret__',
'__grains__',
'__pillar__',
'__sdb__',
# Proxy is commented out on purpose since some code in salt expects a NameError
# and is most of the time not a required dunder
# '__proxy__'
)
for module, module_globals in six.iteritems(
loader_modules_configs):
if not isinstance(module, types.ModuleType):
raise RuntimeError(
'The dictionary keys returned by {}.setup_loader_modules() '
'must be an imported module, not {}'.format(
self.__class__.__name__, type(module)))
if not isinstance(module_globals, dict):
raise RuntimeError(
'The dictionary values returned by {}.setup_loader_modules() '
'must be a dictionary, not {}'.format(
self.__class__.__name__, type(module_globals)))
module_blacklisted_dunders = module_globals.pop(
'blacklisted_dunders', ())
minion_funcs = {}
if '__salt__' in module_globals and module_globals[
'__salt__'] == 'autoload':
if '__opts__' not in module_globals:
raise RuntimeError(
'You must provide \'__opts__\' on the {} module globals dictionary '
'to auto load the minion functions'.format(
module.__name__))
import salt.loader
ctx = {}
if '__utils__' not in module_globals:
utils = salt.loader.utils(
module_globals['__opts__'],
context=module_globals.get('__context__') or ctx)
module_globals['__utils__'] = utils
minion_funcs = salt.loader.minion_mods(
module_globals['__opts__'],
context=module_globals.get('__context__') or ctx,
utils=module_globals.get('__utils__'),
)
module_globals['__salt__'] = minion_funcs
for dunder_name in salt_dunders:
if dunder_name not in module_globals:
if dunder_name in module_blacklisted_dunders:
continue
module_globals[dunder_name] = {}
sys_modules = module_globals.pop('sys.modules', None)
if sys_modules is not None:
if not isinstance(sys_modules, dict):
raise RuntimeError(
'\'sys.modules\' must be a dictionary not: {}'.
format(type(sys_modules)))
patcher = patch.dict(sys.modules, sys_modules)
patcher.start()
def cleanup_sys_modules(patcher, sys_modules):
patcher.stop()
del patcher
del sys_modules
self.addCleanup(cleanup_sys_modules, patcher, sys_modules)
for key in module_globals:
if not hasattr(module, key):
if key in salt_dunders:
setattr(module, key, {})
else:
setattr(module, key, None)
if module_globals:
patcher = patch.multiple(module, **module_globals)
patcher.start()
def cleanup_module_globals(patcher, module_globals):
patcher.stop()
del patcher
del module_globals
self.addCleanup(cleanup_module_globals, patcher,
module_globals)
if minion_funcs:
# Since we autoloaded the minion_funcs, let's namespace the functions with the globals
# used to patch above
import salt.utils
for func in minion_funcs:
minion_funcs[
func] = salt.utils.functools.namespaced_function(
minion_funcs[func],
module_globals,
preserve_context=True)
return setup_func(self)
return wrapper
def setup_loader_modules(self):
raise NotImplementedError(
'\'{}.setup_loader_modules()\' must be implemented'.format(
self.__class__.__name__))
class SaltNodesCommandParser(
six.with_metaclass(
salt.utils.parsers.OptionParserMeta,
salt.utils.parsers.OptionParser,
salt.utils.parsers.ConfigDirMixIn,
salt.utils.parsers.ExtendedTargetOptionsMixIn,
salt.utils.parsers.LogLevelMixIn)):
'''
Argument parser used by SaltGenResource to generate
Rundeck node definitions.
'''
usage = '%prog [options] <target> [<attr>=<value> ...]'
description = 'Salt Mine node source for Rundeck.'
epilog = None
_config_filename_ = 'minion'
_logfile_config_setting_name_ = 'resource_generator_logfile'
_logfile_loglevel_config_setting_name_ = 'resource_generator_log_level_logfile'
_default_logging_logfile_ = os.path.join(syspaths.LOGS_DIR, 'resource-generator')
_setup_mp_logging_listener_ = False
_default_logging_level_ = 'warning'
# Ignore requests to provide reserved attribute names
ignore_attributes = ['hostname', 'osName', 'osVersion', 'osFamily', 'osArch', 'tags']
ignore_servernode = ['username', 'description']
def _mixin_setup(self):
'''
Define arguments specific to SaltGenResource
'''
self.add_option(
'-m', '--mine-function',
default='grains.items',
type=str,
help=('Set the function name for Salt Mine to execute '
'to retrieve grains. Default value is grains.items '
'but this could be different if mine function '
'aliases are used.')
)
self.add_option(
'-s', '--include-server-node',
action="store_true",
help=('Include the Rundeck server node in the output. '
'The server node is required for some workflows '
'and must be provided by exactly one resource provider.')
)
self.add_option(
'-u', '--server-node-user',
type=str,
default='rundeck',
help=('Specify the user name to use when running jobs on the '
'server node. This would typically be the same user that '
'the Rundeck service is running as. Default: \'rundeck\'.')
)
self.add_option(
'-a', '--attributes',
type=str,
default=[],
action='callback',
callback=self.set_callback,
help=('Create Rundeck node attributes from the values of grains. '
'Multiple grains may be specified '
'when separated by a space or comma.')
)
self.add_option(
'-t', '--tags',
type=str,
default=[],
action='callback',
callback=self.set_callback,
help=('Create Rundeck node tags from the values of grains. '
'Multiple grains may be specified '
'when separated by a space or comma.')
)
def _mixin_after_parsed(self):
'''
Validate and process arguments
'''
if not os.path.isfile(self.get_config_file_path()):
log.critical("Configuration file not found")
sys.exit(-1)
# Extract targeting expression
try:
if self.options.list:
if ',' in self.args[0]:
self.config['tgt'] = \
self.args[0].replace(' ', '').split(',')
else:
self.config['tgt'] = self.args[0].split()
else:
self.config['tgt'] = self.args[0]
except IndexError:
self.exit(42, ('\nCannot execute command without '
'defining a target.\n\n'))
if self.options.log_level:
self.config['log_level'] = self.options.log_level
else:
self.config['log_level'] = self._default_logging_level_
# Set default targeting option
if self.config['selected_target_option'] is None:
self.config['selected_target_option'] = 'glob'
# Remove conflicting grains
self.options.attributes = [x for x in self.options.attributes
if x not in self.ignore_attributes]
def setup_config(self):
config_opts = config.minion_config(
self.get_config_file_path(),
cache_minion_id=True,
ignore_config_errors=False)
# Make file based logging work
if getattr(self.options, self._logfile_config_setting_name_, 'None') is None:
# Copy the default log file path into the config dict
if self._logfile_config_setting_name_ not in config_opts:
config_opts[self._logfile_config_setting_name_] = self._default_logging_logfile_
# Prepend the root_dir setting to the log file path
config.prepend_root_dir(config_opts, [self._logfile_config_setting_name_])
# Copy the altered path back to the options or it will revert to the default
setattr(self.options, self._logfile_config_setting_name_, config_opts[self._logfile_config_setting_name_])
else:
# Copy the provided log file path into the config dict
if self._logfile_config_setting_name_ not in config_opts:
config_opts[self._logfile_config_setting_name_] = \
getattr(self.options, self._logfile_config_setting_name_, self._default_logging_logfile_)
return config_opts
# noinspection PyUnusedLocal
@staticmethod
def set_callback(option, opt, value, parser): # pylint: disable=unused-argument
'''
Argument parser callback for handling multi-value sets.
This callback converts comma-delimited or space-delimited strings
to list types.
'''
if ',' in value:
setattr(parser.values, option.dest,
set(value.replace(' ', '').split(',')))
else:
setattr(parser.values, option.dest, set(value.split()))
class SaltStandaloneProxyOptionParser(
six.with_metaclass(
salt.utils.parsers.OptionParserMeta,
salt.utils.parsers.OptionParser,
salt.utils.parsers.ConfigDirMixIn,
salt.utils.parsers.MergeConfigMixIn,
salt.utils.parsers.TimeoutMixIn,
salt.utils.parsers.LogLevelMixIn,
salt.utils.parsers.HardCrashMixin,
salt.utils.parsers.SaltfileMixIn,
salt.utils.parsers.ExtendedTargetOptionsMixIn,
salt.utils.parsers.OutputOptionsMixIn,
salt.utils.parsers.ArgsStdinMixIn,
salt.utils.parsers.ProfilingPMixIn,
salt.utils.parsers.EAuthMixIn,
salt.utils.parsers.NoParseMixin,
)):
default_timeout = 60
description = (
r'''
___ _ _ ___ ___
/ __| __ _ | | | |_ / __| | _ \ _ _ ___ __ __ _ _
\__ \ / _` | | | | _| \__ \ | _/ | '_| / _ \ \ \ / | || |
|___/ \__,_| |_| \__| |___/ |_| |_| \___/ /_\_\ \_, |
|__/
'''
'salt-sproxy is a tool to invoke arbitrary Salt functions on a group\n'
'of (network) devices connecting through a Salt Proxy Minion, without\n'
'having the Proxy Minion services up and running (or the Master).\n')
VERSION = salt_sproxy.version.__version__
usage = '%prog [options] <target> <function> [arguments]'
epilog = ('You can find additional help about %prog at '
'https://salt-sproxy.readthedocs.io/en/latest/')
# ConfigDirMixIn config filename attribute
_config_filename_ = 'master'
# LogLevelMixIn attributes
_default_logging_level_ = config.DEFAULT_MASTER_OPTS['log_level']
_default_logging_logfile_ = config.DEFAULT_MASTER_OPTS['log_file']
def format_description(self, formatter):
return self.description
def _mixin_setup(self):
self.add_option('-r',
'--roster',
default=False,
help='The name of the Salt Roster to use.')
self.add_option(
'--roster-file',
dest='roster_file',
help='Absolute path to the Roster file to use.',
)
self.add_option(
'-s',
'--static',
default=False,
action='store_true',
help=(
'Return the data from devices as a group after they all return.'
),
)
self.add_option(
"--async",
default=False,
dest="async",
action="store_true",
help=('Run the salt-sproxy command but don\'t wait for a reply.'),
)
self.add_option(
'--dont-cache-grains',
default=False,
action='store_true',
help=('Do not cache the collected Grains for the sproxy devices.'),
)
self.add_option(
'--dont-cache-pillar',
default=False,
action='store_true',
help=('Do not cache the compiled Pillar for the sproxy devices.'),
)
self.add_option(
'--no-cached-grains',
default=False,
action='store_true',
help='Do not use the available cached Grains (if any).',
)
self.add_option(
'--no-cached-pillar',
default=False,
action='store_true',
help='Do not use the available cached Pillar (if any)',
)
self.add_option(
'--no-grains',
default=False,
action='store_true',
help=('Do not attempt to collect Grains at all. Use with care, it '
'may lead to unexpected results.'),
)
self.add_option(
'-i',
'--ignore-host-keys',
default=False,
action='store_true',
dest='ignore_host_keys',
help=
('By default ssh host keys are honored and connections will ask '
'for approval. Use this option to disable StrictHostKeyChecking.'
),
)
self.add_option(
'--no-host-keys',
default=False,
action='store_true',
dest='no_host_keys',
help=(
'Fully ignores ssh host keys which by default are honored and '
'connections would ask for approval. Useful if the host key of '
'a remote server has changed and would still error with '
'--ignore-host-keys.'),
)
self.add_option(
'--identities-only',
default=False,
action='store_true',
dest='identities_only',
help=
('Use the only authentication identity files configured in the '
'ssh_config files. See ``IdentitiesOnly`` flag in man ssh_config.'
),
)
self.add_option(
'--priv',
dest='priv',
help=
('Specify the SSH private key file to be used for authentication.'
),
)
self.add_option(
'--priv-passwd',
dest='priv_passwd',
help=(
'Specify the SSH private key file\'s passphrase when required.'
),
)
self.add_option(
'--preload-targeting',
default=False,
action='store_true',
help=(
'Preload Grains for all the devices before targeting.'
'This is useful to match the devices on dynamic Grains that '
'do not require the connection with the remote device - e.g., '
'Grains collected from an external API, etc.'),
)
self.add_option(
'--invasive-targeting',
default=False,
action='store_true',
help=
('Collect all the possible data from every device salt-sproxy '
'is aware of, before distributing the commands. '
'In other words, this option tells salt-sproxy to connect to '
'every possible device defined in the Roster of choice, collect '
'Grains, compile Pillars, etc., and only then execute the '
'command against the devices matching the target expression.'
'Use with care, as this may significantly reduce the '
'performances, but this is the price paid to be able to target '
'using device properties. '
'Consider using this option in conjunction with --cache-grains '
'and / or --cache-pillar to cache the Grains and the Pillars to '
're-use them straight away next time.'),
)
self.add_option(
'--no-pillar',
default=False,
action='store_true',
help=(
'Do not compile Pillar at all. Use with care, it may lead to '
'unexpected results.'),
)
self.add_option(
'-b',
'--batch',
'--batch-size',
dest='batch_size',
help=('The number of devices to connect to in parallel. '
'Default: {} (number of CPUs on your machine)'.format(
CPU_COUNT)),
)
self.add_option(
'--preview-target',
dest='preview_target',
action='store_true',
help='Show the devices expected to match the target.',
)
self.add_option(
'--sync-all',
dest='sync_all',
action='store_true',
help=(
'Load the all extension modules provided with salt-sproxy, as '
'well as the extension modules from your own environment.'),
)
self.add_option(
'--sync-grains',
dest='sync_grains',
action='store_true',
help=(
'Re-sync the Grains modules. Useful if you have custom Grains '
'modules in your own environment.'),
)
self.add_option(
'--sync-modules',
dest='sync_modules',
action='store_true',
help=('Load the salt-sproxy Execution modules.'),
)
self.add_option(
'--sync-roster',
dest='sync_roster',
action='store_true',
help=('Synchronise the Roster modules (both salt-sproxy native '
'and provided by the user in their own environment).'),
)
self.add_option(
'--sync-proxy',
dest='sync_proxy',
action='store_true',
help=('Load the salt-sproxy Proxy modules.'),
)
self.add_option(
'--sync-executors',
dest='sync_executors',
action='store_true',
help=('Load the salt-sproxy Executor modules.'),
)
self.add_option(
'--saltenv',
dest='saltenv_cli',
help='The Salt environment name to load module and files from',
)
self.add_option(
'--events',
dest='events',
action='store_true',
help=('Whether should put the events on the Salt bus (mostly '
'useful when having a Master running).'),
)
self.add_option(
'--use-proxy',
'--use-existing-proxy',
'--use-existing-minion',
dest='use_existing_proxy',
action='store_true',
help=('Use the existing Proxy Minions to execute the commands, '
'whenever available.'),
)
self.add_option(
'--pillar-root',
default=None,
help='Set this directory as the base pillar root.',
)
self.add_option(
'--file-root',
default=None,
help='Set this directory as the base file root.',
)
self.add_option(
'--states-dir',
default=None,
help='Set this directory to search for additional states.',
)
self.add_option(
'-m',
'--module-dirs',
dest='module_dirs_cli',
default=[],
action='append',
help=('Specify an additional directory to pull modules from. '
'Multiple directories can be provided by passing '
'`-m/--module-dirs` multiple times.'),
)
self.add_option(
'--installation-path',
dest='installation_path',
action='store_true',
help=(
'Display the absolute path to where salt-sproxy is installed.'
),
)
self.add_option(
'--display-file-roots',
dest='display_file_roots',
action='store_true',
help=(
'Display the file_roots option you would need to configure '
'in order to use the salt-sproxy extension modules directly, '
'and, implicitly, leverage the event-driven methodology and '
'the Salt REST API.'),
)
self.add_option(
'--save-file-roots',
dest='save_file_roots',
action='store_true',
help=(
'Saves the file_roots configuration so you can start '
'leveraging the event-driven automation and the Salt REST API.'
),
)
self.add_option(
'--config-dump',
dest='config_dump',
default=False,
action='store_true',
help='Dump the salt-sproxy configuration values',
)
self.add_option(
'--no-connect',
dest='no_connect',
action='store_true',
default=False,
help=(
'Do not initiate the connection with the device, only use '
'cached data to compile data and execute Salt functions that '
'do not require the actual connection with the device.'),
)
self.add_option(
'--test-ping',
dest='test_ping',
action='store_true',
default=False,
help=(
'When using together with --use-existing-proxy, this option can'
' help to ensure the existing Proxy Minion is responsive (not '
'only up and running, by executing a ping test.'),
)
self.add_option(
'--failhard',
dest='failhard',
action='store_true',
default=False,
help='Stop execution at the first execution error.',
)
self.add_option(
'--target-cache',
dest='target_cache',
action='store_true',
default=False,
help=(
'Cache the list of devices matched by your target expression.'
),
)
group = self.output_options_group = optparse.OptionGroup(
self, 'Output Options', 'Configure your preferred output format.')
self.add_option_group(group)
group.add_option(
'-q',
'--quiet',
default=False,
action='store_true',
help='Do not display the results of the run.',
)
self.add_option(
'--summary',
default=False,
action='store_true',
help='Display salt execution summary information.',
)
self.add_option(
'-v',
'--verbose',
default=False,
action='store_true',
help='Turn on command verbosity, display jid and detailed summary.',
)
self.add_option(
'--show-jid',
default=False,
action='store_true',
help='Display jid without the additional output of --verbose.',
)
self.add_option(
'--hide-timeout',
default=False,
action='store_true',
help='Hide devices that timeout.',
)
self.add_option(
'--batch-wait',
default=0,
type=float,
help=('Wait the specified time in seconds after each batch is done'
'before executing the next one.'),
)
self.add_option(
'-p',
'--progress',
default=False,
action='store_true',
help='Display a progress graph.',
)
self.add_option(
'--return',
dest='returner',
default='',
metavar='RETURNER',
help=('The name of the Returner module to use for sending data to '
'various external systems.'),
)
self.add_option(
'--return-config',
dest='returner_config',
default='',
metavar='RETURNER_CONF',
help='Specify an alternative Returner config.',
)
self.add_option(
'--return-kwargs',
dest='returner_kwargs',
default={},
metavar='RETURNER_KWARGS',
help='Set Returner options at the command line.',
)
self.add_option(
"-d",
"--doc",
"--documentation",
dest="doc",
default=False,
action="store_true",
help=('Return the documentation for the specified module or for '
'all modules if none are specified.'),
)
# Everything else that follows here is verbatim copy from
# https://github.com/saltstack/salt/blob/develop/salt/utils/parsers.py
def _mixin_after_parsed(self):
if (self.options.display_file_roots or self.options.installation_path
or self.options.save_file_roots or self.options.config_dump):
# Insert dummy arg when displaying the file_roots
self.args.append('not_a_valid_target')
self.args.append('not_a_valid_command')
elif self.options.doc:
if len(self.args) == 1:
self.args.insert(0, 'not_a_valid_target')
elif len(self.args) == 0:
self.args.append('not_a_valid_target')
self.args.append('*')
if self.options.list:
try:
if ',' in self.args[0]:
self.config['tgt'] = self.args[0].replace(' ',
'').split(',')
else:
self.config['tgt'] = self.args[0].split()
except IndexError:
self.exit(
42,
'\nCannot execute command without defining a target.\n\n')
else:
try:
self.config['tgt'] = self.args[0]
except IndexError:
self.exit(
42,
'\nCannot execute command without defining a target.\n\n')
if self.options.preview_target:
# Insert dummy arg which won't be used
self.args.append('not_a_valid_command')
# Detect compound command and set up the data for it
if self.args:
try:
if ',' in self.args[1]:
self.config['fun'] = self.args[1].split(',')
self.config['arg'] = [[]]
cmd_index = 0
if (self.args[2:].count(
self.options.args_separator) == len(
self.config['fun']) - 1):
# new style parsing: standalone argument separator
for arg in self.args[2:]:
if arg == self.options.args_separator:
cmd_index += 1
self.config['arg'].append([])
else:
self.config['arg'][cmd_index].append(arg)
else:
# old style parsing: argument separator can be inside args
for arg in self.args[2:]:
if self.options.args_separator in arg:
sub_args = arg.split(
self.options.args_separator)
for sub_arg_index, sub_arg in enumerate(
sub_args):
if sub_arg:
self.config['arg'][cmd_index].append(
sub_arg)
if sub_arg_index != len(sub_args) - 1:
cmd_index += 1
self.config['arg'].append([])
else:
self.config['arg'][cmd_index].append(arg)
if len(self.config['fun']) > len(self.config['arg']):
self.exit(
42,
'Cannot execute compound command without '
'defining all arguments.\n',
)
elif len(self.config['fun']) < len(self.config['arg']):
self.exit(
42,
'Cannot execute compound command with more '
'arguments than commands.\n',
)
# parse the args and kwargs before sending to the publish
# interface
for i in range(len(self.config['arg'])):
self.config['arg'][i] = salt.utils.args.parse_input(
self.config['arg'][i],
no_parse=self.options.no_parse)
else:
self.config['fun'] = self.args[1]
self.config['arg'] = self.args[2:]
# parse the args and kwargs before sending to the publish
# interface
self.config['arg'] = salt.utils.args.parse_input(
self.config['arg'], no_parse=self.options.no_parse)
except IndexError:
self.exit(42, '\nIncomplete options passed.\n\n')
def setup_config(self):
defaults = config.DEFAULT_MASTER_OPTS.copy()
defaults['timeout'] = 60
return config.client_config(self.get_config_file_path(),
defaults=defaults)