def _expand_paths(paths):
"""
Expand any comma-separated lists of paths, and return a set of all
paths to ensure there are no duplicates.
"""
ret = set()
for path in paths:
for item in [x.strip() for x in path.split(",")]:
if not item:
continue
elif os.path.isabs(item):
try:
with salt.utils.files.fopen(item, "rb") as fp_:
for line in fp_:
line = salt.utils.stringutils.to_unicode(
line.strip())
if os.path.isabs(line):
log.warning(
"Invalid absolute path %s in %s, "
"ignoring",
line,
item,
)
else:
ret.add(line)
except OSError as exc:
log.error("Failed to read from %s: %s", item, exc)
else:
if not os.path.exists(item):
log.info("%s does not exist. Skipping...", item)
continue
ret.add(item)
return ret
def find_file(path, tgt_env='base', **kwargs): # pylint: disable=W0613
'''
Search the environment for the relative path
'''
fnd = {'path': '', 'rel': ''}
if os.path.isabs(path):
return fnd
if tgt_env not in envs():
return fnd
if os.path.basename(path) == 'top.sls':
log.debug(
'minionfs will NOT serve top.sls '
'for security reasons (path requested: %s)', path)
return fnd
mountpoint = salt.utils.url.strip_proto(__opts__['minionfs_mountpoint'])
# Remove the mountpoint to get the "true" path
path = path[len(mountpoint):].lstrip(os.path.sep)
try:
minion, pushed_file = path.split(os.sep, 1)
except ValueError:
return fnd
if not _is_exposed(minion):
return fnd
full = os.path.join(__opts__['cachedir'], 'minions', minion, 'files',
pushed_file)
if os.path.isfile(full) \
and not salt.fileserver.is_file_ignored(__opts__, full):
fnd['path'] = full
fnd['rel'] = path
fnd['stat'] = list(os.stat(full))
return fnd
return fnd
def find_file(path, tgt_env="base", **kwargs): # pylint: disable=W0613
"""
Search the environment for the relative path
"""
fnd = {"path": "", "rel": ""}
if os.path.isabs(path):
return fnd
if tgt_env not in envs():
return fnd
if os.path.basename(path) == "top.sls":
log.debug(
"minionfs will NOT serve top.sls for security reasons (path requested: %s)",
path,
)
return fnd
mountpoint = salt.utils.url.strip_proto(__opts__["minionfs_mountpoint"])
# Remove the mountpoint to get the "true" path
path = path[len(mountpoint):].lstrip(os.path.sep)
try:
minion, pushed_file = path.split(os.sep, 1)
except ValueError:
return fnd
if not _is_exposed(minion):
return fnd
full = os.path.join(__opts__["cachedir"], "minions", minion, "files",
pushed_file)
if os.path.isfile(full) and not salt.fileserver.is_file_ignored(
__opts__, full):
fnd["path"] = full
fnd["rel"] = path
fnd["stat"] = list(os.stat(full))
return fnd
return fnd
def _expand_paths(paths):
'''
Expand any comma-separated lists of paths, and return a set of all
paths to ensure there are no duplicates.
'''
ret = set()
for path in paths:
for item in [x.strip() for x in path.split(',')]:
if not item:
continue
elif os.path.isabs(item):
try:
with salt.utils.files.fopen(item, 'rb') as fp_:
for line in fp_:
line = salt.utils.stringutils.to_unicode(
line.strip())
if os.path.isabs(line):
log.warning(
'Invalid absolute path %s in %s, '
'ignoring', line, item)
else:
ret.add(line)
except (IOError, OSError) as exc:
log.error('Failed to read from %s: %s', item, exc)
else:
ret.add(item)
return ret
def _exit_mockbin(self):
path = os.environ.get('PATH', '')
path_items = path.split(os.pathsep)
try:
path_items.remove(MOCKBIN)
except ValueError:
pass
os.environ['PATH'] = os.pathsep.join(path_items)
def _realpath_darwin(path):
base = ""
for part in path.split(os.path.sep)[1:]:
if base != "":
if os.path.islink(os.path.sep.join([base, part])):
base = os.readlink(os.path.sep.join([base, part]))
else:
base = os.path.abspath(os.path.sep.join([base, part]))
else:
base = os.path.abspath(os.path.sep.join([base, part]))
return base
def _realpath_windows(path):
base = ""
for part in path.split(os.path.sep):
if base != "":
try:
part = os.readlink(os.path.sep.join([base, part]))
base = os.path.abspath(part)
except OSError:
base = os.path.abspath(os.path.sep.join([base, part]))
else:
base = part
return base
def _expand_paths(paths):
'''
Expand any comma-separated lists of paths, and return a set of all
paths to ensure there are no duplicates.
'''
ret = set()
for path in paths:
if ',' in path:
ret.update([x.strip() for x in path.split(',')])
else:
ret.add(path.strip())
return ret
def _realpath_windows(path):
base = ""
for part in path.split(os.path.sep):
if base != "":
try:
# Need to use salt.utils.path.readlink as it handles junctions
part = salt.utils.path.readlink(os.path.sep.join([base, part]))
base = os.path.abspath(part)
except OSError:
base = os.path.abspath(os.path.sep.join([base, part]))
else:
base = part
return base
def find_file(self, path, saltenv, back=None):
'''
Find the path and return the fnd structure, this structure is passed
to other backend interfaces.
'''
path = salt.utils.stringutils.to_unicode(path)
saltenv = salt.utils.stringutils.to_unicode(saltenv)
back = self.backends(back)
kwargs = {}
fnd = {'path': '',
'rel': ''}
if os.path.isabs(path):
return fnd
if '../' in path:
return fnd
if salt.utils.url.is_escaped(path):
# don't attempt to find URL query arguments in the path
path = salt.utils.url.unescape(path)
else:
if '?' in path:
hcomps = path.split('?')
path = hcomps[0]
comps = hcomps[1].split('&')
for comp in comps:
if '=' not in comp:
# Invalid option, skip it
continue
args = comp.split('=', 1)
kwargs[args[0]] = args[1]
if 'env' in kwargs:
# "env" is not supported; Use "saltenv".
kwargs.pop('env')
if 'saltenv' in kwargs:
saltenv = kwargs.pop('saltenv')
if not isinstance(saltenv, six.string_types):
saltenv = six.text_type(saltenv)
for fsb in back:
fstr = '{0}.find_file'.format(fsb)
if fstr in self.servers:
fnd = self.servers[fstr](path, saltenv, **kwargs)
if fnd.get('path'):
fnd['back'] = fsb
return fnd
return fnd
def find_file(self, path, saltenv, back=None):
"""
Find the path and return the fnd structure, this structure is passed
to other backend interfaces.
"""
path = salt.utils.stringutils.to_unicode(path)
saltenv = salt.utils.stringutils.to_unicode(saltenv)
back = self.backends(back)
kwargs = {}
fnd = {"path": "", "rel": ""}
if os.path.isabs(path):
return fnd
if "../" in path:
return fnd
if salt.utils.url.is_escaped(path):
# don't attempt to find URL query arguments in the path
path = salt.utils.url.unescape(path)
else:
if "?" in path:
hcomps = path.split("?")
path = hcomps[0]
comps = hcomps[1].split("&")
for comp in comps:
if "=" not in comp:
# Invalid option, skip it
continue
args = comp.split("=", 1)
kwargs[args[0]] = args[1]
if "env" in kwargs:
# "env" is not supported; Use "saltenv".
kwargs.pop("env")
if "saltenv" in kwargs:
saltenv = kwargs.pop("saltenv")
if not isinstance(saltenv, six.string_types):
saltenv = six.text_type(saltenv)
for fsb in back:
fstr = "{0}.find_file".format(fsb)
if fstr in self.servers:
fnd = self.servers[fstr](path, saltenv, **kwargs)
if fnd.get("path"):
fnd["back"] = fsb
return fnd
return fnd
def _realpath_windows(path):
base = ""
for part in path.split(os.path.sep):
if base != "":
try:
# Need to use salt.utils.path.readlink as it handles junctions
part = salt.utils.path.readlink(os.path.sep.join([base, part]))
base = os.path.abspath(part)
except OSError:
base = os.path.abspath(os.path.sep.join([base, part]))
else:
base = part
# Python 3.8 added support for directory junctions which prefixes the
# return with `\\?\`. We need to strip that off.
# https://docs.python.org/3/library/os.html#os.readlink
if base.startswith("\\\\?\\"):
base = base[4:]
return base
def _get_virtualenv_binary_path():
try:
return _get_virtualenv_binary_path.__virtualenv_binary__
except AttributeError:
# Under windows we can't seem to properly create a virtualenv off of another
# virtualenv, we can on linux but we will still point to the virtualenv binary
# outside the virtualenv running the test suite, if that's the case.
try:
real_prefix = sys.real_prefix
# The above attribute exists, this is a virtualenv
if salt.utils.platform.is_windows():
virtualenv_binary = os.path.join(real_prefix, "Scripts",
"virtualenv.exe")
else:
# We need to remove the virtualenv from PATH or we'll get the virtualenv binary
# from within the virtualenv, we don't want that
path = os.environ.get("PATH")
if path is not None:
path_items = path.split(os.pathsep)
for item in path_items[:]:
if item.startswith(sys.base_prefix):
path_items.remove(item)
os.environ["PATH"] = os.pathsep.join(path_items)
virtualenv_binary = salt.utils.path.which("virtualenv")
if path is not None:
# Restore previous environ PATH
os.environ["PATH"] = path
if not virtualenv_binary.startswith(real_prefix):
virtualenv_binary = None
if virtualenv_binary and not os.path.exists(virtualenv_binary):
# It doesn't exist?!
virtualenv_binary = None
except AttributeError:
# We're not running inside a virtualenv
virtualenv_binary = None
_get_virtualenv_binary_path.__virtualenv_binary__ = virtualenv_binary
return virtualenv_binary
def transplant_configs(cls, transport='zeromq'):
os.makedirs(RUNTIME_VARS.TMP_MM_CONF_DIR)
os.makedirs(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR)
print(' * Transplanting multimaster configuration files to \'{0}\''.
format(RUNTIME_VARS.TMP_CONF_DIR))
tests_known_hosts_file = os.path.join(RUNTIME_VARS.TMP_CONF_DIR,
'salt_ssh_known_hosts')
# Primary master in multimaster environment
master_opts = salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, 'master'))
master_opts.update(
salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, 'mm_master')))
master_opts['known_hosts_file'] = tests_known_hosts_file
master_opts['cachedir'] = 'cache'
master_opts['user'] = RUNTIME_VARS.RUNNING_TESTS_USER
master_opts['config_dir'] = RUNTIME_VARS.TMP_MM_CONF_DIR
master_opts['root_dir'] = os.path.join(TMP, 'rootdir-multimaster')
master_opts['pki_dir'] = 'pki'
file_tree = {
'root_dir': os.path.join(FILES, 'pillar', 'base', 'file_tree'),
'follow_dir_links': False,
'keep_newline': True,
}
master_opts['ext_pillar'].append({'file_tree': file_tree})
# Secondary master in multimaster environment
sub_master_opts = salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, 'master'))
sub_master_opts.update(
salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, 'mm_sub_master')))
sub_master_opts['known_hosts_file'] = tests_known_hosts_file
sub_master_opts['cachedir'] = 'cache'
sub_master_opts['user'] = RUNTIME_VARS.RUNNING_TESTS_USER
sub_master_opts['config_dir'] = RUNTIME_VARS.TMP_MM_SUB_CONF_DIR
sub_master_opts['root_dir'] = os.path.join(TMP,
'rootdir-sub-multimaster')
sub_master_opts['pki_dir'] = 'pki'
sub_master_opts['ext_pillar'].append(
{'file_tree': copy.deepcopy(file_tree)})
# Under windows we can't seem to properly create a virtualenv off of another
# virtualenv, we can on linux but we will still point to the virtualenv binary
# outside the virtualenv running the test suite, if that's the case.
try:
real_prefix = sys.real_prefix
# The above attribute exists, this is a virtualenv
if salt.utils.platform.is_windows():
virtualenv_binary = os.path.join(real_prefix, 'Scripts',
'virtualenv.exe')
else:
# We need to remove the virtualenv from PATH or we'll get the virtualenv binary
# from within the virtualenv, we don't want that
path = os.environ.get('PATH')
if path is not None:
path_items = path.split(os.pathsep)
for item in path_items[:]:
if item.startswith(sys.base_prefix):
path_items.remove(item)
os.environ['PATH'] = os.pathsep.join(path_items)
virtualenv_binary = salt.utils.path.which('virtualenv')
if path is not None:
# Restore previous environ PATH
os.environ['PATH'] = path
if not virtualenv_binary.startswith(real_prefix):
virtualenv_binary = None
if virtualenv_binary and not os.path.exists(virtualenv_binary):
# It doesn't exist?!
virtualenv_binary = None
except AttributeError:
# We're not running inside a virtualenv
virtualenv_binary = None
# This minion connects to both masters
minion_opts = salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, 'minion'))
minion_opts.update(
salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, 'mm_minion')))
minion_opts['cachedir'] = 'cache'
minion_opts['user'] = RUNTIME_VARS.RUNNING_TESTS_USER
minion_opts['config_dir'] = RUNTIME_VARS.TMP_MM_CONF_DIR
minion_opts['root_dir'] = os.path.join(TMP, 'rootdir-multimaster')
minion_opts['pki_dir'] = 'pki'
minion_opts['hosts.file'] = os.path.join(TMP, 'rootdir', 'hosts')
minion_opts['aliases.file'] = os.path.join(TMP, 'rootdir', 'aliases')
if virtualenv_binary:
minion_opts['venv_bin'] = virtualenv_binary
# This sub_minion also connects to both masters
sub_minion_opts = salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, 'sub_minion'))
sub_minion_opts.update(
salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, 'mm_sub_minion')))
sub_minion_opts['cachedir'] = 'cache'
sub_minion_opts['user'] = RUNTIME_VARS.RUNNING_TESTS_USER
sub_minion_opts['config_dir'] = RUNTIME_VARS.TMP_MM_SUB_CONF_DIR
sub_minion_opts['root_dir'] = os.path.join(TMP,
'rootdir-sub-multimaster')
sub_minion_opts['pki_dir'] = 'pki'
sub_minion_opts['hosts.file'] = os.path.join(TMP, 'rootdir', 'hosts')
sub_minion_opts['aliases.file'] = os.path.join(TMP, 'rootdir',
'aliases')
if virtualenv_binary:
sub_minion_opts['venv_bin'] = virtualenv_binary
if transport == 'raet':
master_opts['transport'] = 'raet'
master_opts['raet_port'] = 64506
sub_master_opts['transport'] = 'raet'
sub_master_opts['raet_port'] = 64556
minion_opts['transport'] = 'raet'
minion_opts['raet_port'] = 64510
sub_minion_opts['transport'] = 'raet'
sub_minion_opts['raet_port'] = 64520
# syndic_master_opts['transport'] = 'raet'
if transport == 'tcp':
master_opts['transport'] = 'tcp'
sub_master_opts['transport'] = 'tcp'
minion_opts['transport'] = 'tcp'
sub_minion_opts['transport'] = 'tcp'
# Set up config options that require internal data
master_opts['pillar_roots'] = sub_master_opts['pillar_roots'] = {
'base': [
RUNTIME_VARS.TMP_PILLAR_TREE,
os.path.join(FILES, 'pillar', 'base'),
]
}
minion_opts['pillar_roots'] = {
'base': [
RUNTIME_VARS.TMP_PILLAR_TREE,
os.path.join(FILES, 'pillar', 'base'),
]
}
master_opts['file_roots'] = sub_master_opts['file_roots'] = {
'base': [
os.path.join(FILES, 'file', 'base'),
# Let's support runtime created files that can be used like:
# salt://my-temp-file.txt
RUNTIME_VARS.TMP_STATE_TREE
],
# Alternate root to test __env__ choices
'prod': [
os.path.join(FILES, 'file', 'prod'),
RUNTIME_VARS.TMP_PRODENV_STATE_TREE
]
}
minion_opts['file_roots'] = {
'base': [
os.path.join(FILES, 'file', 'base'),
# Let's support runtime created files that can be used like:
# salt://my-temp-file.txt
RUNTIME_VARS.TMP_STATE_TREE
],
# Alternate root to test __env__ choices
'prod': [
os.path.join(FILES, 'file', 'prod'),
RUNTIME_VARS.TMP_PRODENV_STATE_TREE
]
}
master_opts.setdefault('reactor', []).append({
'salt/minion/*/start':
[os.path.join(FILES, 'reactor-sync-minion.sls')],
})
for opts_dict in (master_opts, sub_master_opts):
if 'ext_pillar' not in opts_dict:
opts_dict['ext_pillar'] = []
if salt.utils.platform.is_windows():
opts_dict['ext_pillar'].append({
'cmd_yaml':
'type {0}'.format(os.path.join(FILES, 'ext.yaml'))
})
else:
opts_dict['ext_pillar'].append({
'cmd_yaml':
'cat {0}'.format(os.path.join(FILES, 'ext.yaml'))
})
# all read, only owner write
autosign_file_permissions = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR
for opts_dict in (master_opts, sub_master_opts):
# We need to copy the extension modules into the new master root_dir or
# it will be prefixed by it
new_extension_modules_path = os.path.join(opts_dict['root_dir'],
'extension_modules')
if not os.path.exists(new_extension_modules_path):
shutil.copytree(
os.path.join(INTEGRATION_TEST_DIR, 'files',
'extension_modules'),
new_extension_modules_path)
opts_dict['extension_modules'] = os.path.join(
opts_dict['root_dir'], 'extension_modules')
# Copy the autosign_file to the new master root_dir
new_autosign_file_path = os.path.join(opts_dict['root_dir'],
'autosign_file')
shutil.copyfile(
os.path.join(INTEGRATION_TEST_DIR, 'files', 'autosign_file'),
new_autosign_file_path)
os.chmod(new_autosign_file_path, autosign_file_permissions)
# Point the config values to the correct temporary paths
for name in ('hosts', 'aliases'):
optname = '{0}.file'.format(name)
optname_path = os.path.join(TMP, name)
master_opts[optname] = optname_path
sub_master_opts[optname] = optname_path
minion_opts[optname] = optname_path
sub_minion_opts[optname] = optname_path
master_opts['runtests_conn_check_port'] = get_unused_localhost_port()
sub_master_opts[
'runtests_conn_check_port'] = get_unused_localhost_port()
minion_opts['runtests_conn_check_port'] = get_unused_localhost_port()
sub_minion_opts[
'runtests_conn_check_port'] = get_unused_localhost_port()
for conf in (master_opts, sub_master_opts, minion_opts,
sub_minion_opts):
if 'engines' not in conf:
conf['engines'] = []
conf['engines'].append({'salt_runtests': {}})
if 'engines_dirs' not in conf:
conf['engines_dirs'] = []
conf['engines_dirs'].insert(0, ENGINES_DIR)
if 'log_handlers_dirs' not in conf:
conf['log_handlers_dirs'] = []
conf['log_handlers_dirs'].insert(0, LOG_HANDLERS_DIR)
conf['runtests_log_port'] = SALT_LOG_PORT
conf['runtests_log_level'] = os.environ.get(
'TESTS_MIN_LOG_LEVEL_NAME') or 'debug'
# ----- Transcribe Configuration ---------------------------------------------------------------------------->
computed_config = copy.deepcopy(master_opts)
with salt.utils.files.fopen(
os.path.join(RUNTIME_VARS.TMP_MM_CONF_DIR, 'master'),
'w') as wfh:
salt.utils.yaml.safe_dump(copy.deepcopy(master_opts),
wfh,
default_flow_style=False)
with salt.utils.files.fopen(
os.path.join(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR, 'master'),
'w') as wfh:
salt.utils.yaml.safe_dump(copy.deepcopy(sub_master_opts),
wfh,
default_flow_style=False)
with salt.utils.files.fopen(
os.path.join(RUNTIME_VARS.TMP_MM_CONF_DIR, 'minion'),
'w') as wfh:
salt.utils.yaml.safe_dump(copy.deepcopy(minion_opts),
wfh,
default_flow_style=False)
with salt.utils.files.fopen(
os.path.join(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR, 'minion'),
'w') as wfh:
salt.utils.yaml.safe_dump(copy.deepcopy(sub_minion_opts),
wfh,
default_flow_style=False)
# <---- Transcribe Configuration -----------------------------------------------------------------------------
# ----- Verify Environment ---------------------------------------------------------------------------------->
master_opts = salt.config.master_config(
os.path.join(RUNTIME_VARS.TMP_MM_CONF_DIR, 'master'))
sub_master_opts = salt.config.master_config(
os.path.join(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR, 'master'))
minion_opts = salt.config.minion_config(
os.path.join(RUNTIME_VARS.TMP_MM_CONF_DIR, 'minion'))
sub_minion_opts = salt.config.minion_config(
os.path.join(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR, 'minion'))
RUNTIME_VARS.RUNTIME_CONFIGS['mm_master'] = freeze(master_opts)
RUNTIME_VARS.RUNTIME_CONFIGS['mm_sub_master'] = freeze(sub_master_opts)
RUNTIME_VARS.RUNTIME_CONFIGS['mm_minion'] = freeze(minion_opts)
RUNTIME_VARS.RUNTIME_CONFIGS['mm_sub_minion'] = freeze(sub_minion_opts)
verify_env(
[
os.path.join(master_opts['pki_dir'], 'minions'),
os.path.join(master_opts['pki_dir'], 'minions_pre'),
os.path.join(master_opts['pki_dir'], 'minions_rejected'),
os.path.join(master_opts['pki_dir'], 'minions_denied'),
os.path.join(master_opts['cachedir'], 'jobs'),
os.path.join(master_opts['cachedir'], 'raet'),
os.path.join(master_opts['root_dir'], 'cache', 'tokens'),
os.path.join(master_opts['pki_dir'], 'accepted'),
os.path.join(master_opts['pki_dir'], 'rejected'),
os.path.join(master_opts['pki_dir'], 'pending'),
os.path.join(master_opts['cachedir'], 'raet'),
os.path.join(sub_master_opts['pki_dir'], 'minions'),
os.path.join(sub_master_opts['pki_dir'], 'minions_pre'),
os.path.join(sub_master_opts['pki_dir'], 'minions_rejected'),
os.path.join(sub_master_opts['pki_dir'], 'minions_denied'),
os.path.join(sub_master_opts['cachedir'], 'jobs'),
os.path.join(sub_master_opts['cachedir'], 'raet'),
os.path.join(sub_master_opts['root_dir'], 'cache', 'tokens'),
os.path.join(sub_master_opts['pki_dir'], 'accepted'),
os.path.join(sub_master_opts['pki_dir'], 'rejected'),
os.path.join(sub_master_opts['pki_dir'], 'pending'),
os.path.join(sub_master_opts['cachedir'], 'raet'),
os.path.join(minion_opts['pki_dir'], 'accepted'),
os.path.join(minion_opts['pki_dir'], 'rejected'),
os.path.join(minion_opts['pki_dir'], 'pending'),
os.path.join(minion_opts['cachedir'], 'raet'),
os.path.join(sub_minion_opts['pki_dir'], 'accepted'),
os.path.join(sub_minion_opts['pki_dir'], 'rejected'),
os.path.join(sub_minion_opts['pki_dir'], 'pending'),
os.path.join(sub_minion_opts['cachedir'], 'raet'),
os.path.dirname(master_opts['log_file']),
minion_opts['extension_modules'],
sub_minion_opts['extension_modules'],
sub_minion_opts['pki_dir'],
master_opts['sock_dir'],
sub_master_opts['sock_dir'],
sub_minion_opts['sock_dir'],
minion_opts['sock_dir'],
],
RUNTIME_VARS.RUNNING_TESTS_USER,
root_dir=master_opts['root_dir'],
)
cls.mm_master_opts = master_opts
cls.mm_sub_master_opts = sub_master_opts
cls.mm_minion_opts = minion_opts
cls.mm_sub_minion_opts = sub_minion_opts
def transplant_configs(cls, transport="zeromq"):
os.makedirs(RUNTIME_VARS.TMP_MM_CONF_DIR)
os.makedirs(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR)
print(
" * Transplanting multimaster configuration files to '{0}'".format(
RUNTIME_VARS.TMP_CONF_DIR))
tests_known_hosts_file = os.path.join(RUNTIME_VARS.TMP_CONF_DIR,
"salt_ssh_known_hosts")
# Primary master in multimaster environment
master_opts = salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, "master"))
master_opts.update(
salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, "mm_master")))
master_opts["known_hosts_file"] = tests_known_hosts_file
master_opts["cachedir"] = "cache"
master_opts["user"] = RUNTIME_VARS.RUNNING_TESTS_USER
master_opts["config_dir"] = RUNTIME_VARS.TMP_MM_CONF_DIR
master_opts["root_dir"] = os.path.join(TMP, "rootdir-multimaster")
master_opts["pki_dir"] = "pki"
file_tree = {
"root_dir": os.path.join(FILES, "pillar", "base", "file_tree"),
"follow_dir_links": False,
"keep_newline": True,
}
master_opts["ext_pillar"].append({"file_tree": file_tree})
# Secondary master in multimaster environment
sub_master_opts = salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, "master"))
sub_master_opts.update(
salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, "mm_sub_master")))
sub_master_opts["known_hosts_file"] = tests_known_hosts_file
sub_master_opts["cachedir"] = "cache"
sub_master_opts["user"] = RUNTIME_VARS.RUNNING_TESTS_USER
sub_master_opts["config_dir"] = RUNTIME_VARS.TMP_MM_SUB_CONF_DIR
sub_master_opts["root_dir"] = os.path.join(TMP,
"rootdir-sub-multimaster")
sub_master_opts["pki_dir"] = "pki"
sub_master_opts["ext_pillar"].append(
{"file_tree": copy.deepcopy(file_tree)})
# Under windows we can't seem to properly create a virtualenv off of another
# virtualenv, we can on linux but we will still point to the virtualenv binary
# outside the virtualenv running the test suite, if that's the case.
try:
real_prefix = sys.real_prefix
# The above attribute exists, this is a virtualenv
if salt.utils.platform.is_windows():
virtualenv_binary = os.path.join(real_prefix, "Scripts",
"virtualenv.exe")
else:
# We need to remove the virtualenv from PATH or we'll get the virtualenv binary
# from within the virtualenv, we don't want that
path = os.environ.get("PATH")
if path is not None:
path_items = path.split(os.pathsep)
for item in path_items[:]:
if item.startswith(sys.base_prefix):
path_items.remove(item)
os.environ["PATH"] = os.pathsep.join(path_items)
virtualenv_binary = salt.utils.path.which("virtualenv")
if path is not None:
# Restore previous environ PATH
os.environ["PATH"] = path
if not virtualenv_binary.startswith(real_prefix):
virtualenv_binary = None
if virtualenv_binary and not os.path.exists(virtualenv_binary):
# It doesn't exist?!
virtualenv_binary = None
except AttributeError:
# We're not running inside a virtualenv
virtualenv_binary = None
# This minion connects to both masters
minion_opts = salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, "minion"))
minion_opts.update(
salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, "mm_minion")))
minion_opts["cachedir"] = "cache"
minion_opts["user"] = RUNTIME_VARS.RUNNING_TESTS_USER
minion_opts["config_dir"] = RUNTIME_VARS.TMP_MM_CONF_DIR
minion_opts["root_dir"] = os.path.join(TMP, "rootdir-multimaster")
minion_opts["pki_dir"] = "pki"
minion_opts["hosts.file"] = os.path.join(TMP, "rootdir", "hosts")
minion_opts["aliases.file"] = os.path.join(TMP, "rootdir", "aliases")
if virtualenv_binary:
minion_opts["venv_bin"] = virtualenv_binary
# This sub_minion also connects to both masters
sub_minion_opts = salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, "sub_minion"))
sub_minion_opts.update(
salt.config._read_conf_file(
os.path.join(RUNTIME_VARS.CONF_DIR, "mm_sub_minion")))
sub_minion_opts["cachedir"] = "cache"
sub_minion_opts["user"] = RUNTIME_VARS.RUNNING_TESTS_USER
sub_minion_opts["config_dir"] = RUNTIME_VARS.TMP_MM_SUB_CONF_DIR
sub_minion_opts["root_dir"] = os.path.join(TMP,
"rootdir-sub-multimaster")
sub_minion_opts["pki_dir"] = "pki"
sub_minion_opts["hosts.file"] = os.path.join(TMP, "rootdir", "hosts")
sub_minion_opts["aliases.file"] = os.path.join(TMP, "rootdir",
"aliases")
if virtualenv_binary:
sub_minion_opts["venv_bin"] = virtualenv_binary
if transport == "raet":
master_opts["transport"] = "raet"
master_opts["raet_port"] = 64506
sub_master_opts["transport"] = "raet"
sub_master_opts["raet_port"] = 64556
minion_opts["transport"] = "raet"
minion_opts["raet_port"] = 64510
sub_minion_opts["transport"] = "raet"
sub_minion_opts["raet_port"] = 64520
# syndic_master_opts['transport'] = 'raet'
if transport == "tcp":
master_opts["transport"] = "tcp"
sub_master_opts["transport"] = "tcp"
minion_opts["transport"] = "tcp"
sub_minion_opts["transport"] = "tcp"
# Set up config options that require internal data
master_opts["pillar_roots"] = sub_master_opts["pillar_roots"] = {
"base": [
RUNTIME_VARS.TMP_PILLAR_TREE,
os.path.join(FILES, "pillar", "base"),
]
}
minion_opts["pillar_roots"] = {
"base": [
RUNTIME_VARS.TMP_PILLAR_TREE,
os.path.join(FILES, "pillar", "base"),
]
}
master_opts["file_roots"] = sub_master_opts["file_roots"] = {
"base": [
os.path.join(FILES, "file", "base"),
# Let's support runtime created files that can be used like:
# salt://my-temp-file.txt
RUNTIME_VARS.TMP_STATE_TREE,
],
# Alternate root to test __env__ choices
"prod": [
os.path.join(FILES, "file", "prod"),
RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
],
}
minion_opts["file_roots"] = {
"base": [
os.path.join(FILES, "file", "base"),
# Let's support runtime created files that can be used like:
# salt://my-temp-file.txt
RUNTIME_VARS.TMP_STATE_TREE,
],
# Alternate root to test __env__ choices
"prod": [
os.path.join(FILES, "file", "prod"),
RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
],
}
master_opts.setdefault("reactor", []).append({
"salt/minion/*/start":
[os.path.join(FILES, "reactor-sync-minion.sls")]
})
for opts_dict in (master_opts, sub_master_opts):
if "ext_pillar" not in opts_dict:
opts_dict["ext_pillar"] = []
if salt.utils.platform.is_windows():
opts_dict["ext_pillar"].append({
"cmd_yaml":
"type {0}".format(os.path.join(FILES, "ext.yaml"))
})
else:
opts_dict["ext_pillar"].append({
"cmd_yaml":
"cat {0}".format(os.path.join(FILES, "ext.yaml"))
})
# all read, only owner write
autosign_file_permissions = (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
| stat.S_IWUSR)
for opts_dict in (master_opts, sub_master_opts):
# We need to copy the extension modules into the new master root_dir or
# it will be prefixed by it
new_extension_modules_path = os.path.join(opts_dict["root_dir"],
"extension_modules")
if not os.path.exists(new_extension_modules_path):
shutil.copytree(
os.path.join(INTEGRATION_TEST_DIR, "files",
"extension_modules"),
new_extension_modules_path,
)
opts_dict["extension_modules"] = os.path.join(
opts_dict["root_dir"], "extension_modules")
# Copy the autosign_file to the new master root_dir
new_autosign_file_path = os.path.join(opts_dict["root_dir"],
"autosign_file")
shutil.copyfile(
os.path.join(INTEGRATION_TEST_DIR, "files", "autosign_file"),
new_autosign_file_path,
)
os.chmod(new_autosign_file_path, autosign_file_permissions)
# Point the config values to the correct temporary paths
for name in ("hosts", "aliases"):
optname = "{0}.file".format(name)
optname_path = os.path.join(TMP, name)
master_opts[optname] = optname_path
sub_master_opts[optname] = optname_path
minion_opts[optname] = optname_path
sub_minion_opts[optname] = optname_path
master_opts["runtests_conn_check_port"] = get_unused_localhost_port()
sub_master_opts[
"runtests_conn_check_port"] = get_unused_localhost_port()
minion_opts["runtests_conn_check_port"] = get_unused_localhost_port()
sub_minion_opts[
"runtests_conn_check_port"] = get_unused_localhost_port()
for conf in (master_opts, sub_master_opts, minion_opts,
sub_minion_opts):
if "engines" not in conf:
conf["engines"] = []
conf["engines"].append({"salt_runtests": {}})
if "engines_dirs" not in conf:
conf["engines_dirs"] = []
conf["engines_dirs"].insert(0, ENGINES_DIR)
if "log_handlers_dirs" not in conf:
conf["log_handlers_dirs"] = []
conf["log_handlers_dirs"].insert(0, LOG_HANDLERS_DIR)
conf["runtests_log_port"] = SALT_LOG_PORT
conf["runtests_log_level"] = (
os.environ.get("TESTS_MIN_LOG_LEVEL_NAME") or "debug")
# ----- Transcribe Configuration ---------------------------------------------------------------------------->
computed_config = copy.deepcopy(master_opts)
with salt.utils.files.fopen(
os.path.join(RUNTIME_VARS.TMP_MM_CONF_DIR, "master"),
"w") as wfh:
salt.utils.yaml.safe_dump(copy.deepcopy(master_opts),
wfh,
default_flow_style=False)
with salt.utils.files.fopen(
os.path.join(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR, "master"),
"w") as wfh:
salt.utils.yaml.safe_dump(copy.deepcopy(sub_master_opts),
wfh,
default_flow_style=False)
with salt.utils.files.fopen(
os.path.join(RUNTIME_VARS.TMP_MM_CONF_DIR, "minion"),
"w") as wfh:
salt.utils.yaml.safe_dump(copy.deepcopy(minion_opts),
wfh,
default_flow_style=False)
with salt.utils.files.fopen(
os.path.join(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR, "minion"),
"w") as wfh:
salt.utils.yaml.safe_dump(copy.deepcopy(sub_minion_opts),
wfh,
default_flow_style=False)
# <---- Transcribe Configuration -----------------------------------------------------------------------------
# ----- Verify Environment ---------------------------------------------------------------------------------->
master_opts = salt.config.master_config(
os.path.join(RUNTIME_VARS.TMP_MM_CONF_DIR, "master"))
sub_master_opts = salt.config.master_config(
os.path.join(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR, "master"))
minion_opts = salt.config.minion_config(
os.path.join(RUNTIME_VARS.TMP_MM_CONF_DIR, "minion"))
sub_minion_opts = salt.config.minion_config(
os.path.join(RUNTIME_VARS.TMP_MM_SUB_CONF_DIR, "minion"))
RUNTIME_VARS.RUNTIME_CONFIGS["mm_master"] = freeze(master_opts)
RUNTIME_VARS.RUNTIME_CONFIGS["mm_sub_master"] = freeze(sub_master_opts)
RUNTIME_VARS.RUNTIME_CONFIGS["mm_minion"] = freeze(minion_opts)
RUNTIME_VARS.RUNTIME_CONFIGS["mm_sub_minion"] = freeze(sub_minion_opts)
verify_env(
[
os.path.join(master_opts["pki_dir"], "minions"),
os.path.join(master_opts["pki_dir"], "minions_pre"),
os.path.join(master_opts["pki_dir"], "minions_rejected"),
os.path.join(master_opts["pki_dir"], "minions_denied"),
os.path.join(master_opts["cachedir"], "jobs"),
os.path.join(master_opts["cachedir"], "raet"),
os.path.join(master_opts["root_dir"], "cache", "tokens"),
os.path.join(master_opts["pki_dir"], "accepted"),
os.path.join(master_opts["pki_dir"], "rejected"),
os.path.join(master_opts["pki_dir"], "pending"),
os.path.join(master_opts["cachedir"], "raet"),
os.path.join(sub_master_opts["pki_dir"], "minions"),
os.path.join(sub_master_opts["pki_dir"], "minions_pre"),
os.path.join(sub_master_opts["pki_dir"], "minions_rejected"),
os.path.join(sub_master_opts["pki_dir"], "minions_denied"),
os.path.join(sub_master_opts["cachedir"], "jobs"),
os.path.join(sub_master_opts["cachedir"], "raet"),
os.path.join(sub_master_opts["root_dir"], "cache", "tokens"),
os.path.join(sub_master_opts["pki_dir"], "accepted"),
os.path.join(sub_master_opts["pki_dir"], "rejected"),
os.path.join(sub_master_opts["pki_dir"], "pending"),
os.path.join(sub_master_opts["cachedir"], "raet"),
os.path.join(minion_opts["pki_dir"], "accepted"),
os.path.join(minion_opts["pki_dir"], "rejected"),
os.path.join(minion_opts["pki_dir"], "pending"),
os.path.join(minion_opts["cachedir"], "raet"),
os.path.join(sub_minion_opts["pki_dir"], "accepted"),
os.path.join(sub_minion_opts["pki_dir"], "rejected"),
os.path.join(sub_minion_opts["pki_dir"], "pending"),
os.path.join(sub_minion_opts["cachedir"], "raet"),
os.path.dirname(master_opts["log_file"]),
minion_opts["extension_modules"],
sub_minion_opts["extension_modules"],
sub_minion_opts["pki_dir"],
master_opts["sock_dir"],
sub_master_opts["sock_dir"],
sub_minion_opts["sock_dir"],
minion_opts["sock_dir"],
],
RUNTIME_VARS.RUNNING_TESTS_USER,
root_dir=master_opts["root_dir"],
)
cls.mm_master_opts = master_opts
cls.mm_sub_master_opts = sub_master_opts
cls.mm_minion_opts = minion_opts
cls.mm_sub_minion_opts = sub_minion_opts
def build_minion_opts(
minion_id=None,
root_dir=None,
initial_conf_file=None,
minion_opts_overrides=None,
skip_cached_opts=False,
cache_opts=True,
minion_role=None,
):
if minion_id is None:
minion_id = "pytest-internal-sminion"
if skip_cached_opts is False:
try:
opts_cache = build_minion_opts.__cached_opts__
except AttributeError:
opts_cache = build_minion_opts.__cached_opts__ = {}
cached_opts = opts_cache.get(minion_id)
if cached_opts:
return cached_opts
log.info("Generating testing minion %r configuration...", minion_id)
if root_dir is None:
hashed_minion_id = hashlib.sha1()
hashed_minion_id.update(salt.utils.stringutils.to_bytes(minion_id))
root_dir = os.path.join(RUNTIME_VARS.TMP_ROOT_DIR,
hashed_minion_id.hexdigest()[:6])
if initial_conf_file is not None:
minion_opts = salt.config._read_conf_file(initial_conf_file) # pylint: disable=protected-access
else:
minion_opts = {}
conf_dir = os.path.join(root_dir, "conf")
conf_file = os.path.join(conf_dir, "minion")
minion_opts["id"] = minion_id
minion_opts["conf_file"] = conf_file
minion_opts["root_dir"] = root_dir
minion_opts["cachedir"] = "cache"
minion_opts["user"] = RUNTIME_VARS.RUNNING_TESTS_USER
minion_opts["pki_dir"] = "pki"
minion_opts["hosts.file"] = os.path.join(RUNTIME_VARS.TMP_ROOT_DIR,
"hosts")
minion_opts["aliases.file"] = os.path.join(RUNTIME_VARS.TMP_ROOT_DIR,
"aliases")
minion_opts["file_client"] = "local"
minion_opts["server_id_use_crc"] = "adler32"
minion_opts["pillar_roots"] = {"base": [RUNTIME_VARS.TMP_PILLAR_TREE]}
minion_opts["file_roots"] = {
"base": [
# Let's support runtime created files that can be used like:
# salt://my-temp-file.txt
RUNTIME_VARS.TMP_STATE_TREE
],
# Alternate root to test __env__ choices
"prod": [
os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
],
}
if initial_conf_file and initial_conf_file.startswith(RUNTIME_VARS.FILES):
# We assume we were passed a minion configuration file defined fo testing and, as such
# we define the file and pillar roots to include the testing states/pillar trees
minion_opts["pillar_roots"]["base"].append(
os.path.join(RUNTIME_VARS.FILES, "pillar", "base"), )
minion_opts["file_roots"]["base"].append(
os.path.join(RUNTIME_VARS.FILES, "file", "base"), )
minion_opts["file_roots"]["prod"].append(
os.path.join(RUNTIME_VARS.FILES, "file", "prod"), )
# We need to copy the extension modules into the new master root_dir or
# it will be prefixed by it
extension_modules_path = os.path.join(root_dir, "extension_modules")
if not os.path.exists(extension_modules_path):
shutil.copytree(
os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
extension_modules_path,
)
minion_opts["extension_modules"] = extension_modules_path
# Custom grains
if "grains" not in minion_opts:
minion_opts["grains"] = {}
if minion_role is not None:
minion_opts["grains"]["role"] = minion_role
# Under windows we can't seem to properly create a virtualenv off of another
# virtualenv, we can on linux but we will still point to the virtualenv binary
# outside the virtualenv running the test suite, if that's the case.
try:
real_prefix = sys.real_prefix
# The above attribute exists, this is a virtualenv
if salt.utils.platform.is_windows():
virtualenv_binary = os.path.join(real_prefix, "Scripts",
"virtualenv.exe")
else:
# We need to remove the virtualenv from PATH or we'll get the virtualenv binary
# from within the virtualenv, we don't want that
path = os.environ.get("PATH")
if path is not None:
path_items = path.split(os.pathsep)
for item in path_items[:]:
if item.startswith(sys.base_prefix):
path_items.remove(item)
os.environ["PATH"] = os.pathsep.join(path_items)
virtualenv_binary = salt.utils.path.which("virtualenv")
if path is not None:
# Restore previous environ PATH
os.environ["PATH"] = path
if not virtualenv_binary.startswith(real_prefix):
virtualenv_binary = None
if virtualenv_binary and not os.path.exists(virtualenv_binary):
# It doesn't exist?!
virtualenv_binary = None
except AttributeError:
# We're not running inside a virtualenv
virtualenv_binary = None
if virtualenv_binary:
minion_opts["venv_bin"] = virtualenv_binary
# Override minion_opts with minion_opts_overrides
if minion_opts_overrides:
minion_opts.update(minion_opts_overrides)
if not os.path.exists(conf_dir):
os.makedirs(conf_dir)
with salt.utils.files.fopen(conf_file, "w") as fp_:
salt.utils.yaml.safe_dump(minion_opts, fp_, default_flow_style=False)
log.info("Generating testing minion %r configuration completed.",
minion_id)
minion_opts = salt.config.minion_config(conf_file,
minion_id=minion_id,
cache_minion_id=True)
salt.utils.verify.verify_env(
[
os.path.join(minion_opts["pki_dir"], "accepted"),
os.path.join(minion_opts["pki_dir"], "rejected"),
os.path.join(minion_opts["pki_dir"], "pending"),
os.path.dirname(minion_opts["log_file"]),
minion_opts["extension_modules"],
minion_opts["cachedir"],
minion_opts["sock_dir"],
RUNTIME_VARS.TMP_STATE_TREE,
RUNTIME_VARS.TMP_PILLAR_TREE,
RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
RUNTIME_VARS.TMP,
],
RUNTIME_VARS.RUNNING_TESTS_USER,
root_dir=root_dir,
)
if cache_opts:
try:
opts_cache = build_minion_opts.__cached_opts__
except AttributeError:
opts_cache = build_minion_opts.__cached_opts__ = {}
opts_cache[minion_id] = minion_opts
return minion_opts
def _enter_mockbin(self):
path = os.environ.get('PATH', '')
path_items = path.split(os.pathsep)
if MOCKBIN not in path_items:
path_items.insert(0, MOCKBIN)
os.environ['PATH'] = os.pathsep.join(path_items)
