def _get_serv(ret=None, commit=False):
'''
Return a Pg cursor
'''
_options = _get_options(ret)
try:
conn = psycopg2.connect(host=_options.get('host'),
user=_options.get('user'),
password=_options.get('passwd'),
database=_options.get('db'),
port=_options.get('port'))
except psycopg2.OperationalError as exc:
raise salt.exceptions.SaltMasterError(
'postgres returner could not connect to database: {exc}'.format(
exc=exc))
cursor = conn.cursor()
try:
yield cursor
except psycopg2.DatabaseError as err:
error = err.args
sys.stderr.write(six.text_type(error))
cursor.execute("ROLLBACK")
six.reraise(*sys.exc_info())
else:
if commit:
cursor.execute("COMMIT")
else:
cursor.execute("ROLLBACK")
finally:
conn.close()
def run(self):
'''
Start the ret port binding
'''
self.context = zmq.Context(self.opts['worker_threads'])
self.uri = 'tcp://{interface}:{ret_port}'.format(**self.opts)
log.info('ZMQ Ret port binding to %s', self.uri)
self.clients = self.context.socket(zmq.ROUTER)
if self.opts['ipv6'] is True and hasattr(zmq, 'IPV4ONLY'):
# IPv6 sockets work for both IPv6 and IPv4 addresses
self.clients.setsockopt(zmq.IPV4ONLY, 0)
try:
self.clients.setsockopt(zmq.HWM, self.opts['rep_hwm'])
except AttributeError:
self.clients.setsockopt(zmq.SNDHWM, self.opts['rep_hwm'])
self.clients.setsockopt(zmq.RCVHWM, self.opts['rep_hwm'])
self.clients.setsockopt(zmq.BACKLOG, self.opts['zmq_backlog'])
self.workers = self.context.socket(zmq.DEALER)
self.w_uri = 'ipc://{0}'.format(
os.path.join(self.opts['sock_dir'], 'workers.ipc'))
log.info('Setting up the master communication server')
self.clients.bind(self.uri)
self.workers.bind(self.w_uri)
while True:
try:
zmq.device(zmq.QUEUE, self.clients, self.workers)
except zmq.ZMQError as exc:
if exc.errno == errno.EINTR:
continue
six.reraise(*sys.exc_info())
def _get_serv(ret=None, commit=False):
"""
Return a mysql cursor
"""
_options = _get_options(ret)
connect = True
if __context__ and "mysql_returner_conn" in __context__:
try:
log.debug("Trying to reuse MySQL connection pool")
conn = __context__["mysql_returner_conn"]
conn.ping()
connect = False
except OperationalError as exc:
log.debug("OperationalError on ping: %s", exc)
if connect:
log.debug("Generating new MySQL connection pool")
try:
# An empty ssl_options dictionary passed to MySQLdb.connect will
# effectively connect w/o SSL.
ssl_options = {}
if _options.get("ssl_ca"):
ssl_options["ca"] = _options.get("ssl_ca")
if _options.get("ssl_cert"):
ssl_options["cert"] = _options.get("ssl_cert")
if _options.get("ssl_key"):
ssl_options["key"] = _options.get("ssl_key")
conn = MySQLdb.connect(
host=_options.get("host"),
user=_options.get("user"),
passwd=_options.get("pass"),
db=_options.get("db"),
port=_options.get("port"),
ssl=ssl_options,
)
try:
__context__["mysql_returner_conn"] = conn
except TypeError:
pass
except OperationalError as exc:
raise salt.exceptions.SaltMasterError(
"MySQL returner could not connect to database: {exc}".format(
exc=exc))
cursor = conn.cursor()
try:
yield cursor
except MySQLdb.DatabaseError as err:
error = err.args
sys.stderr.write(six.text_type(error))
cursor.execute("ROLLBACK")
six.reraise(*sys.exc_info())
else:
if commit:
cursor.execute("COMMIT")
else:
cursor.execute("ROLLBACK")
def _call_function(self, kwargs):
'''
Call target function that has been decorated.
:return:
'''
if self._raise_later:
raise self._raise_later # pylint: disable=E0702
if self._function:
args, kwargs = self._get_args(kwargs)
try:
return self._function(*args, **kwargs)
except TypeError as error:
error = six.text_type(error).replace(
self._function, self._orig_f_name) # Hide hidden functions
log.error('Function "%s" was not properly called: %s',
self._orig_f_name, error)
return self._function.__doc__
except Exception as error: # pylint: disable=broad-except
log.error('Unhandled exception occurred in function "%s: %s',
self._function.__name__, error)
six.reraise(*sys.exc_info())
else:
raise CommandExecutionError(
"Function is deprecated, but the successor function was not found."
)
def _convert_exception(e):
"""Convert an ldap backend exception to an LDAPError and raise it."""
args = ("exception in ldap backend: {0}".format(repr(e)), e)
if six.PY2:
six.reraise(LDAPError, args, sys.exc_info()[2])
else:
six.raise_from(LDAPError(*args), e)
def _convert_exception(e):
"""Convert an ldap backend exception to an LDAPError and raise it."""
args = ("exception in ldap backend: {0}".format(repr(e)), e)
if six.PY2:
six.reraise(LDAPError, args, sys.exc_info()[2])
else:
six.raise_from(LDAPError(*args), e)
def fire_master_started_event(self):
master_start_event_tag = 'salt/master/{}/start'.format(self.id)
log.info('Firing salt-%s started event. Tag: %s', self.role,
master_start_event_tag)
event_bus = salt.utils.event.get_master_event(self.opts,
self.sock_dir,
listen=False)
load = {'id': self.id, 'tag': master_start_event_tag, 'data': {}}
# 30 seconds should be more than enough to fire these events every second in order
# for pytest-salt to pickup that the master is running
timeout = 30
event_bus = None
call_destroy = False
try:
with salt.utils.event.get_master_event(self.opts,
self.sock_dir,
listen=False) as event_bus:
yield self._fire_master_started_event(event_bus, load,
master_start_event_tag,
timeout)
except AttributeError as exc:
if '__enter__' not in str(exc):
six.reraise(*sys.exc_info())
call_destroy = True
event_bus = salt.utils.event.get_master_event(self.opts,
self.sock_dir,
listen=False)
yield self._fire_master_started_event(event_bus, load,
master_start_event_tag,
timeout)
finally:
if call_destroy and event_bus is not None:
event_bus.destroy()
def _conn(commit=False):
'''
Return an postgres cursor
'''
defaults = {'host': 'localhost',
'user': '******',
'password': '******',
'dbname': 'salt',
'port': 5432}
conn_kwargs = {}
for key, value in defaults.items():
conn_kwargs[key] = __opts__.get('queue.{0}.{1}'.format(__virtualname__, key), value)
try:
conn = psycopg2.connect(**conn_kwargs)
except psycopg2.OperationalError as exc:
raise SaltMasterError('pgjsonb returner could not connect to database: {exc}'.format(exc=exc))
cursor = conn.cursor()
try:
yield cursor
except psycopg2.DatabaseError as err:
error = err.args
sys.stderr.write(six.text_type(error))
cursor.execute("ROLLBACK")
six.reraise(*sys.exc_info())
else:
if commit:
cursor.execute("COMMIT")
else:
cursor.execute("ROLLBACK")
finally:
conn.close()
def _get_serv(ret=None, commit=False):
'''
Return a mysql cursor
'''
_options = _get_options(ret)
connect = True
if __context__ and 'mysql_returner_conn' in __context__:
try:
log.debug('Trying to reuse MySQL connection pool')
conn = __context__['mysql_returner_conn']
conn.ping()
connect = False
except OperationalError as exc:
log.debug('OperationalError on ping: %s', exc)
if connect:
log.debug('Generating new MySQL connection pool')
try:
# An empty ssl_options dictionary passed to MySQLdb.connect will
# effectively connect w/o SSL.
ssl_options = {}
if _options.get('ssl_ca'):
ssl_options['ca'] = _options.get('ssl_ca')
if _options.get('ssl_cert'):
ssl_options['cert'] = _options.get('ssl_cert')
if _options.get('ssl_key'):
ssl_options['key'] = _options.get('ssl_key')
conn = MySQLdb.connect(host=_options.get('host'),
user=_options.get('user'),
passwd=_options.get('pass'),
db=_options.get('db'),
port=_options.get('port'),
ssl=ssl_options,
unix_socket=_options.get('unix_socket'))
try:
__context__['mysql_returner_conn'] = conn
except TypeError:
pass
except OperationalError as exc:
raise salt.exceptions.SaltMasterError(
'MySQL returner could not connect to database: {exc}'.format(
exc=exc))
cursor = conn.cursor()
try:
yield cursor
except MySQLdb.DatabaseError as err:
error = err.args
sys.stderr.write(six.text_type(error))
cursor.execute("ROLLBACK")
six.reraise(*sys.exc_info())
else:
if commit:
cursor.execute("COMMIT")
else:
cursor.execute("ROLLBACK")
def wrap(cls):
# Let's add the user to the system.
log.debug('Creating system user {0!r}'.format(username))
create_user = cls.run_function('user.add', [username])
if not create_user:
log.debug('Failed to create system user')
# The user was not created
if on_existing == 'skip':
cls.skipTest(
'Failed to create system user {0!r}'.format(username))
if on_existing == 'delete':
log.debug(
'Deleting the system user {0!r}'.format(username))
delete_user = cls.run_function('user.delete',
[username, True, True])
if not delete_user:
cls.skipTest(
'A user named {0!r} already existed on the '
'system and re-creating it was not possible'.
format(username))
log.debug('Second time creating system user {0!r}'.format(
username))
create_user = cls.run_function('user.add', [username])
if not create_user:
cls.skipTest(
'A user named {0!r} already existed, was deleted '
'as requested, but re-creating it was not possible'
.format(username))
failure = None
try:
try:
return func(cls, username)
except Exception as exc: # pylint: disable=W0703
log.error('Running {0!r} raised an exception: {1}'.format(
func, exc),
exc_info=True)
# Store the original exception details which will be raised
# a little further down the code
failure = sys.exc_info()
finally:
if delete:
delete_user = cls.run_function('user.delete',
[username, True, True])
if not delete_user:
if failure is None:
log.warning(
'Although the actual test-case did not fail, '
'deleting the created system user {0!r} '
'afterwards did.'.format(username))
else:
log.warning(
'The test-case failed and also did the removal'
' of the system user {0!r}'.format(username))
if failure is not None:
# If an exception was thrown, raise it
six.reraise(failure[0], failure[1], failure[2])
def _mkstemp():
fd, ret = tempfile.mkstemp()
try:
os.close(fd)
except OSError as exc:
if exc.errno != errno.EBADF:
six.reraise(*sys.exc_info())
else:
self.addCleanup(os.remove, ret)
return ret
def wrap(*args, **kwargs):
results = []
thread = threading.Thread(
target=self._target,
args=(key, args, kwargs, results, self.io_loop),
)
thread.start()
thread.join()
if results[0]:
return results[1]
else:
six.reraise(*results[1])
def _to_dict(objects):
'''
Potentially interprets a string as JSON for usage with mongo
'''
try:
if isinstance(objects, six.string_types):
objects = salt.utils.json.loads(objects)
except ValueError as err:
log.error("Could not parse objects: %s", err)
six.reraise(*sys.exc_info())
return objects
def export_distributions(region=None, key=None, keyid=None, profile=None):
'''
Get details of all CloudFront distributions.
Produces results that can be used to create an SLS file.
CLI Example:
.. code-block:: bash
salt-call boto_cloudfront.export_distributions --out=txt |\
sed "s/local: //" > cloudfront_distributions.sls
'''
results = OrderedDict()
conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)
try:
for name, distribution in _list_distributions(
conn,
region=region,
key=key,
keyid=keyid,
profile=profile,
):
config = distribution['distribution']['DistributionConfig']
tags = distribution['tags']
distribution_sls_data = [
{
'name': name
},
{
'config': config
},
{
'tags': tags
},
]
results['Manage CloudFront distribution {0}'.format(name)] = {
'boto_cloudfront.present': distribution_sls_data,
}
except botocore.exceptions.ClientError as err:
# Raise an exception, as this is meant to be user-invoked at the CLI
# as opposed to being called from execution or state modules
six.reraise(*sys.exc_info())
dumper = __utils__['yaml.get_dumper']('IndentedSafeOrderedDumper')
return __utils__['yaml.dump'](
results,
default_flow_style=False,
Dumper=dumper,
)
def _load(self, full_path):
try:
fd = open(full_path, 'r')
except (IOError):
six.reraise(spacewalk_unknown_report, None, sys.exc_info()[2])
tag = None
value = ''
re_comment = re.compile('^\s*#')
re_tag_name = re.compile('^(\S+):\s*$')
for line in fd:
result = re_comment.match(line)
if result != None:
continue
result = re_tag_name.match(line)
if result != None:
if tag != None:
self._set(tag, value)
tag = None
value = ''
tag = result.group(1)
else:
value += line
if tag != None:
self._set(tag, value)
if self.multival_column_names != None:
unknown_columns = []
for c in self.multival_column_names:
if c in self.column_indexes:
c_id = self.column_indexes[c]
v = self.multival_column_names[c]
if v == None:
self.multival_columns_stop.append(c_id)
elif v in self.column_indexes:
v_id = self.column_indexes[v]
if v_id in self.multival_columns_reverted:
self.multival_columns_reverted[v_id].append(c_id)
else:
self.multival_columns_reverted[v_id] = [c_id]
else:
unknown_columns.append(c)
if len(unknown_columns) > 0:
raise spacewalk_report_unknown_multival_column_exception(
unknown_columns)
def _get_serv(ret=None, commit=False):
'''
Return a Pg cursor
'''
_options = _get_options(ret)
try:
# An empty ssl_options dictionary passed to MySQLdb.connect will
# effectively connect w/o SSL.
ssl_options = {
k: v
for k, v in six.iteritems(_options)
if k in ['sslmode', 'sslcert', 'sslkey', 'sslrootcert', 'sslcrl']
}
conn = psycopg2.connect(host=_options.get('host'),
port=_options.get('port'),
dbname=_options.get('db'),
user=_options.get('user'),
password=_options.get('pass'),
**ssl_options)
except psycopg2.OperationalError as exc:
raise salt.exceptions.SaltMasterError(
'pgjsonb returner could not connect to database: {exc}'.format(
exc=exc))
if conn.server_version is not None and conn.server_version >= 90500:
global PG_SAVE_LOAD_SQL
PG_SAVE_LOAD_SQL = '''INSERT INTO jids
(jid, load)
VALUES (%(jid)s, %(load)s)
ON CONFLICT (jid) DO UPDATE
SET load=%(load)s'''
cursor = conn.cursor()
try:
yield cursor
except psycopg2.DatabaseError as err:
error = err.args
sys.stderr.write(six.text_type(error))
cursor.execute("ROLLBACK")
six.reraise(*sys.exc_info())
else:
if commit:
cursor.execute("COMMIT")
else:
cursor.execute("ROLLBACK")
finally:
conn.close()
def _wait_for_sync(change, conn, tries=10, sleep=20):
for retry in range(1, tries+1):
log.info('Getting route53 status (attempt %s)', retry)
status = 'wait'
try:
status = conn.get_change(Id=change)['ChangeInfo']['Status']
except ClientError as e:
if e.response.get('Error', {}).get('Code') == 'Throttling':
log.debug('Throttled by AWS API.')
else:
six.reraise(*sys.exc_info())
if status == 'INSYNC':
return True
time.sleep(sleep)
log.error('Timed out waiting for Route53 INSYNC status.')
return False
def _wait_for_sync(change, conn, tries=10, sleep=20):
for retry in range(1, tries + 1):
log.info("Getting route53 status (attempt %s)", retry)
status = "wait"
try:
status = conn.get_change(Id=change)["ChangeInfo"]["Status"]
except ClientError as e:
if e.response.get("Error", {}).get("Code") == "Throttling":
log.debug("Throttled by AWS API.")
else:
six.reraise(*sys.exc_info())
if status == "INSYNC":
return True
time.sleep(sleep)
log.error("Timed out waiting for Route53 INSYNC status.")
return False
def display_output(data, out=None, opts=None, **kwargs):
"""
Print the passed data using the desired output
"""
if opts is None:
opts = {}
display_data = try_printout(data, out, opts, **kwargs)
output_filename = opts.get("output_file", None)
log.trace("data = %s", data)
try:
# output filename can be either '' or None
if output_filename:
if not hasattr(output_filename, "write"):
# pylint: disable=resource-leakage
ofh = salt.utils.files.fopen(output_filename, "a")
# pylint: enable=resource-leakage
fh_opened = True
else:
# Filehandle/file-like object
ofh = output_filename
fh_opened = False
try:
fdata = display_data
if isinstance(fdata, six.text_type):
try:
fdata = fdata.encode("utf-8")
except (UnicodeDecodeError, UnicodeEncodeError):
# try to let the stream write
# even if we didn't encode it
pass
if fdata:
ofh.write(salt.utils.stringutils.to_str(fdata))
ofh.write("\n")
finally:
if fh_opened:
ofh.close()
return
if display_data:
salt.utils.stringutils.print_cli(display_data)
except IOError as exc:
# Only raise if it's NOT a broken pipe
if exc.errno != errno.EPIPE:
six.reraise(*sys.exc_info())
def loads(s, **kwargs):
"""
.. versionadded:: 2018.3.0
Wraps json.loads and prevents a traceback in the event that a bytestring is
passed to the function. (Python < 3.6 cannot load bytestrings)
You can pass an alternate json module (loaded via import_json() above)
using the _json_module argument)
"""
json_module = kwargs.pop("_json_module", json)
try:
return json_module.loads(s, **kwargs)
except TypeError as exc:
# json.loads cannot load bytestrings in Python < 3.6
if six.PY3 and isinstance(s, bytes):
return json_module.loads(salt.utils.stringutils.to_unicode(s), **kwargs)
else:
six.reraise(*sys.exc_info())
def __getattribute__(self, key):
try:
return object.__getattribute__(self, key)
except AttributeError:
if key == "asynchronous":
six.reraise(*sys.exc_info())
attr = getattr(self.asynchronous, key)
if hasattr(attr, "__call__"):
def wrap(*args, **kwargs):
# Overload the ioloop for the func call-- since it might call .current()
with current_ioloop(self.io_loop):
ret = attr(*args, **kwargs)
if isinstance(ret, salt.ext.tornado.concurrent.Future):
ret = self._block_future(ret)
return ret
return wrap
return attr
def _parallel_map(func, inputs):
'''
Applies a function to each element of a list, returning the resulting list.
A separate thread is created for each element in the input list and the
passed function is called for each of the elements. When all threads have
finished execution a list with the results corresponding to the inputs is
returned.
If one of the threads fails (because the function throws an exception),
that exception is reraised. If more than one thread fails, the exception
from the first thread (according to the index of the input element) is
reraised.
func:
function that is applied on each input element.
inputs:
list of elements that shall be processed. The length of this list also
defines the number of threads created.
'''
outputs = len(inputs) * [None]
errors = len(inputs) * [None]
def create_thread(index):
def run_thread():
try:
outputs[index] = func(inputs[index])
except: # pylint: disable=bare-except
errors[index] = sys.exc_info()
thread = threading.Thread(target=run_thread)
thread.start()
return thread
threads = list(six.moves.map(create_thread, six.moves.range(len(inputs))))
for thread in threads:
thread.join()
for error in errors:
if error is not None:
exc_type, exc_value, exc_traceback = error
six.reraise(exc_type, exc_value, exc_traceback)
return outputs
def close(self):
"""
Routines to handle any cleanup before the instance shuts down.
Sockets and filehandles should be closed explicitly, to prevent
leaks.
"""
if self._closing:
return
self._closing = True
log.debug("Closing %s instance", self.__class__.__name__)
if self.stream is not None and not self.stream.closed():
try:
self.stream.close()
except socket.error as exc:
if exc.errno != errno.EBADF:
# If its not a bad file descriptor error, raise
six.reraise(*sys.exc_info())
def wrapped_run_func():
for method, args, kwargs in self._after_fork_methods:
method(*args, **kwargs)
try:
return run_func()
except SystemExit:
# These are handled by multiprocessing.Process._bootstrap()
six.reraise(*sys.exc_info())
except Exception as exc: # pylint: disable=broad-except
log.error(
'An un-handled exception from the multiprocessing process '
'\'%s\' was caught:\n',
self.name,
exc_info=True)
# Re-raise the exception. multiprocessing.Process will write it to
# sys.stderr and set the proper exitcode and we have already logged
# it above.
six.reraise(*sys.exc_info())
finally:
for method, args, kwargs in self._finalize_methods:
method(*args, **kwargs)
def ping(allow_failure=False, hosts=None, profile=None):
'''
.. versionadded:: 2017.7.0
Test connection to Elasticsearch instance. This method does not fail if not explicitly specified.
allow_failure
Throw exception if ping fails
CLI example::
salt myminion elasticsearch.ping allow_failure=True
salt myminion elasticsearch.ping profile=elasticsearch-extra
'''
try:
_get_instance(hosts, profile)
except CommandExecutionError as e:
if allow_failure:
six.reraise(*sys.exc_info())
return False
return True
def fetch_plain(self):
'''
Handle unicode literal strings which appear inline in the YAML
'''
orig_line = self.line
orig_column = self.column
orig_pointer = self.pointer
try:
return super(SaltYamlSafeLoader, self).fetch_plain()
except yaml.scanner.ScannerError as exc:
problem_line = self.line
problem_column = self.column
problem_pointer = self.pointer
if exc.problem == "found unexpected ':'":
# Reset to prior position
self.line = orig_line
self.column = orig_column
self.pointer = orig_pointer
if self.peek(0) == 'u':
# Might be a unicode literal string, check for 2nd char and
# call the appropriate fetch func if it's a quote
quote_char = self.peek(1)
if quote_char in ("'", '"'):
# Skip the "u" prefix by advancing the column and
# pointer by 1
self.column += 1
self.pointer += 1
if quote_char == '\'':
return self.fetch_single()
else:
return self.fetch_double()
else:
# This wasn't a unicode literal string, so the caught
# exception was correct. Restore the old position and
# then raise the caught exception.
self.line = problem_line
self.column = problem_column
self.pointer = problem_pointer
# Raise the caught exception
six.reraise(*sys.exc_info())
def _get_serv(ret=None, commit=False):
'''
Return a Pg cursor
'''
_options = _get_options(ret)
try:
# An empty ssl_options dictionary passed to MySQLdb.connect will
# effectively connect w/o SSL.
ssl_options = {
k: v for k, v in six.iteritems(_options)
if k in ['sslmode', 'sslcert', 'sslkey', 'sslrootcert', 'sslcrl']
}
conn = psycopg2.connect(
host=_options.get('host'),
port=_options.get('port'),
dbname=_options.get('db'),
user=_options.get('user'),
password=_options.get('pass'),
**ssl_options
)
except psycopg2.OperationalError as exc:
raise salt.exceptions.SaltMasterError('pgjsonb returner could not connect to database: {exc}'.format(exc=exc))
cursor = conn.cursor()
try:
yield cursor
except psycopg2.DatabaseError as err:
error = err.args
sys.stderr.write(six.text_type(error))
cursor.execute("ROLLBACK")
six.reraise(*sys.exc_info())
else:
if commit:
cursor.execute("COMMIT")
else:
cursor.execute("ROLLBACK")
finally:
conn.close()
def setUp(self):
self.state_name = "run_redirect"
state_filename = self.state_name + ".sls"
self.state_file = os.path.join(RUNTIME_VARS.TMP_STATE_TREE, state_filename)
# Create the testfile and release the handle
fd, self.test_file = tempfile.mkstemp()
try:
os.close(fd)
except OSError as exc:
if exc.errno != errno.EBADF:
six.reraise(*sys.exc_info())
# Create the testfile and release the handle
fd, self.test_tmp_path = tempfile.mkstemp()
try:
os.close(fd)
except OSError as exc:
if exc.errno != errno.EBADF:
six.reraise(*sys.exc_info())
super(CMDRunRedirectTest, self).setUp()
def setUpClass(cls): # pylint: disable=arguments-differ
super(SSHDMixin, cls).setUpClass()
try:
log.info("%s: prep_server()", cls.__name__)
cls.sshd_bin = salt.utils.path.which("sshd")
cls.sshd_config_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
cls.sshd_config = os.path.join(cls.sshd_config_dir, "sshd_config")
cls.sshd_port = get_unused_localhost_port()
cls.url = "ssh://{username}@127.0.0.1:{port}/~/repo.git".format(
username=cls.username, port=cls.sshd_port
)
cls.url_extra_repo = "ssh://{username}@127.0.0.1:{port}/~/extra_repo.git".format(
username=cls.username, port=cls.sshd_port
)
home = "/root/.ssh"
cls.ext_opts = {
"url": cls.url,
"url_extra_repo": cls.url_extra_repo,
"privkey_nopass": os.path.join(home, cls.id_rsa_nopass),
"pubkey_nopass": os.path.join(home, cls.id_rsa_nopass + ".pub"),
"privkey_withpass": os.path.join(home, cls.id_rsa_withpass),
"pubkey_withpass": os.path.join(home, cls.id_rsa_withpass + ".pub"),
"passphrase": cls.passphrase,
}
if cls.prep_states_ran is False:
ret = cls.cls_run_function(
"state.apply",
mods="git_pillar.ssh",
pillar={
"git_pillar": {
"git_ssh": cls.git_ssh,
"id_rsa_nopass": cls.id_rsa_nopass,
"id_rsa_withpass": cls.id_rsa_withpass,
"sshd_bin": cls.sshd_bin,
"sshd_port": cls.sshd_port,
"sshd_config_dir": cls.sshd_config_dir,
"master_user": cls.user,
"user": cls.username,
}
},
)
assert next(six.itervalues(ret))["result"] is True
cls.prep_states_ran = True
log.info("%s: States applied", cls.__name__)
if cls.sshd_proc is not None:
if not psutil.pid_exists(cls.sshd_proc.pid):
log.info(
"%s: sshd started but appears to be dead now. Will try to restart it.",
cls.__name__,
)
cls.sshd_proc = None
if cls.sshd_proc is None:
cls.sshd_proc = start_daemon(
cls.sshd_bin, cls.sshd_config_dir, cls.sshd_port, SshdDaemon
)
log.info("%s: sshd started", cls.__name__)
except AssertionError:
cls.tearDownClass()
six.reraise(*sys.exc_info())
if cls.known_hosts_setup is False:
known_hosts_ret = cls.cls_run_function(
"ssh.set_known_host",
user=cls.user,
hostname="127.0.0.1",
port=cls.sshd_port,
enc="ssh-rsa",
fingerprint="fd:6f:7f:5d:06:6b:f2:06:0d:26:93:9e:5a:b5:19:46",
hash_known_hosts=False,
fingerprint_hash_type="md5",
)
if "error" in known_hosts_ret:
cls.tearDownClass()
raise AssertionError(
"Failed to add key to {0} user's known_hosts "
"file: {1}".format(
cls.master_opts["user"], known_hosts_ret["error"]
)
)
cls.known_hosts_setup = True
def render_tmpl(tmplsrc,
from_str=False,
to_str=False,
context=None,
tmplpath=None,
**kws):
if context is None:
context = {}
# Alias cmd.run to cmd.shell to make python_shell=True the default for
# templated calls
if 'salt' in kws:
kws['salt'] = AliasedLoader(kws['salt'])
# We want explicit context to overwrite the **kws
kws.update(context)
context = kws
assert 'opts' in context
assert 'saltenv' in context
if 'sls' in context:
slspath = context['sls'].replace('.', '/')
if tmplpath is not None:
context['tplpath'] = tmplpath
if not tmplpath.lower().replace('\\',
'/').endswith('/init.sls'):
slspath = os.path.dirname(slspath)
template = tmplpath.replace('\\', '/')
i = template.rfind(slspath.replace('.', '/'))
if i != -1:
template = template[i:]
tpldir = os.path.dirname(template).replace('\\', '/')
tpldata = {
'tplfile': template,
'tpldir': '.' if tpldir == '' else tpldir,
'tpldot': tpldir.replace('/', '.'),
'tplroot': tpldir.split('/')[0],
}
context.update(tpldata)
context['slsdotpath'] = slspath.replace('/', '.')
context['slscolonpath'] = slspath.replace('/', ':')
context['sls_path'] = slspath.replace('/', '_')
context['slspath'] = slspath
if isinstance(tmplsrc, six.string_types):
if from_str:
tmplstr = tmplsrc
else:
try:
if tmplpath is not None:
tmplsrc = os.path.join(tmplpath, tmplsrc)
with codecs.open(tmplsrc, 'r', SLS_ENCODING) as _tmplsrc:
tmplstr = _tmplsrc.read()
except (UnicodeDecodeError, ValueError, OSError,
IOError) as exc:
if salt.utils.files.is_binary(tmplsrc):
# Template is a bin file, return the raw file
return dict(result=True, data=tmplsrc)
log.error('Exception occurred while reading file %s: %s',
tmplsrc,
exc,
exc_info_on_loglevel=logging.DEBUG)
six.reraise(*sys.exc_info())
else: # assume tmplsrc is file-like.
tmplstr = tmplsrc.read()
tmplsrc.close()
try:
output = render_str(tmplstr, context, tmplpath)
if salt.utils.platform.is_windows():
newline = False
if salt.utils.stringutils.to_unicode(
output, encoding=SLS_ENCODING).endswith(
('\n', os.linesep)):
newline = True
# Write out with Windows newlines
output = os.linesep.join(output.splitlines())
if newline:
output += os.linesep
except SaltRenderError as exc:
log.exception('Rendering exception occurred')
#return dict(result=False, data=six.text_type(exc))
raise
except Exception:
return dict(result=False, data=traceback.format_exc())
else:
if to_str: # then render as string
return dict(result=True, data=output)
with tempfile.NamedTemporaryFile(
'wb', delete=False,
prefix=salt.utils.files.TEMPFILE_PREFIX) as outf:
outf.write(
salt.utils.stringutils.to_bytes(output,
encoding=SLS_ENCODING))
# Note: If nothing is replaced or added by the rendering
# function, then the contents of the output file will
# be exactly the same as the input.
return dict(result=True, data=outf.name)
try:
from tests.support.paths import TMP, SYS_TMP_DIR, INTEGRATION_TEST_DIR
from tests.support.paths import CODE_DIR as SALT_ROOT
except ImportError as exc:
try:
import tests
print("Found tests module not from salt in {}".format(tests.__file__))
except ImportError:
print("Unable to import salt test module")
print("PYTHONPATH:", os.environ.get("PYTHONPATH"))
print("Current sys.path:")
import pprint
pprint.pprint(sys.path)
six.reraise(*sys.exc_info())
from tests.integration import TestDaemon, TestDaemonStartFailed # isort:skip
from tests.multimaster import MultimasterTestDaemon # isort:skip
import salt.utils.platform # isort:skip
if not salt.utils.platform.is_windows():
import resource
# Import Salt Testing libs
from tests.support.parser import PNUM, print_header # isort:skip
from tests.support.parser.cover import SaltCoverageTestingParser # isort:skip
XML_OUTPUT_DIR = os.environ.get(
"SALT_XML_TEST_REPORTS_DIR", os.path.join(TMP, "xml-test-reports")
)
