def test_run_command_timeout(self):
# Make sure stdout and stderr is included on timeout
hosts = ["localhost", "127.0.0.1", "st2build001"]
client = ParallelSSHClient(hosts=hosts,
user="******",
pkey_file="~/.ssh/id_rsa",
connect=True)
mock_run = Mock(
side_effect=SSHCommandTimeoutError(cmd="pwd",
timeout=10,
stdout="a",
stderr="b",
ssh_connect_timeout=30))
for host in hosts:
hostname, _ = client._get_host_port_info(host)
host_client = client._hosts_client[host]
host_client.run = mock_run
results = client.run("pwd")
for host in hosts:
result = results[host]
self.assertEqual(result["failed"], True)
self.assertEqual(result["stdout"], "a")
self.assertEqual(result["stderr"], "b")
self.assertEqual(result["return_code"], -9)
def test_run_command_timeout(self):
# Make sure stdout and stderr is included on timeout
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
mock_run = Mock(
side_effect=SSHCommandTimeoutError(cmd='pwd',
timeout=10,
stdout='a',
stderr='b',
ssh_connect_timeout=30))
for host in hosts:
hostname, _ = client._get_host_port_info(host)
host_client = client._hosts_client[host]
host_client.run = mock_run
results = client.run('pwd')
for host in hosts:
result = results[host]
self.assertEqual(result['failed'], True)
self.assertEqual(result['stdout'], 'a')
self.assertEqual(result['stderr'], 'b')
self.assertEqual(result['return_code'], -9)
def test_run_command_json_output_transformed_to_object(self):
hosts = ["127.0.0.1"]
client = ParallelSSHClient(hosts=hosts,
user="******",
pkey_file="~/.ssh/id_rsa",
connect=True)
results = client.run("stuff", timeout=60)
self.assertIn("127.0.0.1", results)
self.assertDictEqual(results["127.0.0.1"]["stdout"], {"foo": "bar"})
def test_run_command_json_output_transformed_to_object(self):
hosts = ['127.0.0.1']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
results = client.run('stuff', timeout=60)
self.assertTrue('127.0.0.1' in results)
self.assertDictEqual(results['127.0.0.1']['stdout'], {'foo': 'bar'})
def test_run_command_json_output_transformed_to_object(self):
hosts = ['127.0.0.1']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
results = client.run('stuff', timeout=60)
self.assertIn('127.0.0.1', results)
self.assertDictEqual(results['127.0.0.1']['stdout'], {'foo': 'bar'})
def test_delete_file(self):
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
client.delete_file('/remote/stuff')
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].delete_file.assert_called_with('/remote/stuff')
def test_delete_file(self):
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
client.delete_file('/remote/stuff')
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].delete_file.assert_called_with('/remote/stuff')
def test_delete_file(self):
hosts = ["localhost", "127.0.0.1", "st2build001"]
client = ParallelSSHClient(hosts=hosts,
user="******",
pkey_file="~/.ssh/id_rsa",
connect=True)
client.delete_file("/remote/stuff")
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].delete_file.assert_called_with(
"/remote/stuff")
def test_delete_dir(self):
hosts = ["localhost", "127.0.0.1", "st2build001"]
client = ParallelSSHClient(hosts=hosts,
user="******",
pkey_file="~/.ssh/id_rsa",
connect=True)
client.delete_dir("/remote/stuff/", force=True)
expected_kwargs = {"force": True, "timeout": None}
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].delete_dir.assert_called_with(
"/remote/stuff/", **expected_kwargs)
def test_put(self):
hosts = ["localhost", "127.0.0.1", "st2build001"]
client = ParallelSSHClient(hosts=hosts,
user="******",
pkey_file="~/.ssh/id_rsa",
connect=True)
client.put("/local/stuff", "/remote/stuff", mode=0o744)
expected_kwargs = {"mode": 0o744, "mirror_local_mode": False}
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].put.assert_called_with(
"/local/stuff", "/remote/stuff", **expected_kwargs)
def test_run_command(self):
hosts = ["localhost", "127.0.0.1", "st2build001"]
client = ParallelSSHClient(hosts=hosts,
user="******",
pkey_file="~/.ssh/id_rsa",
connect=True)
client.run("pwd", timeout=60)
expected_kwargs = {"timeout": 60, "call_line_handler_func": True}
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].run.assert_called_with(
"pwd", **expected_kwargs)
def test_put(self):
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
client.put('/local/stuff', '/remote/stuff', mode=0744)
expected_kwargs = {'mode': 0744, 'mirror_local_mode': False}
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].put.assert_called_with(
'/local/stuff', '/remote/stuff', **expected_kwargs)
def test_connect_with_bastion(self):
hosts = ['localhost', '127.0.0.1']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
bastion_host='testing_bastion_host',
connect=False)
client.connect()
for host in hosts:
hostname, _ = client._get_host_port_info(host)
self.assertEqual(client._hosts_client[hostname].bastion_host, 'testing_bastion_host')
def test_connect_with_bastion(self):
hosts = ['localhost', '127.0.0.1']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
bastion_host='testing_bastion_host',
connect=False)
client.connect()
for host in hosts:
hostname, _ = client._get_host_port_info(host)
self.assertEqual(client._hosts_client[hostname].bastion_host, 'testing_bastion_host')
def test_delete_dir(self):
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
client.delete_dir('/remote/stuff/', force=True)
expected_kwargs = {'force': True, 'timeout': None}
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].delete_dir.assert_called_with(
'/remote/stuff/', **expected_kwargs)
def test_run_command(self):
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
client.run('pwd', timeout=60)
expected_kwargs = {'timeout': 60, 'call_line_handler_func': True}
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].run.assert_called_with(
'pwd', **expected_kwargs)
def test_run_command(self):
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
client.run('pwd', timeout=60)
expected_kwargs = {
'timeout': 60
}
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].run.assert_called_with('pwd', **expected_kwargs)
def main(user, pkey, password, hosts_str, cmd, file_path, dir_path,
delete_dir):
hosts = hosts_str.split(",")
client = ParallelSSHClient(user=user,
pkey_file=pkey,
password=password,
hosts=hosts)
pp = pprint.PrettyPrinter(indent=4)
if file_path:
if not os.path.exists(file_path):
raise Exception("File not found.")
results = client.put(file_path, "/home/lakshmi/test_file", mode="0660")
pp.pprint("Copy results: \n%s" % results)
results = client.run("ls -rlth")
pp.pprint("ls results: \n%s" % results)
if dir_path:
if not os.path.exists(dir_path):
raise Exception("File not found.")
results = client.put(dir_path, "/home/lakshmi/", mode="0660")
pp.pprint("Copy results: \n%s" % results)
results = client.run("ls -rlth")
pp.pprint("ls results: \n%s" % results)
if cmd:
results = client.run(cmd)
pp.pprint("cmd results: \n%s" % results)
if delete_dir:
results = client.delete_dir(delete_dir, force=True)
pp.pprint("Delete results: \n%s" % results)
def test_put(self):
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
client.put('/local/stuff', '/remote/stuff', mode=0o744)
expected_kwargs = {
'mode': 0o744,
'mirror_local_mode': False
}
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].put.assert_called_with('/local/stuff', '/remote/stuff',
**expected_kwargs)
def test_connect_with_bastion(self):
hosts = ["localhost", "127.0.0.1"]
client = ParallelSSHClient(
hosts=hosts,
user="******",
pkey_file="~/.ssh/id_rsa",
bastion_host="testing_bastion_host",
connect=False,
)
client.connect()
for host in hosts:
hostname, _ = client._get_host_port_info(host)
self.assertEqual(client._hosts_client[hostname].bastion_host,
"testing_bastion_host")
def test_delete_dir(self):
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
client.delete_dir('/remote/stuff/', force=True)
expected_kwargs = {
'force': True,
'timeout': None
}
for host in hosts:
hostname, _ = client._get_host_port_info(host)
client._hosts_client[hostname].delete_dir.assert_called_with('/remote/stuff/',
**expected_kwargs)
def test_host_port_info(self):
client = ParallelSSHClient(hosts=['dummy'],
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
# No port case. Port should be 22.
host_str = '1.2.3.4'
host, port = client._get_host_port_info(host_str)
self.assertEqual(host, host_str)
self.assertEqual(port, 22)
# IPv6 with square brackets with port specified.
host_str = '[fec2::10]:55'
host, port = client._get_host_port_info(host_str)
self.assertEqual(host, 'fec2::10')
self.assertEqual(port, 55)
def test_host_port_info(self):
client = ParallelSSHClient(hosts=["dummy"],
user="******",
pkey_file="~/.ssh/id_rsa",
connect=True)
# No port case. Port should be 22.
host_str = "1.2.3.4"
host, port = client._get_host_port_info(host_str)
self.assertEqual(host, host_str)
self.assertEqual(port, 22)
# IPv6 with square brackets with port specified.
host_str = "[fec2::10]:55"
host, port = client._get_host_port_info(host_str)
self.assertEqual(host, "fec2::10")
self.assertEqual(port, 55)
def test_host_port_info(self):
client = ParallelSSHClient(hosts=['dummy'],
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
# No port case. Port should be 22.
host_str = '1.2.3.4'
host, port = client._get_host_port_info(host_str)
self.assertEqual(host, host_str)
self.assertEqual(port, 22)
# IPv6 with square brackets with port specified.
host_str = '[fec2::10]:55'
host, port = client._get_host_port_info(host_str)
self.assertEqual(host, 'fec2::10')
self.assertEqual(port, 55)
def test_run_sudo_password_user_friendly_error(self):
hosts = ['127.0.0.1']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True,
sudo_password=True)
results = client.run('stuff', timeout=60)
expected_error = ('Failed executing command "stuff" on host "127.0.0.1" '
'Invalid sudo password provided or sudo is not configured for '
'this user (bar)')
self.assertIn('127.0.0.1', results)
self.assertEqual(results['127.0.0.1']['succeeded'], False)
self.assertEqual(results['127.0.0.1']['failed'], True)
self.assertIn(expected_error, results['127.0.0.1']['error'])
def test_run_sudo_password_user_friendly_error(self):
hosts = ['127.0.0.1']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True,
sudo_password=True)
results = client.run('stuff', timeout=60)
expected_error = ('Failed executing command "stuff" on host "127.0.0.1" '
'Invalid sudo password provided or sudo is not configured for '
'this user (bar)')
self.assertTrue('127.0.0.1' in results)
self.assertEqual(results['127.0.0.1']['succeeded'], False)
self.assertEqual(results['127.0.0.1']['failed'], True)
self.assertTrue(expected_error in results['127.0.0.1']['error'])
def test_connect_with_password(self):
hosts = ['localhost', '127.0.0.1']
client = ParallelSSHClient(hosts=hosts,
user='******',
password='******',
connect=False)
client.connect()
expected_conn = {
'allow_agent': False,
'look_for_keys': False,
'password': '******',
'username': '******',
'timeout': 60,
'port': 22
}
for host in hosts:
expected_conn['hostname'] = host
client._hosts_client[host].client.connect.assert_called_once_with(**expected_conn)
def test_connect_with_password(self):
hosts = ['localhost', '127.0.0.1']
client = ParallelSSHClient(hosts=hosts,
user='******',
password='******',
connect=False)
client.connect()
expected_conn = {
'allow_agent': False,
'look_for_keys': False,
'password': '******',
'username': '******',
'timeout': 60,
'port': 22
}
for host in hosts:
expected_conn['hostname'] = host
client._hosts_client[host].client.connect.assert_called_once_with(**expected_conn)
def test_connect_with_password(self):
hosts = ["localhost", "127.0.0.1"]
client = ParallelSSHClient(hosts=hosts,
user="******",
password="******",
connect=False)
client.connect()
expected_conn = {
"allow_agent": False,
"look_for_keys": False,
"password": "******",
"username": "******",
"timeout": 60,
"port": 22,
}
for host in hosts:
expected_conn["hostname"] = host
client._hosts_client[host].client.connect.assert_called_once_with(
**expected_conn)
def test_run_sudo_password_user_friendly_error(self):
hosts = ["127.0.0.1"]
client = ParallelSSHClient(
hosts=hosts,
user="******",
pkey_file="~/.ssh/id_rsa",
connect=True,
sudo_password=True,
)
results = client.run("stuff", timeout=60)
expected_error = (
'Failed executing command "stuff" on host "127.0.0.1" '
"Invalid sudo password provided or sudo is not configured for "
"this user (bar)")
self.assertIn("127.0.0.1", results)
self.assertEqual(results["127.0.0.1"]["succeeded"], False)
self.assertEqual(results["127.0.0.1"]["failed"], True)
self.assertIn(expected_error, results["127.0.0.1"]["error"])
def test_connect_with_key(self):
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=False)
client.connect()
expected_conn = {
'allow_agent': False,
'look_for_keys': False,
'key_filename': '~/.ssh/id_rsa',
'username': '******',
'timeout': 60,
'port': 22
}
for host in hosts:
hostname, port = client._get_host_port_info(host)
expected_conn['hostname'] = hostname
expected_conn['port'] = port
client._hosts_client[hostname].client.connect.assert_called_once_with(**expected_conn)
def test_connect_with_key(self):
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=False)
client.connect()
expected_conn = {
'allow_agent': False,
'look_for_keys': False,
'key_filename': '~/.ssh/id_rsa',
'username': '******',
'timeout': 60,
'port': 22
}
for host in hosts:
hostname, port = client._get_host_port_info(host)
expected_conn['hostname'] = hostname
expected_conn['port'] = port
client._hosts_client[hostname].client.connect.assert_called_once_with(**expected_conn)
def test_connect_with_key(self):
hosts = ["localhost", "127.0.0.1", "st2build001"]
client = ParallelSSHClient(hosts=hosts,
user="******",
pkey_file="~/.ssh/id_rsa",
connect=False)
client.connect()
expected_conn = {
"allow_agent": False,
"look_for_keys": False,
"key_filename": "~/.ssh/id_rsa",
"username": "******",
"timeout": 60,
"port": 22,
}
for host in hosts:
hostname, port = client._get_host_port_info(host)
expected_conn["hostname"] = hostname
expected_conn["port"] = port
client._hosts_client[
hostname].client.connect.assert_called_once_with(
**expected_conn)
def test_run_command_timeout(self):
# Make sure stdout and stderr is included on timeout
hosts = ['localhost', '127.0.0.1', 'st2build001']
client = ParallelSSHClient(hosts=hosts,
user='******',
pkey_file='~/.ssh/id_rsa',
connect=True)
mock_run = Mock(side_effect=SSHCommandTimeoutError(cmd='pwd', timeout=10,
stdout='a',
stderr='b'))
for host in hosts:
hostname, _ = client._get_host_port_info(host)
host_client = client._hosts_client[host]
host_client.run = mock_run
results = client.run('pwd')
for host in hosts:
result = results[host]
self.assertEqual(result['failed'], True)
self.assertEqual(result['stdout'], 'a')
self.assertEqual(result['stderr'], 'b')
self.assertEqual(result['return_code'], -9)
def test_cwd_used_correctly(self):
remote_action = ParamikoRemoteScriptAction(
'foo-script', bson.ObjectId(),
script_local_path_abs='/home/stanley/shiz_storm.py',
script_local_libs_path_abs=None,
named_args={}, positional_args=['blank space'], env_vars={},
on_behalf_user='******', user='******',
private_key='---SOME RSA KEY---',
remote_dir='/tmp', hosts=['127.0.0.1'], cwd='/test/cwd/'
)
paramiko_runner = ParamikoRemoteScriptRunner('runner_1')
paramiko_runner._parallel_ssh_client = ParallelSSHClient(['127.0.0.1'], 'stanley')
paramiko_runner._run_script_on_remote_host(remote_action)
exp_cmd = "cd /test/cwd/ && /tmp/shiz_storm.py 'blank space'"
ParallelSSHClient.run.assert_called_with(exp_cmd,
timeout=None)
def test_cwd_used_correctly(self):
remote_action = ParamikoRemoteScriptAction(
"foo-script",
bson.ObjectId(),
script_local_path_abs="/home/stanley/shiz_storm.py",
script_local_libs_path_abs=None,
named_args={},
positional_args=["blank space"],
env_vars={},
on_behalf_user="******",
user="******",
private_key="---SOME RSA KEY---",
remote_dir="/tmp",
hosts=["127.0.0.1"],
cwd="/test/cwd/",
)
paramiko_runner = ParamikoRemoteScriptRunner("runner_1")
paramiko_runner._parallel_ssh_client = ParallelSSHClient(["127.0.0.1"],
"stanley")
paramiko_runner._run_script_on_remote_host(remote_action)
exp_cmd = "cd /test/cwd/ && /tmp/shiz_storm.py 'blank space'"
ParallelSSHClient.run.assert_called_with(exp_cmd, timeout=None)
class BaseParallelSSHRunner(ActionRunner, ShellRunnerMixin):
def __init__(self, runner_id):
super(BaseParallelSSHRunner, self).__init__(runner_id=runner_id)
self._hosts = None
self._parallel = True
self._sudo = False
self._sudo_password = None
self._on_behalf_user = None
self._username = None
self._password = None
self._private_key = None
self._passphrase = None
self._kwarg_op = '--'
self._cwd = None
self._env = None
self._ssh_port = None
self._timeout = None
self._bastion_host = None
self._on_behalf_user = cfg.CONF.system_user.user
self._ssh_key_file = None
self._parallel_ssh_client = None
self._max_concurrency = cfg.CONF.ssh_runner.max_parallel_actions
def pre_run(self):
super(BaseParallelSSHRunner, self).pre_run()
LOG.debug('Entering BaseParallelSSHRunner.pre_run() for liveaction_id="%s"',
self.liveaction_id)
hosts = self.runner_parameters.get(RUNNER_HOSTS, '').split(',')
self._hosts = [h.strip() for h in hosts if len(h) > 0]
if len(self._hosts) < 1:
raise ActionRunnerPreRunError('No hosts specified to run action for action %s.',
self.liveaction_id)
self._username = self.runner_parameters.get(RUNNER_USERNAME, None)
self._password = self.runner_parameters.get(RUNNER_PASSWORD, None)
self._private_key = self.runner_parameters.get(RUNNER_PRIVATE_KEY, None)
self._passphrase = self.runner_parameters.get(RUNNER_PASSPHRASE, None)
self._ssh_port = self.runner_parameters.get(RUNNER_SSH_PORT, None)
self._ssh_key_file = self._private_key
self._parallel = self.runner_parameters.get(RUNNER_PARALLEL, True)
self._sudo = self.runner_parameters.get(RUNNER_SUDO, False)
self._sudo = self._sudo if self._sudo else False
self._sudo_password = self.runner_parameters.get(RUNNER_SUDO_PASSWORD, None)
if self.context:
self._on_behalf_user = self.context.get(RUNNER_ON_BEHALF_USER, self._on_behalf_user)
self._cwd = self.runner_parameters.get(RUNNER_CWD, None)
self._env = self.runner_parameters.get(RUNNER_ENV, {})
self._kwarg_op = self.runner_parameters.get(RUNNER_KWARG_OP, '--')
self._timeout = self.runner_parameters.get(RUNNER_TIMEOUT,
REMOTE_RUNNER_DEFAULT_ACTION_TIMEOUT)
self._bastion_host = self.runner_parameters.get(RUNNER_BASTION_HOST, None)
LOG.info('[BaseParallelSSHRunner="%s", liveaction_id="%s"] Finished pre_run.',
self.runner_id, self.liveaction_id)
concurrency = int(len(self._hosts) / 3) + 1 if self._parallel else 1
if concurrency > self._max_concurrency:
LOG.debug('Limiting parallel SSH concurrency to %d.', concurrency)
concurrency = self._max_concurrency
client_kwargs = {
'hosts': self._hosts,
'user': self._username,
'port': self._ssh_port,
'concurrency': concurrency,
'bastion_host': self._bastion_host,
'raise_on_any_error': False,
'connect': True
}
def make_store_stdout_line_func(execution_db, action_db):
def store_stdout_line(line):
if cfg.CONF.actionrunner.stream_output:
store_execution_output_data(execution_db=execution_db, action_db=action_db,
data=line, output_type='stdout')
return store_stdout_line
def make_store_stderr_line_func(execution_db, action_db):
def store_stderr_line(line):
if cfg.CONF.actionrunner.stream_output:
store_execution_output_data(execution_db=execution_db, action_db=action_db,
data=line, output_type='stderr')
return store_stderr_line
handle_stdout_line_func = make_store_stdout_line_func(execution_db=self.execution,
action_db=self.action)
handle_stderr_line_func = make_store_stderr_line_func(execution_db=self.execution,
action_db=self.action)
if len(self._hosts) == 1:
# We only support streaming output when running action on one host. That is because
# the action output is tied to a particulat execution. User can still achieve output
# streaming for multiple hosts by running one execution per host.
client_kwargs['handle_stdout_line_func'] = handle_stdout_line_func
client_kwargs['handle_stderr_line_func'] = handle_stderr_line_func
else:
LOG.debug('Real-time action output streaming is disabled, because action is running '
'on more than one host')
if self._password:
client_kwargs['password'] = self._password
elif self._private_key:
# Determine if the private_key is a path to the key file or the raw key material
is_key_material = self._is_private_key_material(private_key=self._private_key)
if is_key_material:
# Raw key material
client_kwargs['pkey_material'] = self._private_key
else:
# Assume it's a path to the key file, verify the file exists
client_kwargs['pkey_file'] = self._private_key
if self._passphrase:
client_kwargs['passphrase'] = self._passphrase
else:
# Default to stanley key file specified in the config
client_kwargs['pkey_file'] = self._ssh_key_file
if self._sudo_password:
client_kwargs['sudo_password'] = True
self._parallel_ssh_client = ParallelSSHClient(**client_kwargs)
def post_run(self, status, result):
super(BaseParallelSSHRunner, self).post_run(status=status, result=result)
# Ensure we close the connection when the action execution finishes
if self._parallel_ssh_client:
self._parallel_ssh_client.close()
def _is_private_key_material(self, private_key):
return private_key and REMOTE_RUNNER_PRIVATE_KEY_HEADER in private_key.lower()
def _get_env_vars(self):
"""
:rtype: ``dict``
"""
env_vars = {}
if self._env:
env_vars.update(self._env)
# Include common st2 env vars
st2_env_vars = self._get_common_action_env_variables()
env_vars.update(st2_env_vars)
return env_vars
@staticmethod
def _get_result_status(result, allow_partial_failure):
if 'error' in result and 'traceback' in result:
# Assume this is a global failure where the result dictionary doesn't contain entry
# per host
timeout = False
success = result.get('succeeded', False)
status = BaseParallelSSHRunner._get_status_for_success_and_timeout(success=success,
timeout=timeout)
return status
success = not allow_partial_failure
timeout = True
for r in six.itervalues(result):
r_succeess = r.get('succeeded', False) if r else False
r_timeout = r.get('timeout', False) if r else False
timeout &= r_timeout
if allow_partial_failure:
success |= r_succeess
if success:
break
else:
success &= r_succeess
if not success:
break
status = BaseParallelSSHRunner._get_status_for_success_and_timeout(success=success,
timeout=timeout)
return status
@staticmethod
def _get_status_for_success_and_timeout(success, timeout):
if success:
status = LIVEACTION_STATUS_SUCCEEDED
elif timeout:
# Note: Right now we only set status to timeout if all the hosts have timed out
status = LIVEACTION_STATUS_TIMED_OUT
else:
status = LIVEACTION_STATUS_FAILED
return status
def pre_run(self):
super(BaseParallelSSHRunner, self).pre_run()
LOG.debug('Entering BaseParallelSSHRunner.pre_run() for liveaction_id="%s"',
self.liveaction_id)
hosts = self.runner_parameters.get(RUNNER_HOSTS, '').split(',')
self._hosts = [h.strip() for h in hosts if len(h) > 0]
if len(self._hosts) < 1:
raise ActionRunnerPreRunError('No hosts specified to run action for action %s.',
self.liveaction_id)
self._username = self.runner_parameters.get(RUNNER_USERNAME, None)
self._password = self.runner_parameters.get(RUNNER_PASSWORD, None)
self._private_key = self.runner_parameters.get(RUNNER_PRIVATE_KEY, None)
self._passphrase = self.runner_parameters.get(RUNNER_PASSPHRASE, None)
self._ssh_port = self.runner_parameters.get(RUNNER_SSH_PORT, None)
self._ssh_key_file = self._private_key
self._parallel = self.runner_parameters.get(RUNNER_PARALLEL, True)
self._sudo = self.runner_parameters.get(RUNNER_SUDO, False)
self._sudo = self._sudo if self._sudo else False
self._sudo_password = self.runner_parameters.get(RUNNER_SUDO_PASSWORD, None)
if self.context:
self._on_behalf_user = self.context.get(RUNNER_ON_BEHALF_USER, self._on_behalf_user)
self._cwd = self.runner_parameters.get(RUNNER_CWD, None)
self._env = self.runner_parameters.get(RUNNER_ENV, {})
self._kwarg_op = self.runner_parameters.get(RUNNER_KWARG_OP, '--')
self._timeout = self.runner_parameters.get(RUNNER_TIMEOUT,
REMOTE_RUNNER_DEFAULT_ACTION_TIMEOUT)
self._bastion_host = self.runner_parameters.get(RUNNER_BASTION_HOST, None)
LOG.info('[BaseParallelSSHRunner="%s", liveaction_id="%s"] Finished pre_run.',
self.runner_id, self.liveaction_id)
concurrency = int(len(self._hosts) / 3) + 1 if self._parallel else 1
if concurrency > self._max_concurrency:
LOG.debug('Limiting parallel SSH concurrency to %d.', concurrency)
concurrency = self._max_concurrency
client_kwargs = {
'hosts': self._hosts,
'user': self._username,
'port': self._ssh_port,
'concurrency': concurrency,
'bastion_host': self._bastion_host,
'raise_on_any_error': False,
'connect': True
}
def make_store_stdout_line_func(execution_db, action_db):
def store_stdout_line(line):
if cfg.CONF.actionrunner.stream_output:
store_execution_output_data(execution_db=execution_db, action_db=action_db,
data=line, output_type='stdout')
return store_stdout_line
def make_store_stderr_line_func(execution_db, action_db):
def store_stderr_line(line):
if cfg.CONF.actionrunner.stream_output:
store_execution_output_data(execution_db=execution_db, action_db=action_db,
data=line, output_type='stderr')
return store_stderr_line
handle_stdout_line_func = make_store_stdout_line_func(execution_db=self.execution,
action_db=self.action)
handle_stderr_line_func = make_store_stderr_line_func(execution_db=self.execution,
action_db=self.action)
if len(self._hosts) == 1:
# We only support streaming output when running action on one host. That is because
# the action output is tied to a particulat execution. User can still achieve output
# streaming for multiple hosts by running one execution per host.
client_kwargs['handle_stdout_line_func'] = handle_stdout_line_func
client_kwargs['handle_stderr_line_func'] = handle_stderr_line_func
else:
LOG.debug('Real-time action output streaming is disabled, because action is running '
'on more than one host')
if self._password:
client_kwargs['password'] = self._password
elif self._private_key:
# Determine if the private_key is a path to the key file or the raw key material
is_key_material = self._is_private_key_material(private_key=self._private_key)
if is_key_material:
# Raw key material
client_kwargs['pkey_material'] = self._private_key
else:
# Assume it's a path to the key file, verify the file exists
client_kwargs['pkey_file'] = self._private_key
if self._passphrase:
client_kwargs['passphrase'] = self._passphrase
else:
# Default to stanley key file specified in the config
client_kwargs['pkey_file'] = self._ssh_key_file
if self._sudo_password:
client_kwargs['sudo_password'] = True
self._parallel_ssh_client = ParallelSSHClient(**client_kwargs)
def pre_run(self):
super(BaseParallelSSHRunner, self).pre_run()
LOG.debug('Entering BaseParallelSSHRunner.pre_run() for liveaction_id="%s"',
self.liveaction_id)
hosts = self.runner_parameters.get(RUNNER_HOSTS, '').split(',')
self._hosts = [h.strip() for h in hosts if len(h) > 0]
if len(self._hosts) < 1:
raise ActionRunnerPreRunError('No hosts specified to run action for action %s.',
self.liveaction_id)
self._username = self.runner_parameters.get(RUNNER_USERNAME, None)
self._password = self.runner_parameters.get(RUNNER_PASSWORD, None)
self._private_key = self.runner_parameters.get(RUNNER_PRIVATE_KEY, None)
self._passphrase = self.runner_parameters.get(RUNNER_PASSPHRASE, None)
self._ssh_port = self.runner_parameters.get(RUNNER_SSH_PORT, None)
self._ssh_key_file = self._private_key
self._parallel = self.runner_parameters.get(RUNNER_PARALLEL, True)
self._sudo = self.runner_parameters.get(RUNNER_SUDO, False)
self._sudo = self._sudo if self._sudo else False
if self.context:
self._on_behalf_user = self.context.get(RUNNER_ON_BEHALF_USER, self._on_behalf_user)
self._cwd = self.runner_parameters.get(RUNNER_CWD, None)
self._env = self.runner_parameters.get(RUNNER_ENV, {})
self._kwarg_op = self.runner_parameters.get(RUNNER_KWARG_OP, '--')
self._timeout = self.runner_parameters.get(RUNNER_TIMEOUT,
REMOTE_RUNNER_DEFAULT_ACTION_TIMEOUT)
self._bastion_host = self.runner_parameters.get(RUNNER_BASTION_HOST, None)
LOG.info('[BaseParallelSSHRunner="%s", liveaction_id="%s"] Finished pre_run.',
self.runner_id, self.liveaction_id)
concurrency = int(len(self._hosts) / 3) + 1 if self._parallel else 1
if concurrency > self._max_concurrency:
LOG.debug('Limiting parallel SSH concurrency to %d.', concurrency)
concurrency = self._max_concurrency
client_kwargs = {
'hosts': self._hosts,
'user': self._username,
'port': self._ssh_port,
'concurrency': concurrency,
'bastion_host': self._bastion_host,
'raise_on_any_error': False,
'connect': True
}
if self._password:
client_kwargs['password'] = self._password
elif self._private_key:
# Determine if the private_key is a path to the key file or the raw key material
is_key_material = self._is_private_key_material(private_key=self._private_key)
if is_key_material:
# Raw key material
client_kwargs['pkey_material'] = self._private_key
else:
# Assume it's a path to the key file, verify the file exists
client_kwargs['pkey_file'] = self._private_key
if self._passphrase:
client_kwargs['passphrase'] = self._passphrase
else:
# Default to stanley key file specified in the config
client_kwargs['pkey_file'] = self._ssh_key_file
self._parallel_ssh_client = ParallelSSHClient(**client_kwargs)
def pre_run(self):
super(BaseParallelSSHRunner, self).pre_run()
LOG.debug(
'Entering BaseParallelSSHRunner.pre_run() for liveaction_id="%s"',
self.liveaction_id)
hosts = self.runner_parameters.get(RUNNER_HOSTS, '').split(',')
self._hosts = [h.strip() for h in hosts if len(h) > 0]
if len(self._hosts) < 1:
raise ActionRunnerPreRunError(
'No hosts specified to run action for action %s.',
self.liveaction_id)
self._username = self.runner_parameters.get(RUNNER_USERNAME, None)
self._password = self.runner_parameters.get(RUNNER_PASSWORD, None)
self._private_key = self.runner_parameters.get(RUNNER_PRIVATE_KEY,
None)
self._passphrase = self.runner_parameters.get(RUNNER_PASSPHRASE, None)
self._ssh_port = self.runner_parameters.get(RUNNER_SSH_PORT, None)
self._ssh_key_file = self._private_key
self._parallel = self.runner_parameters.get(RUNNER_PARALLEL, True)
self._sudo = self.runner_parameters.get(RUNNER_SUDO, False)
self._sudo = self._sudo if self._sudo else False
self._sudo_password = self.runner_parameters.get(
RUNNER_SUDO_PASSWORD, None)
if self.context:
self._on_behalf_user = self.context.get(RUNNER_ON_BEHALF_USER,
self._on_behalf_user)
self._cwd = self.runner_parameters.get(RUNNER_CWD, None)
self._env = self.runner_parameters.get(RUNNER_ENV, {})
self._kwarg_op = self.runner_parameters.get(RUNNER_KWARG_OP, '--')
self._timeout = self.runner_parameters.get(
RUNNER_TIMEOUT, REMOTE_RUNNER_DEFAULT_ACTION_TIMEOUT)
self._bastion_host = self.runner_parameters.get(
RUNNER_BASTION_HOST, None)
LOG.info(
'[BaseParallelSSHRunner="%s", liveaction_id="%s"] Finished pre_run.',
self.runner_id, self.liveaction_id)
concurrency = int(len(self._hosts) / 3) + 1 if self._parallel else 1
if concurrency > self._max_concurrency:
LOG.debug('Limiting parallel SSH concurrency to %d.', concurrency)
concurrency = self._max_concurrency
client_kwargs = {
'hosts': self._hosts,
'user': self._username,
'port': self._ssh_port,
'concurrency': concurrency,
'bastion_host': self._bastion_host,
'raise_on_any_error': False,
'connect': True
}
def make_store_stdout_line_func(execution_db, action_db):
def store_stdout_line(line):
if cfg.CONF.actionrunner.stream_output:
store_execution_output_data(execution_db=execution_db,
action_db=action_db,
data=line,
output_type='stdout')
return store_stdout_line
def make_store_stderr_line_func(execution_db, action_db):
def store_stderr_line(line):
if cfg.CONF.actionrunner.stream_output:
store_execution_output_data(execution_db=execution_db,
action_db=action_db,
data=line,
output_type='stderr')
return store_stderr_line
handle_stdout_line_func = make_store_stdout_line_func(
execution_db=self.execution, action_db=self.action)
handle_stderr_line_func = make_store_stderr_line_func(
execution_db=self.execution, action_db=self.action)
if len(self._hosts) == 1:
# We only support streaming output when running action on one host. That is because
# the action output is tied to a particulat execution. User can still achieve output
# streaming for multiple hosts by running one execution per host.
client_kwargs['handle_stdout_line_func'] = handle_stdout_line_func
client_kwargs['handle_stderr_line_func'] = handle_stderr_line_func
if self._password:
client_kwargs['password'] = self._password
elif self._private_key:
# Determine if the private_key is a path to the key file or the raw key material
is_key_material = self._is_private_key_material(
private_key=self._private_key)
if is_key_material:
# Raw key material
client_kwargs['pkey_material'] = self._private_key
else:
# Assume it's a path to the key file, verify the file exists
client_kwargs['pkey_file'] = self._private_key
if self._passphrase:
client_kwargs['passphrase'] = self._passphrase
else:
# Default to stanley key file specified in the config
client_kwargs['pkey_file'] = self._ssh_key_file
if self._sudo_password:
client_kwargs['sudo_password'] = True
self._parallel_ssh_client = ParallelSSHClient(**client_kwargs)
class BaseParallelSSHRunner(ActionRunner, ShellRunnerMixin):
def __init__(self, runner_id):
super(BaseParallelSSHRunner, self).__init__(runner_id=runner_id)
self._hosts = None
self._parallel = True
self._sudo = False
self._sudo_password = None
self._on_behalf_user = None
self._username = None
self._password = None
self._private_key = None
self._passphrase = None
self._kwarg_op = '--'
self._cwd = None
self._env = None
self._ssh_port = None
self._timeout = None
self._bastion_host = None
self._on_behalf_user = cfg.CONF.system_user.user
self._ssh_key_file = None
self._parallel_ssh_client = None
self._max_concurrency = cfg.CONF.ssh_runner.max_parallel_actions
def pre_run(self):
super(BaseParallelSSHRunner, self).pre_run()
LOG.debug(
'Entering BaseParallelSSHRunner.pre_run() for liveaction_id="%s"',
self.liveaction_id)
hosts = self.runner_parameters.get(RUNNER_HOSTS, '').split(',')
self._hosts = [h.strip() for h in hosts if len(h) > 0]
if len(self._hosts) < 1:
raise ActionRunnerPreRunError(
'No hosts specified to run action for action %s.',
self.liveaction_id)
self._username = self.runner_parameters.get(RUNNER_USERNAME, None)
self._password = self.runner_parameters.get(RUNNER_PASSWORD, None)
self._private_key = self.runner_parameters.get(RUNNER_PRIVATE_KEY,
None)
self._passphrase = self.runner_parameters.get(RUNNER_PASSPHRASE, None)
self._ssh_port = self.runner_parameters.get(RUNNER_SSH_PORT, None)
self._ssh_key_file = self._private_key
self._parallel = self.runner_parameters.get(RUNNER_PARALLEL, True)
self._sudo = self.runner_parameters.get(RUNNER_SUDO, False)
self._sudo = self._sudo if self._sudo else False
self._sudo_password = self.runner_parameters.get(
RUNNER_SUDO_PASSWORD, None)
if self.context:
self._on_behalf_user = self.context.get(RUNNER_ON_BEHALF_USER,
self._on_behalf_user)
self._cwd = self.runner_parameters.get(RUNNER_CWD, None)
self._env = self.runner_parameters.get(RUNNER_ENV, {})
self._kwarg_op = self.runner_parameters.get(RUNNER_KWARG_OP, '--')
self._timeout = self.runner_parameters.get(
RUNNER_TIMEOUT, REMOTE_RUNNER_DEFAULT_ACTION_TIMEOUT)
self._bastion_host = self.runner_parameters.get(
RUNNER_BASTION_HOST, None)
LOG.info(
'[BaseParallelSSHRunner="%s", liveaction_id="%s"] Finished pre_run.',
self.runner_id, self.liveaction_id)
concurrency = int(len(self._hosts) / 3) + 1 if self._parallel else 1
if concurrency > self._max_concurrency:
LOG.debug('Limiting parallel SSH concurrency to %d.', concurrency)
concurrency = self._max_concurrency
client_kwargs = {
'hosts': self._hosts,
'user': self._username,
'port': self._ssh_port,
'concurrency': concurrency,
'bastion_host': self._bastion_host,
'raise_on_any_error': False,
'connect': True
}
def make_store_stdout_line_func(execution_db, action_db):
def store_stdout_line(line):
if cfg.CONF.actionrunner.stream_output:
store_execution_output_data(execution_db=execution_db,
action_db=action_db,
data=line,
output_type='stdout')
return store_stdout_line
def make_store_stderr_line_func(execution_db, action_db):
def store_stderr_line(line):
if cfg.CONF.actionrunner.stream_output:
store_execution_output_data(execution_db=execution_db,
action_db=action_db,
data=line,
output_type='stderr')
return store_stderr_line
handle_stdout_line_func = make_store_stdout_line_func(
execution_db=self.execution, action_db=self.action)
handle_stderr_line_func = make_store_stderr_line_func(
execution_db=self.execution, action_db=self.action)
if len(self._hosts) == 1:
# We only support streaming output when running action on one host. That is because
# the action output is tied to a particulat execution. User can still achieve output
# streaming for multiple hosts by running one execution per host.
client_kwargs['handle_stdout_line_func'] = handle_stdout_line_func
client_kwargs['handle_stderr_line_func'] = handle_stderr_line_func
if self._password:
client_kwargs['password'] = self._password
elif self._private_key:
# Determine if the private_key is a path to the key file or the raw key material
is_key_material = self._is_private_key_material(
private_key=self._private_key)
if is_key_material:
# Raw key material
client_kwargs['pkey_material'] = self._private_key
else:
# Assume it's a path to the key file, verify the file exists
client_kwargs['pkey_file'] = self._private_key
if self._passphrase:
client_kwargs['passphrase'] = self._passphrase
else:
# Default to stanley key file specified in the config
client_kwargs['pkey_file'] = self._ssh_key_file
if self._sudo_password:
client_kwargs['sudo_password'] = True
self._parallel_ssh_client = ParallelSSHClient(**client_kwargs)
def post_run(self, status, result):
super(BaseParallelSSHRunner, self).post_run(status=status,
result=result)
# Ensure we close the connection when the action execution finishes
if self._parallel_ssh_client:
self._parallel_ssh_client.close()
def _is_private_key_material(self, private_key):
return private_key and REMOTE_RUNNER_PRIVATE_KEY_HEADER in private_key.lower(
)
def _get_env_vars(self):
"""
:rtype: ``dict``
"""
env_vars = {}
if self._env:
env_vars.update(self._env)
# Include common st2 env vars
st2_env_vars = self._get_common_action_env_variables()
env_vars.update(st2_env_vars)
return env_vars
@staticmethod
def _get_result_status(result, allow_partial_failure):
if 'error' in result and 'traceback' in result:
# Assume this is a global failure where the result dictionary doesn't contain entry
# per host
timeout = False
success = result.get('succeeded', False)
status = BaseParallelSSHRunner._get_status_for_success_and_timeout(
success=success, timeout=timeout)
return status
success = not allow_partial_failure
timeout = True
for r in six.itervalues(result):
r_succeess = r.get('succeeded', False) if r else False
r_timeout = r.get('timeout', False) if r else False
timeout &= r_timeout
if allow_partial_failure:
success |= r_succeess
if success:
break
else:
success &= r_succeess
if not success:
break
status = BaseParallelSSHRunner._get_status_for_success_and_timeout(
success=success, timeout=timeout)
return status
@staticmethod
def _get_status_for_success_and_timeout(success, timeout):
if success:
status = LIVEACTION_STATUS_SUCCEEDED
elif timeout:
# Note: Right now we only set status to timeout if all the hosts have timed out
status = LIVEACTION_STATUS_TIMED_OUT
else:
status = LIVEACTION_STATUS_FAILED
return status
