class BaseParallelSSHRunner(ActionRunner, ShellRunnerMixin): def __init__(self, runner_id): super(BaseParallelSSHRunner, self).__init__(runner_id=runner_id) self._hosts = None self._parallel = True self._sudo = False self._sudo_password = None self._on_behalf_user = None self._username = None self._password = None self._private_key = None self._passphrase = None self._kwarg_op = '--' self._cwd = None self._env = None self._ssh_port = None self._timeout = None self._bastion_host = None self._on_behalf_user = cfg.CONF.system_user.user self._ssh_key_file = None self._parallel_ssh_client = None self._max_concurrency = cfg.CONF.ssh_runner.max_parallel_actions def pre_run(self): super(BaseParallelSSHRunner, self).pre_run() LOG.debug( 'Entering BaseParallelSSHRunner.pre_run() for liveaction_id="%s"', self.liveaction_id) hosts = self.runner_parameters.get(RUNNER_HOSTS, '').split(',') self._hosts = [h.strip() for h in hosts if len(h) > 0] if len(self._hosts) < 1: raise ActionRunnerPreRunError( 'No hosts specified to run action for action %s.', self.liveaction_id) self._username = self.runner_parameters.get(RUNNER_USERNAME, None) self._password = self.runner_parameters.get(RUNNER_PASSWORD, None) self._private_key = self.runner_parameters.get(RUNNER_PRIVATE_KEY, None) self._passphrase = self.runner_parameters.get(RUNNER_PASSPHRASE, None) self._ssh_port = self.runner_parameters.get(RUNNER_SSH_PORT, None) self._ssh_key_file = self._private_key self._parallel = self.runner_parameters.get(RUNNER_PARALLEL, True) self._sudo = self.runner_parameters.get(RUNNER_SUDO, False) self._sudo = self._sudo if self._sudo else False self._sudo_password = self.runner_parameters.get( RUNNER_SUDO_PASSWORD, None) if self.context: self._on_behalf_user = self.context.get(RUNNER_ON_BEHALF_USER, self._on_behalf_user) self._cwd = self.runner_parameters.get(RUNNER_CWD, None) self._env = self.runner_parameters.get(RUNNER_ENV, {}) self._kwarg_op = self.runner_parameters.get(RUNNER_KWARG_OP, '--') self._timeout = self.runner_parameters.get( RUNNER_TIMEOUT, REMOTE_RUNNER_DEFAULT_ACTION_TIMEOUT) self._bastion_host = self.runner_parameters.get( RUNNER_BASTION_HOST, None) LOG.info( '[BaseParallelSSHRunner="%s", liveaction_id="%s"] Finished pre_run.', self.runner_id, self.liveaction_id) concurrency = int(len(self._hosts) / 3) + 1 if self._parallel else 1 if concurrency > self._max_concurrency: LOG.debug('Limiting parallel SSH concurrency to %d.', concurrency) concurrency = self._max_concurrency client_kwargs = { 'hosts': self._hosts, 'user': self._username, 'port': self._ssh_port, 'concurrency': concurrency, 'bastion_host': self._bastion_host, 'raise_on_any_error': False, 'connect': True } def make_store_stdout_line_func(execution_db, action_db): def store_stdout_line(line): if cfg.CONF.actionrunner.stream_output: store_execution_output_data(execution_db=execution_db, action_db=action_db, data=line, output_type='stdout') return store_stdout_line def make_store_stderr_line_func(execution_db, action_db): def store_stderr_line(line): if cfg.CONF.actionrunner.stream_output: store_execution_output_data(execution_db=execution_db, action_db=action_db, data=line, output_type='stderr') return store_stderr_line handle_stdout_line_func = make_store_stdout_line_func( execution_db=self.execution, action_db=self.action) handle_stderr_line_func = make_store_stderr_line_func( execution_db=self.execution, action_db=self.action) if len(self._hosts) == 1: # We only support streaming output when running action on one host. That is because # the action output is tied to a particulat execution. User can still achieve output # streaming for multiple hosts by running one execution per host. client_kwargs['handle_stdout_line_func'] = handle_stdout_line_func client_kwargs['handle_stderr_line_func'] = handle_stderr_line_func if self._password: client_kwargs['password'] = self._password elif self._private_key: # Determine if the private_key is a path to the key file or the raw key material is_key_material = self._is_private_key_material( private_key=self._private_key) if is_key_material: # Raw key material client_kwargs['pkey_material'] = self._private_key else: # Assume it's a path to the key file, verify the file exists client_kwargs['pkey_file'] = self._private_key if self._passphrase: client_kwargs['passphrase'] = self._passphrase else: # Default to stanley key file specified in the config client_kwargs['pkey_file'] = self._ssh_key_file if self._sudo_password: client_kwargs['sudo_password'] = True self._parallel_ssh_client = ParallelSSHClient(**client_kwargs) def post_run(self, status, result): super(BaseParallelSSHRunner, self).post_run(status=status, result=result) # Ensure we close the connection when the action execution finishes if self._parallel_ssh_client: self._parallel_ssh_client.close() def _is_private_key_material(self, private_key): return private_key and REMOTE_RUNNER_PRIVATE_KEY_HEADER in private_key.lower( ) def _get_env_vars(self): """ :rtype: ``dict`` """ env_vars = {} if self._env: env_vars.update(self._env) # Include common st2 env vars st2_env_vars = self._get_common_action_env_variables() env_vars.update(st2_env_vars) return env_vars @staticmethod def _get_result_status(result, allow_partial_failure): if 'error' in result and 'traceback' in result: # Assume this is a global failure where the result dictionary doesn't contain entry # per host timeout = False success = result.get('succeeded', False) status = BaseParallelSSHRunner._get_status_for_success_and_timeout( success=success, timeout=timeout) return status success = not allow_partial_failure timeout = True for r in six.itervalues(result): r_succeess = r.get('succeeded', False) if r else False r_timeout = r.get('timeout', False) if r else False timeout &= r_timeout if allow_partial_failure: success |= r_succeess if success: break else: success &= r_succeess if not success: break status = BaseParallelSSHRunner._get_status_for_success_and_timeout( success=success, timeout=timeout) return status @staticmethod def _get_status_for_success_and_timeout(success, timeout): if success: status = LIVEACTION_STATUS_SUCCEEDED elif timeout: # Note: Right now we only set status to timeout if all the hosts have timed out status = LIVEACTION_STATUS_TIMED_OUT else: status = LIVEACTION_STATUS_FAILED return status
class BaseParallelSSHRunner(ActionRunner, ShellRunnerMixin): def __init__(self, runner_id): super(BaseParallelSSHRunner, self).__init__(runner_id=runner_id) self._hosts = None self._parallel = True self._sudo = False self._sudo_password = None self._on_behalf_user = None self._username = None self._password = None self._private_key = None self._passphrase = None self._kwarg_op = '--' self._cwd = None self._env = None self._ssh_port = None self._timeout = None self._bastion_host = None self._on_behalf_user = cfg.CONF.system_user.user self._ssh_key_file = None self._parallel_ssh_client = None self._max_concurrency = cfg.CONF.ssh_runner.max_parallel_actions def pre_run(self): super(BaseParallelSSHRunner, self).pre_run() LOG.debug('Entering BaseParallelSSHRunner.pre_run() for liveaction_id="%s"', self.liveaction_id) hosts = self.runner_parameters.get(RUNNER_HOSTS, '').split(',') self._hosts = [h.strip() for h in hosts if len(h) > 0] if len(self._hosts) < 1: raise ActionRunnerPreRunError('No hosts specified to run action for action %s.', self.liveaction_id) self._username = self.runner_parameters.get(RUNNER_USERNAME, None) self._password = self.runner_parameters.get(RUNNER_PASSWORD, None) self._private_key = self.runner_parameters.get(RUNNER_PRIVATE_KEY, None) self._passphrase = self.runner_parameters.get(RUNNER_PASSPHRASE, None) self._ssh_port = self.runner_parameters.get(RUNNER_SSH_PORT, None) self._ssh_key_file = self._private_key self._parallel = self.runner_parameters.get(RUNNER_PARALLEL, True) self._sudo = self.runner_parameters.get(RUNNER_SUDO, False) self._sudo = self._sudo if self._sudo else False self._sudo_password = self.runner_parameters.get(RUNNER_SUDO_PASSWORD, None) if self.context: self._on_behalf_user = self.context.get(RUNNER_ON_BEHALF_USER, self._on_behalf_user) self._cwd = self.runner_parameters.get(RUNNER_CWD, None) self._env = self.runner_parameters.get(RUNNER_ENV, {}) self._kwarg_op = self.runner_parameters.get(RUNNER_KWARG_OP, '--') self._timeout = self.runner_parameters.get(RUNNER_TIMEOUT, REMOTE_RUNNER_DEFAULT_ACTION_TIMEOUT) self._bastion_host = self.runner_parameters.get(RUNNER_BASTION_HOST, None) LOG.info('[BaseParallelSSHRunner="%s", liveaction_id="%s"] Finished pre_run.', self.runner_id, self.liveaction_id) concurrency = int(len(self._hosts) / 3) + 1 if self._parallel else 1 if concurrency > self._max_concurrency: LOG.debug('Limiting parallel SSH concurrency to %d.', concurrency) concurrency = self._max_concurrency client_kwargs = { 'hosts': self._hosts, 'user': self._username, 'port': self._ssh_port, 'concurrency': concurrency, 'bastion_host': self._bastion_host, 'raise_on_any_error': False, 'connect': True } def make_store_stdout_line_func(execution_db, action_db): def store_stdout_line(line): if cfg.CONF.actionrunner.stream_output: store_execution_output_data(execution_db=execution_db, action_db=action_db, data=line, output_type='stdout') return store_stdout_line def make_store_stderr_line_func(execution_db, action_db): def store_stderr_line(line): if cfg.CONF.actionrunner.stream_output: store_execution_output_data(execution_db=execution_db, action_db=action_db, data=line, output_type='stderr') return store_stderr_line handle_stdout_line_func = make_store_stdout_line_func(execution_db=self.execution, action_db=self.action) handle_stderr_line_func = make_store_stderr_line_func(execution_db=self.execution, action_db=self.action) if len(self._hosts) == 1: # We only support streaming output when running action on one host. That is because # the action output is tied to a particulat execution. User can still achieve output # streaming for multiple hosts by running one execution per host. client_kwargs['handle_stdout_line_func'] = handle_stdout_line_func client_kwargs['handle_stderr_line_func'] = handle_stderr_line_func else: LOG.debug('Real-time action output streaming is disabled, because action is running ' 'on more than one host') if self._password: client_kwargs['password'] = self._password elif self._private_key: # Determine if the private_key is a path to the key file or the raw key material is_key_material = self._is_private_key_material(private_key=self._private_key) if is_key_material: # Raw key material client_kwargs['pkey_material'] = self._private_key else: # Assume it's a path to the key file, verify the file exists client_kwargs['pkey_file'] = self._private_key if self._passphrase: client_kwargs['passphrase'] = self._passphrase else: # Default to stanley key file specified in the config client_kwargs['pkey_file'] = self._ssh_key_file if self._sudo_password: client_kwargs['sudo_password'] = True self._parallel_ssh_client = ParallelSSHClient(**client_kwargs) def post_run(self, status, result): super(BaseParallelSSHRunner, self).post_run(status=status, result=result) # Ensure we close the connection when the action execution finishes if self._parallel_ssh_client: self._parallel_ssh_client.close() def _is_private_key_material(self, private_key): return private_key and REMOTE_RUNNER_PRIVATE_KEY_HEADER in private_key.lower() def _get_env_vars(self): """ :rtype: ``dict`` """ env_vars = {} if self._env: env_vars.update(self._env) # Include common st2 env vars st2_env_vars = self._get_common_action_env_variables() env_vars.update(st2_env_vars) return env_vars @staticmethod def _get_result_status(result, allow_partial_failure): if 'error' in result and 'traceback' in result: # Assume this is a global failure where the result dictionary doesn't contain entry # per host timeout = False success = result.get('succeeded', False) status = BaseParallelSSHRunner._get_status_for_success_and_timeout(success=success, timeout=timeout) return status success = not allow_partial_failure timeout = True for r in six.itervalues(result): r_succeess = r.get('succeeded', False) if r else False r_timeout = r.get('timeout', False) if r else False timeout &= r_timeout if allow_partial_failure: success |= r_succeess if success: break else: success &= r_succeess if not success: break status = BaseParallelSSHRunner._get_status_for_success_and_timeout(success=success, timeout=timeout) return status @staticmethod def _get_status_for_success_and_timeout(success, timeout): if success: status = LIVEACTION_STATUS_SUCCEEDED elif timeout: # Note: Right now we only set status to timeout if all the hosts have timed out status = LIVEACTION_STATUS_TIMED_OUT else: status = LIVEACTION_STATUS_FAILED return status