def setUpClass(cls):
super(SensorContainerTestCase, cls).setUpClass()
st2tests.config.parse_args()
username = cfg.CONF.database.username if hasattr(
cfg.CONF.database, 'username') else None
password = cfg.CONF.database.password if hasattr(
cfg.CONF.database, 'password') else None
cls.db_connection = db_setup(cfg.CONF.database.db_name,
cfg.CONF.database.host,
cfg.CONF.database.port,
username=username,
password=password,
ensure_indexes=False)
# Register sensors
register_sensors(packs_base_paths=[PACKS_BASE_PATH],
use_pack_cache=False)
# Create virtualenv for examples pack
virtualenv_path = '/tmp/virtualenvs/examples'
run_command(cmd=['rm', '-rf', virtualenv_path])
cmd = [
'virtualenv', '--system-site-packages', '--python', PYTHON_BINARY,
virtualenv_path
]
run_command(cmd=cmd)
def test_run_command_success(self):
# 0 exit code
exit_code, stdout, stderr, timed_out = run_command(
cmd='echo "test stdout" ; >&2 echo "test stderr"', shell=True)
self.assertEqual(exit_code, 0)
self.assertEqual(stdout.strip(), b"test stdout")
self.assertEqual(stderr.strip(), b"test stderr")
self.assertFalse(timed_out)
# non-zero exit code
exit_code, stdout, stderr, timed_out = run_command(
cmd='echo "test stdout" ; >&2 echo "test stderr" ; exit 5',
shell=True)
self.assertEqual(exit_code, 5)
self.assertEqual(stdout.strip(), b"test stdout")
self.assertEqual(stderr.strip(), b"test stderr")
self.assertFalse(timed_out)
# implicit non zero code (invalid command)
exit_code, stdout, stderr, timed_out = run_command(
cmd="foobarbarbazrbar", shell=True)
self.assertEqual(exit_code, 127)
self.assertEqual(stdout.strip(), b"")
self.assertTrue(b"foobarbarbazrbar: not found" in stderr.strip())
self.assertFalse(timed_out)
def setUpClass(cls):
super(SensorContainerTestCase, cls).setUpClass()
st2tests.config.parse_args()
username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None
cls.db_connection = db_setup(
cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
username=username, password=password, ensure_indexes=False)
# NOTE: We need to perform this patching because test fixtures are located outside of the
# packs base paths directory. This will never happen outside the context of test fixtures.
cfg.CONF.content.packs_base_paths = PACKS_BASE_PATH
# Register sensors
register_sensors(packs_base_paths=[PACKS_BASE_PATH], use_pack_cache=False)
# Create virtualenv for examples pack
virtualenv_path = '/tmp/virtualenvs/examples'
run_command(cmd=['rm', '-rf', virtualenv_path])
cmd = ['virtualenv', '--system-site-packages', '--python', PYTHON_BINARY, virtualenv_path]
run_command(cmd=cmd)
def setUpClass(cls):
super(SensorContainerTestCase, cls).setUpClass()
st2tests.config.parse_args()
username = cfg.CONF.database.username if hasattr(
cfg.CONF.database, 'username') else None
password = cfg.CONF.database.password if hasattr(
cfg.CONF.database, 'password') else None
cls.db_connection = db_setup(cfg.CONF.database.db_name,
cfg.CONF.database.host,
cfg.CONF.database.port,
username=username,
password=password,
ensure_indexes=False)
# NOTE: We need to perform this patching because test fixtures are located outside of the
# packs base paths directory. This will never happen outside the context of test fixtures.
cfg.CONF.content.packs_base_paths = PACKS_BASE_PATH
# Register sensors
register_sensors(packs_base_paths=[PACKS_BASE_PATH],
use_pack_cache=False)
# Create virtualenv for examples pack
virtualenv_path = '/tmp/virtualenvs/examples'
run_command(cmd=['rm', '-rf', virtualenv_path])
cmd = [
'virtualenv', '--system-site-packages', '--python', PYTHON_BINARY,
virtualenv_path
]
run_command(cmd=cmd)
def setUpClass(cls):
super(SensorContainerTestCase, cls).setUpClass()
return
st2tests.config.parse_args()
username = cfg.CONF.database.username if hasattr(
cfg.CONF.database, 'username') else None
password = cfg.CONF.database.password if hasattr(
cfg.CONF.database, 'password') else None
cls.db_connection = db_setup(cfg.CONF.database.db_name,
cfg.CONF.database.host,
cfg.CONF.database.port,
username=username,
password=password,
ensure_indexes=False)
# Register sensors
register_sensors(packs_base_paths=['/opt/stackstorm/packs'],
use_pack_cache=False)
# Create virtualenv for examples pack
virtualenv_path = '/opt/stackstorm/virtualenvs/examples'
cmd = ['virtualenv', '--system-site-packages', virtualenv_path]
run_command(cmd=cmd)
def _upload_file(self, local_path, base_path):
"""
Upload provided file to the remote server in a temporary directory.
:param local_path: Local path to the file to upload.
:type local_path: ``str``
:param base_path: Absolute base path for the share.
:type base_path: ``str``
"""
file_name = os.path.basename(local_path)
temporary_directory_name = str(uuid.uuid4())
command = 'mkdir %s' % (quote_windows(temporary_directory_name))
# 1. Create a temporary dir for out scripts (ignore errors if it already exists)
# Note: We don't necessary have access to $TEMP so we create a temporary directory for our
# us in the root of the share we are using and have access to
args = self._get_smbclient_command_args(host=self._host, username=self._username,
password=self._password, command=command,
share=self._share)
LOG.debug('Creating temp directory "%s"' % (temporary_directory_name))
exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
timeout=CREATE_DIRECTORY_TIMEOUT)
extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
LOG.debug('Directory created', extra=extra)
# 2. Upload file to temporary directory
remote_path = PATH_SEPARATOR.join([temporary_directory_name, file_name])
values = {
'local_path': quote_windows(local_path),
'remote_path': quote_windows(remote_path)
}
command = 'put %(local_path)s %(remote_path)s' % values
args = self._get_smbclient_command_args(host=self._host, username=self._username,
password=self._password, command=command,
share=self._share)
extra = {'local_path': local_path, 'remote_path': remote_path}
LOG.debug('Uploading file to "%s"' % (remote_path))
exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
timeout=UPLOAD_FILE_TIMEOUT)
extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
LOG.debug('File uploaded to "%s"' % (remote_path), extra=extra)
full_remote_file_path = base_path + '\\' + remote_path
full_temporary_directory_path = base_path + '\\' + temporary_directory_name
return full_remote_file_path, full_temporary_directory_path
def _upload_file(self, local_path, base_path):
"""
Upload provided file to the remote server in a temporary directory.
:param local_path: Local path to the file to upload.
:type local_path: ``str``
:param base_path: Absolute base path for the share.
:type base_path: ``str``
"""
file_name = os.path.basename(local_path)
temporary_directory_name = str(uuid.uuid4())
command = 'mkdir %s' % (quote_windows(temporary_directory_name))
# 1. Create a temporary dir for out scripts (ignore errors if it already exists)
# Note: We don't necessary have access to $TEMP so we create a temporary directory for our
# us in the root of the share we are using and have access to
args = self._get_smbclient_command_args(host=self._host, username=self._username,
password=self._password, command=command,
share=self._share)
LOG.debug('Creating temp directory "%s"' % (temporary_directory_name))
exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
timeout=CREATE_DIRECTORY_TIMEOUT)
extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
LOG.debug('Directory created', extra=extra)
# 2. Upload file to temporary directory
remote_path = PATH_SEPARATOR.join([temporary_directory_name, file_name])
values = {
'local_path': quote_windows(local_path),
'remote_path': quote_windows(remote_path)
}
command = 'put %(local_path)s %(remote_path)s' % values
args = self._get_smbclient_command_args(host=self._host, username=self._username,
password=self._password, command=command,
share=self._share)
extra = {'local_path': local_path, 'remote_path': remote_path}
LOG.debug('Uploading file to "%s"' % (remote_path))
exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
timeout=UPLOAD_FILE_TIMEOUT)
extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
LOG.debug('File uploaded to "%s"' % (remote_path), extra=extra)
full_remote_file_path = base_path + '\\' + remote_path
full_temporary_directory_path = base_path + '\\' + temporary_directory_name
return full_remote_file_path, full_temporary_directory_path
def _get_share_absolute_path(self, share):
"""
Retrieve full absolute path for a share with the provided name.
:param share: Share name.
:type share: ``str``
"""
command = 'net share %s' % (quote_windows(share))
args = self._get_winexe_command_args(host=self._host, username=self._username,
password=self._password,
command=command)
LOG.debug('Retrieving full absolute path for share "%s"' % (share))
exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
timeout=self._timeout)
if exit_code != 0:
msg = 'Failed to retrieve absolute path for share "%s"' % (share)
raise Exception(msg)
share_info = self._parse_share_information(stdout=stdout)
share_path = share_info.get('path', None)
if not share_path:
msg = 'Failed to retrieve absolute path for share "%s"' % (share)
raise Exception(msg)
return share_path
def _run_cli_command(self, command):
exit_code, stdout, stderr, timed_out = run_command(
cmd=command,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
timeout=self._timeout,
kill_func=kill_process)
error = None
if timed_out:
error = 'Action failed to complete in %s seconds' % self._timeout
exit_code = -9
succeeded = (exit_code == 0)
result = {
'failed': not succeeded,
'succeeded': succeeded,
'return_code': exit_code,
'stdout': stdout,
'stderr': stderr
}
if error:
result['error'] = error
status = LIVEACTION_STATUS_SUCCEEDED if succeeded else LIVEACTION_STATUS_FAILED
return result, status
def _get_share_absolute_path(self, share):
"""
Retrieve full absolute path for a share with the provided name.
:param share: Share name.
:type share: ``str``
"""
command = 'net share %s' % (quote_windows(share))
args = self._get_winexe_command_args(host=self._host,
username=self._username,
password=self._password,
command=command)
LOG.debug('Retrieving full absolute path for share "%s"' % (share))
exit_code, stdout, stderr, timed_out = run_command(
cmd=args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
timeout=self._timeout)
if exit_code != 0:
msg = 'Failed to retrieve absolute path for share "%s"' % (share)
raise Exception(msg)
share_info = self._parse_share_information(stdout=stdout)
share_path = share_info.get('path', None)
if not share_path:
msg = 'Failed to retrieve absolute path for share "%s"' % (share)
raise Exception(msg)
return share_path
def _run_script(self, script_path, arguments=None):
"""
:param script_path: Full path to the script on the remote server.
:type script_path: ``str``
:param arguments: The arguments to pass to the script.
:type arguments: ``str``
"""
if arguments is not None:
command = '%s %s %s' % (POWERSHELL_COMMAND,
quote_windows(script_path), arguments)
else:
command = '%s %s' % (POWERSHELL_COMMAND,
quote_windows(script_path))
args = self._get_winexe_command_args(host=self._host,
username=self._username,
password=self._password,
command=command)
LOG.debug('Running script "%s"' % (script_path))
# Note: We don't send anything over stdin, we just create an unused pipe
# to avoid some obscure failures
exit_code, stdout, stderr, timed_out = run_command(
cmd=args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
timeout=self._timeout)
extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
LOG.debug('Command returned', extra=extra)
return exit_code, stdout, stderr, timed_out
def _run_script(self, script_path, arguments=None):
"""
:param script_path: Full path to the script on the remote server.
:type script_path: ``str``
:param arguments: The arguments to pass to the script.
:type arguments: ``str``
"""
if arguments is not None:
command = '%s %s %s' % (POWERSHELL_COMMAND, quote_windows(script_path), arguments)
else:
command = '%s %s' % (POWERSHELL_COMMAND, quote_windows(script_path))
args = self._get_winexe_command_args(host=self._host, username=self._username,
password=self._password,
command=command)
LOG.debug('Running script "%s"' % (script_path))
# Note: We don't send anything over stdin, we just create an unused pipe
# to avoid some obscure failures
exit_code, stdout, stderr, timed_out = run_command(cmd=args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
timeout=self._timeout)
extra = {'exit_code': exit_code, 'stdout': stdout, 'stderr': stderr}
LOG.debug('Command returned', extra=extra)
return exit_code, stdout, stderr, timed_out
def apply_pack_owner_group(pack_path):
"""
Switch owner group of the pack / virtualenv directory to the configured
group.
NOTE: This requires sudo access.
"""
pack_group = utils.get_pack_group()
if pack_group:
LOG.debug('Changing owner group of "%s" directory to %s' % (pack_path, pack_group))
if SUDO_BINARY:
args = ['sudo', 'chgrp', '-R', pack_group, pack_path]
else:
# Environments where sudo is not available (e.g. docker)
args = ['chgrp', '-R', pack_group, pack_path]
exit_code, _, stderr, _ = shell.run_command(args)
if exit_code != 0:
# Non fatal, but we still log it
LOG.debug('Failed to change owner group on directory "%s" to "%s": %s' %
(pack_path, pack_group, stderr))
return True
def _run_cli_command(self, command):
exit_code, stdout, stderr, timed_out = run_command(
cmd=command, stdin=None,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
shell=True, timeout=self._timeout, kill_func=kill_process)
error = None
if timed_out:
error = 'Action failed to complete in %s seconds' % self._timeout
exit_code = -9
succeeded = (exit_code == 0)
result = {
'failed': not succeeded,
'succeeded': succeeded,
'return_code': exit_code,
'stdout': stdout,
'stderr': stderr
}
if error:
result['error'] = error
status = LIVEACTION_STATUS_SUCCEEDED if succeeded else LIVEACTION_STATUS_FAILED
self._log_action_completion(logger=LOG, result=result, status=status, exit_code=exit_code)
return result, status
def test_run_command_timeout_no_shell_no_custom_kill_func(self):
exit_code, stdout, stderr, timed_out = run_command(
cmd=["sleep", "1599"], preexec_func=os.setsid, timeout=1)
self.assertEqual(exit_code, TIMEOUT_EXIT_CODE)
self.assertEqual(stdout.strip(), b"")
self.assertEqual(stderr.strip(), b"")
self.assertTrue(timed_out)
# Verify there is no zombie process left laying around
self.assertNoStrayProcessesLeft("sleep 1599")
def setUpClass(cls):
super(SensorContainerTestCase, cls).setUpClass()
return
st2tests.config.parse_args()
username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None
cls.db_connection = db_setup(
cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
username=username, password=password, ensure_indexes=False)
# Register sensors
register_sensors(packs_base_paths=['/opt/stackstorm/packs'], use_pack_cache=False)
# Create virtualenv for examples pack
virtualenv_path = '/opt/stackstorm/virtualenvs/examples'
cmd = ['virtualenv', '--system-site-packages', virtualenv_path]
run_command(cmd=cmd)
def _delete_file(self, file_path):
command = 'rm %(file_path)s' % {'file_path': quote_windows(file_path)}
args = self._get_smbclient_command_args(host=self._host, username=self._username,
password=self._password, command=command,
share=self._share)
exit_code, _, _, _ = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
timeout=DELETE_FILE_TIMEOUT)
return exit_code == 0
def _apply_pack_permissions(self, pack_path):
"""
Will recursively apply permission 770 to pack and its contents.
"""
# 1. switch owner group to configured group
pack_group = utils.get_pack_group()
if pack_group:
shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])
# 2. Setup the right permissions and group ownership
# These mask is same as mode = 775
mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
os.chmod(pack_path, mode)
# Yuck! Since os.chmod does not support chmod -R walk manually.
for root, dirs, files in os.walk(pack_path):
for d in dirs:
os.chmod(os.path.join(root, d), mode)
for f in files:
os.chmod(os.path.join(root, f), mode)
def _apply_pack_permissions(self, pack_path):
"""
Will recursively apply permission 770 to pack and its contents.
"""
# 1. switch owner group to configured group
pack_group = utils.get_pack_group()
if pack_group:
shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])
# 2. Setup the right permissions and group ownership
# These mask is same as mode = 775
mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
os.chmod(pack_path, mode)
# Yuck! Since os.chmod does not support chmod -R walk manually.
for root, dirs, files in os.walk(pack_path):
for d in dirs:
os.chmod(os.path.join(root, d), mode)
for f in files:
os.chmod(os.path.join(root, f), mode)
def _delete_file(self, file_path):
command = 'rm %(file_path)s' % {'file_path': quote_windows(file_path)}
args = self._get_smbclient_command_args(host=self._host, username=self._username,
password=self._password, command=command,
share=self._share)
exit_code, _, _, _ = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
timeout=DELETE_FILE_TIMEOUT)
return exit_code == 0
def _delete_directory(self, directory_path):
command = 'rmdir %(directory_path)s' % {'directory_path': quote_windows(directory_path)}
args = self._get_smbclient_command_args(host=self._host, username=self._username,
password=self._password, command=command,
share=self._share)
LOG.debug('Removing directory "%s"' % (directory_path))
exit_code, _, _, _ = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
timeout=DELETE_DIRECTORY_TIMEOUT)
return exit_code == 0
def _delete_directory(self, directory_path):
command = 'rmdir %(directory_path)s' % {'directory_path': quote_windows(directory_path)}
args = self._get_smbclient_command_args(host=self._host, username=self._username,
password=self._password, command=command,
share=self._share)
LOG.debug('Removing directory "%s"' % (directory_path))
exit_code, _, _, _ = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
timeout=DELETE_DIRECTORY_TIMEOUT)
return exit_code == 0
def setUpClass(cls):
super(SensorContainerTestCase, cls).setUpClass()
st2tests.config.parse_args()
username = cfg.CONF.database.username if hasattr(cfg.CONF.database, 'username') else None
password = cfg.CONF.database.password if hasattr(cfg.CONF.database, 'password') else None
cls.db_connection = db_setup(
cfg.CONF.database.db_name, cfg.CONF.database.host, cfg.CONF.database.port,
username=username, password=password, ensure_indexes=False)
# Register sensors
register_sensors(packs_base_paths=[PACKS_BASE_PATH], use_pack_cache=False)
# Create virtualenv for examples pack
virtualenv_path = '/tmp/virtualenvs/examples'
run_command(cmd=['rm', '-rf', virtualenv_path])
cmd = ['virtualenv', '--system-site-packages', '--python', PYTHON_BINARY, virtualenv_path]
run_command(cmd=cmd)
def run(self, action_parameters):
pack = self.get_pack_name()
user = self.get_user()
serialized_parameters = json.dumps(
action_parameters) if action_parameters else ''
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
python_path = get_sandbox_python_binary_path(pack=pack)
if virtualenv_path and not os.path.isdir(virtualenv_path):
format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
raise Exception(msg)
if not self.entry_point:
raise Exception('Action "%s" is missing entry_point attribute' %
(self.action.name))
args = [
python_path, WRAPPER_SCRIPT_PATH,
'--pack=%s' % (pack),
'--file-path=%s' % (self.entry_point),
'--parameters=%s' % (serialized_parameters),
'--user=%s' % (user),
'--parent-args=%s' % (json.dumps(sys.argv[1:]))
]
# We need to ensure all the st2 dependencies are also available to the
# subprocess
env = os.environ.copy()
env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
env['PYTHONPATH'] = get_sandbox_python_path(
inherit_from_parent=True, inherit_parent_virtualenv=True)
# Include user provided environment variables (if any)
user_env_vars = self._get_env_vars()
env.update(user_env_vars)
# Include common st2 environment variables
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
datastore_env_vars = self._get_datastore_access_env_vars()
env.update(datastore_env_vars)
exit_code, stdout, stderr, timed_out = run_command(
cmd=args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
env=env,
timeout=self._timeout)
return self._get_output_values(exit_code, stdout, stderr, timed_out)
def assertNoStrayProcessesLeft(self, grep_string: str) -> None:
"""
Assert that there are no stray / zombie processes left with the provided command line
string.
"""
exit_code, stdout, stderr, timed_out = run_command(
cmd="ps aux | grep %s | grep -v grep" % (quote_unix(grep_string)),
shell=True,
)
if stdout.strip() != b"" and stderr.strip() != b"":
raise AssertionError(
"Expected no stray processes, but found Some. stdout: %s, stderr: %s"
% (stdout, stderr))
def run(self, action_parameters):
pack = self.get_pack_name()
user = self.get_user()
serialized_parameters = json.dumps(action_parameters) if action_parameters else ''
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
python_path = get_sandbox_python_binary_path(pack=pack)
if virtualenv_path and not os.path.isdir(virtualenv_path):
format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
raise Exception(msg)
if not self.entry_point:
raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))
args = [
python_path,
WRAPPER_SCRIPT_PATH,
'--pack=%s' % (pack),
'--file-path=%s' % (self.entry_point),
'--parameters=%s' % (serialized_parameters),
'--user=%s' % (user),
'--parent-args=%s' % (json.dumps(sys.argv[1:]))
]
# We need to ensure all the st2 dependencies are also available to the
# subprocess
env = os.environ.copy()
env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
env['PYTHONPATH'] = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=True)
# Include user provided environment variables (if any)
user_env_vars = self._get_env_vars()
env.update(user_env_vars)
# Include common st2 environment variables
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
datastore_env_vars = self._get_datastore_access_env_vars()
env.update(datastore_env_vars)
exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
env=env, timeout=self._timeout)
return self._get_output_values(exit_code, stdout, stderr, timed_out)
def run(self, action_parameters):
# Make sure the dependencies are available
self._verify_winexe_exists()
args = self._get_winexe_command_args(host=self._host,
username=self._username,
password=self._password,
command=self._command)
# Note: We don't send anything over stdin, we just create an unused pipe
# to avoid some obscure failures
exit_code, stdout, stderr, timed_out = run_command(
cmd=args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
timeout=self._timeout)
if timed_out:
error = 'Action failed to complete in %s seconds' % (self._timeout)
else:
error = None
if exit_code != 0:
error = self._parse_winexe_error(stdout=stdout, stderr=stderr)
result = stdout
output = {
'stdout': stdout,
'stderr': stderr,
'exit_code': exit_code,
'result': result
}
if error:
output['error'] = error
status = LIVEACTION_STATUS_SUCCEEDED if exit_code == 0 else LIVEACTION_STATUS_FAILED
self._log_action_completion(logger=LOG,
result=output,
status=status,
exit_code=exit_code)
return (status, output, None)
def test_run_command_timeout_shell_and_custom_kill_func(self):
# This test represents our local runner setup where we use a preexec_func + custom kill_func
# NOTE: When using shell=True. we should alaways use custom kill_func to ensure child shell
# processses are in fact killed as well.
exit_code, stdout, stderr, timed_out = run_command(
cmd='echo "pre sleep"; sleep 1589; echo "post sleep"',
preexec_func=os.setsid,
timeout=1,
kill_func=kill_process,
shell=True,
)
self.assertEqual(exit_code, TIMEOUT_EXIT_CODE)
self.assertEqual(stdout.strip(), b"pre sleep")
self.assertEqual(stderr.strip(), b"")
self.assertTrue(timed_out)
# Verify there is no zombie process left laying around
self.assertNoStrayProcessesLeft("sleep 1589")
def apply_pack_owner_group(pack_path):
"""
Switch owner group of the pack / virtualenv directory to the configured
group.
NOTE: This requires sudo access.
"""
pack_group = utils.get_pack_group()
if pack_group:
LOG.debug('Changing owner group of "%s" directory to %s' % (pack_path, pack_group))
exit_code, _, stderr, _ = shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])
if exit_code != 0:
# Non fatal, but we still log it
LOG.debug('Failed to change owner group on directory "%s" to "%s": %s' %
(pack_path, pack_group, stderr))
return True
def apply_pack_owner_group(pack_path):
"""
Switch owner group of the pack / virtualenv directory to the configured
group.
NOTE: This requires sudo access.
"""
pack_group = utils.get_pack_group()
if pack_group:
LOG.debug('Changing owner group of "%s" directory to %s' % (pack_path, pack_group))
exit_code, _, stderr, _ = shell.run_command(['sudo', 'chgrp', '-R', pack_group, pack_path])
if exit_code != 0:
# Non fatal, but we still log it
LOG.debug('Failed to change owner group on directory "%s" to "%s": %s' %
(pack_path, pack_group, stderr))
return True
def test_run_command_timeout_no_shell_no_custom_kill_func_and_read_funcs(
self):
def mock_read_stdout(process_stdout, stdout_buffer):
try:
stdout_buffer.write(process_stdout.readline())
except greenlet.GreenletExit:
pass
def mock_read_stderr(process_stderr, stderr_buffer):
try:
stderr_buffer.write(process_stderr.readline())
except greenlet.GreenletExit:
pass
read_stdout_buffer = BytesIO()
read_stderr_buffer = BytesIO()
script_path = os.path.abspath(
os.path.join(BASE_DIR,
"../fixtures/print_to_stdout_stderr_sleep.sh"))
exit_code, stdout, stderr, timed_out = run_command(
cmd=[script_path],
preexec_func=os.setsid,
timeout=3,
read_stdout_func=mock_read_stdout,
read_stderr_func=mock_read_stderr,
read_stdout_buffer=read_stdout_buffer,
read_stderr_buffer=read_stderr_buffer,
)
self.assertEqual(exit_code, TIMEOUT_EXIT_CODE)
self.assertEqual(stdout.strip(), b"pre sleep")
self.assertEqual(stderr.strip(), b"pre sleep stderr")
self.assertTrue(timed_out)
# Only initial produced line should be read
self.assertEqual(read_stdout_buffer.getvalue().strip(), b"pre sleep")
self.assertEqual(read_stderr_buffer.getvalue().strip(),
b"pre sleep stderr")
# Verify there is no zombie process left laying around
self.assertNoStrayProcessesLeft("print_to_stdout_stderr_sleep")
def run(self, action_parameters):
# Make sure the dependencies are available
self._verify_winexe_exists()
args = self._get_winexe_command_args(host=self._host, username=self._username,
password=self._password,
command=self._command)
# Note: We don't send anything over stdin, we just create an unused pipe
# to avoid some obscure failures
exit_code, stdout, stderr, timed_out = run_command(cmd=args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
timeout=self._timeout)
if timed_out:
error = 'Action failed to complete in %s seconds' % (self._timeout)
else:
error = None
if exit_code != 0:
error = self._parse_winexe_error(stdout=stdout, stderr=stderr)
result = stdout
output = {
'stdout': stdout,
'stderr': stderr,
'exit_code': exit_code,
'result': result
}
if error:
output['error'] = error
status = LIVEACTION_STATUS_SUCCEEDED if exit_code == 0 else LIVEACTION_STATUS_FAILED
self._log_action_completion(logger=LOG, result=output, status=status, exit_code=exit_code)
return (status, output, None)
def test_run_command_timeout_shell_and_custom_kill_func_and_read_funcs(
self):
# This test represents our local runner setup where we use a preexec_func + custom kill_func
# NOTE: When using shell=True. we should alaways use custom kill_func to ensure child shell
# processses are in fact killed as well.
def mock_read_stdout(process_stdout, stdout_buffer):
stdout_buffer.write(process_stdout.read())
def mock_read_stderr(process_stderr, stderr_buffer):
stderr_buffer.write(process_stderr.read())
read_stdout_buffer = BytesIO()
read_stderr_buffer = BytesIO()
exit_code, stdout, stderr, timed_out = run_command(
cmd=
'echo "pre sleep"; >&2 echo "pre sleep stderr" ; sleep 1589; echo "post sleep"',
preexec_func=os.setsid,
timeout=1,
kill_func=kill_process,
shell=True,
read_stdout_func=mock_read_stdout,
read_stderr_func=mock_read_stderr,
read_stdout_buffer=read_stdout_buffer,
read_stderr_buffer=read_stderr_buffer,
)
self.assertEqual(exit_code, TIMEOUT_EXIT_CODE)
self.assertEqual(stdout.strip(), b"pre sleep")
self.assertEqual(stderr.strip(), b"pre sleep stderr")
self.assertTrue(timed_out)
# Only initial produced line should be read
self.assertEqual(read_stdout_buffer.getvalue().strip(), b"pre sleep")
self.assertEqual(read_stderr_buffer.getvalue().strip(),
b"pre sleep stderr")
# Verify there is no zombie process left laying around
self.assertNoStrayProcessesLeft("sleep 1589")
def run(self, action_parameters):
LOG.debug('Running pythonrunner.')
LOG.debug('Getting pack name.')
pack = self.get_pack_ref()
LOG.debug('Getting user.')
user = self.get_user()
LOG.debug('Serializing parameters.')
serialized_parameters = json.dumps(action_parameters if action_parameters else {})
LOG.debug('Getting virtualenv_path.')
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
LOG.debug('Getting python path.')
if self._sandbox:
python_path = get_sandbox_python_binary_path(pack=pack)
else:
python_path = sys.executable
LOG.debug('Checking virtualenv path.')
if virtualenv_path and not os.path.isdir(virtualenv_path):
format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
LOG.error('virtualenv_path set but not a directory: %s', msg)
raise Exception(msg)
LOG.debug('Checking entry_point.')
if not self.entry_point:
LOG.error('Action "%s" is missing entry_point attribute' % (self.action.name))
raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))
# Note: We pass config as command line args so the actual wrapper process is standalone
# and doesn't need access to db
LOG.debug('Setting args.')
if self._use_parent_args:
parent_args = json.dumps(sys.argv[1:])
else:
parent_args = json.dumps([])
args = [
python_path,
'-u', # unbuffered mode so streaming mode works as expected
WRAPPER_SCRIPT_PATH,
'--pack=%s' % (pack),
'--file-path=%s' % (self.entry_point),
'--user=%s' % (user),
'--parent-args=%s' % (parent_args),
]
# If parameter size is larger than the maximum allowed by Linux kernel
# we need to swap to stdin to communicate parameters. This avoids a
# failure to fork the wrapper process when using large parameters.
stdin = None
stdin_params = None
if len(serialized_parameters) >= MAX_PARAM_LENGTH:
stdin = subprocess.PIPE
LOG.debug('Parameters are too big...changing to stdin')
stdin_params = '{"parameters": %s}\n' % (serialized_parameters)
args.append('--stdin-parameters')
else:
LOG.debug('Parameters are just right...adding them to arguments')
args.append('--parameters=%s' % (serialized_parameters))
if self._config:
args.append('--config=%s' % (json.dumps(self._config)))
if self._log_level != PYTHON_RUNNER_DEFAULT_LOG_LEVEL:
# We only pass --log-level parameter if non default log level value is specified
args.append('--log-level=%s' % (self._log_level))
# We need to ensure all the st2 dependencies are also available to the
# subprocess
LOG.debug('Setting env.')
env = os.environ.copy()
env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
sandbox_python_path = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=True)
if self._enable_common_pack_libs:
try:
pack_common_libs_path = self._get_pack_common_libs_path(pack_ref=pack)
except Exception as e:
LOG.debug('Failed to retrieve pack common lib path: %s' % (str(e)))
print(e)
# There is no MongoDB connection available in Lambda and pack common lib
# functionality is not also mandatory for Lambda so we simply ignore those errors.
# Note: We should eventually refactor this code to make runner standalone and not
# depend on a db connection (as it was in the past) - this param should be passed
# to the runner by the action runner container
pack_common_libs_path = None
else:
pack_common_libs_path = None
# Remove leading : (if any)
if sandbox_python_path.startswith(':'):
sandbox_python_path = sandbox_python_path[1:]
if self._enable_common_pack_libs and pack_common_libs_path:
env['PYTHONPATH'] = pack_common_libs_path + ':' + sandbox_python_path
else:
env['PYTHONPATH'] = sandbox_python_path
# Include user provided environment variables (if any)
user_env_vars = self._get_env_vars()
env.update(user_env_vars)
# Include common st2 environment variables
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
datastore_env_vars = self._get_datastore_access_env_vars()
env.update(datastore_env_vars)
stdout = StringIO()
stderr = StringIO()
store_execution_stdout_line = functools.partial(store_execution_output_data,
output_type='stdout')
store_execution_stderr_line = functools.partial(store_execution_output_data,
output_type='stderr')
read_and_store_stdout = make_read_and_store_stream_func(execution_db=self.execution,
action_db=self.action, store_data_func=store_execution_stdout_line)
read_and_store_stderr = make_read_and_store_stream_func(execution_db=self.execution,
action_db=self.action, store_data_func=store_execution_stderr_line)
command_string = list2cmdline(args)
if stdin_params:
command_string = 'echo %s | %s' % (quote_unix(stdin_params), command_string)
LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'],
command_string))
exit_code, stdout, stderr, timed_out = run_command(cmd=args,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
env=env,
timeout=self._timeout,
read_stdout_func=read_and_store_stdout,
read_stderr_func=read_and_store_stderr,
read_stdout_buffer=stdout,
read_stderr_buffer=stderr,
stdin_value=stdin_params)
LOG.debug('Returning values: %s, %s, %s, %s', exit_code, stdout, stderr, timed_out)
LOG.debug('Returning.')
return self._get_output_values(exit_code, stdout, stderr, timed_out)
def _run(self, action):
env_vars = self._env
if not self.entry_point:
script_action = False
else:
script_action = True
args = action.get_full_command_string()
sanitized_args = action.get_sanitized_full_command_string()
# For consistency with the old Fabric based runner, make sure the file is executable
if script_action:
script_local_path_abs = self.entry_point
args = 'chmod +x %s ; %s' % (script_local_path_abs, args)
sanitized_args = 'chmod +x %s ; %s' % (script_local_path_abs, sanitized_args)
env = os.environ.copy()
# Include user provided env vars (if any)
env.update(env_vars)
# Include common st2 env vars
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
LOG.info('Executing action via LocalRunner: %s', self.runner_id)
LOG.info('[Action info] name: %s, Id: %s, command: %s, user: %s, sudo: %s' %
(action.name, action.action_exec_id, sanitized_args, action.user, action.sudo))
stdout = StringIO()
stderr = StringIO()
store_execution_stdout_line = functools.partial(store_execution_output_data,
output_type='stdout')
store_execution_stderr_line = functools.partial(store_execution_output_data,
output_type='stderr')
read_and_store_stdout = make_read_and_store_stream_func(execution_db=self.execution,
action_db=self.action, store_data_func=store_execution_stdout_line)
read_and_store_stderr = make_read_and_store_stream_func(execution_db=self.execution,
action_db=self.action, store_data_func=store_execution_stderr_line)
# If sudo password is provided, pass it to the subprocess via stdin>
# Note: We don't need to explicitly escape the argument because we pass command as a list
# to subprocess.Popen and all the arguments are escaped by the function.
if self._sudo_password:
LOG.debug('Supplying sudo password via stdin')
echo_process = subprocess.Popen(['echo', self._sudo_password + '\n'],
stdout=subprocess.PIPE)
stdin = echo_process.stdout
else:
stdin = None
# Make sure os.setsid is called on each spawned process so that all processes
# are in the same group.
# Process is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
# bash means that multiple independent processes are spawned without them being
# children of the process we have access to and this requires use of pkill.
# Ideally os.killpg should have done the trick but for some reason that failed.
# Note: pkill will set the returncode to 143 so we don't need to explicitly set
# it to some non-zero value.
exit_code, stdout, stderr, timed_out = shell.run_command(cmd=args,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=self._cwd,
env=env,
timeout=self._timeout,
preexec_func=os.setsid,
kill_func=kill_process,
read_stdout_func=read_and_store_stdout,
read_stderr_func=read_and_store_stderr,
read_stdout_buffer=stdout,
read_stderr_buffer=stderr)
error = None
if timed_out:
error = 'Action failed to complete in %s seconds' % (self._timeout)
exit_code = -1 * exit_code_constants.SIGKILL_EXIT_CODE
# Detect if user provided an invalid sudo password or sudo is not configured for that user
if self._sudo_password:
if re.search(r'sudo: \d+ incorrect password attempts', stderr):
match = re.search(r'\[sudo\] password for (.+?)\:', stderr)
if match:
username = match.groups()[0]
else:
username = '******'
error = ('Invalid sudo password provided or sudo is not configured for this user '
'(%s)' % (username))
exit_code = -1
succeeded = (exit_code == exit_code_constants.SUCCESS_EXIT_CODE)
result = {
'failed': not succeeded,
'succeeded': succeeded,
'return_code': exit_code,
'stdout': strip_shell_chars(stdout),
'stderr': strip_shell_chars(stderr)
}
if error:
result['error'] = error
status = PROC_EXIT_CODE_TO_LIVEACTION_STATUS_MAP.get(
str(exit_code),
action_constants.LIVEACTION_STATUS_FAILED
)
return (status, jsonify.json_loads(result, BaseLocalShellRunner.KEYS_TO_TRANSFORM), None)
def run(self, action_parameters):
LOG.debug('Running pythonrunner.')
LOG.debug('Getting pack name.')
pack = self.get_pack_ref()
LOG.debug('Getting user.')
user = self.get_user()
LOG.debug('Serializing parameters.')
serialized_parameters = json.dumps(
action_parameters) if action_parameters else ''
LOG.debug('Getting virtualenv_path.')
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
LOG.debug('Getting python path.')
if self._sandbox:
python_path = get_sandbox_python_binary_path(pack=pack)
else:
python_path = sys.executable
LOG.debug('Checking virtualenv path.')
if virtualenv_path and not os.path.isdir(virtualenv_path):
format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
LOG.error('virtualenv_path set but not a directory: %s', msg)
raise Exception(msg)
LOG.debug('Checking entry_point.')
if not self.entry_point:
LOG.error('Action "%s" is missing entry_point attribute' %
(self.action.name))
raise Exception('Action "%s" is missing entry_point attribute' %
(self.action.name))
# Note: We pass config as command line args so the actual wrapper process is standalone
# and doesn't need access to db
LOG.debug('Setting args.')
args = [
python_path,
'-u', # unbuffered mode so streaming mode works as expected
WRAPPER_SCRIPT_PATH,
'--pack=%s' % (pack),
'--file-path=%s' % (self.entry_point),
'--parameters=%s' % (serialized_parameters),
'--user=%s' % (user),
'--parent-args=%s' % (json.dumps(sys.argv[1:])),
]
if self._config:
args.append('--config=%s' % (json.dumps(self._config)))
if self._log_level != 'debug':
# We only pass --log-level parameter if non default log level value is specified
args.append('--log-level=%s' % (self._log_level))
# We need to ensure all the st2 dependencies are also available to the
# subprocess
LOG.debug('Setting env.')
env = os.environ.copy()
env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
sandbox_python_path = get_sandbox_python_path(
inherit_from_parent=True, inherit_parent_virtualenv=True)
if self._enable_common_pack_libs:
try:
pack_common_libs_path = get_pack_common_libs_path_for_pack_ref(
pack_ref=pack)
except Exception:
# There is no MongoDB connection available in Lambda and pack common lib
# functionality is not also mandatory for Lambda so we simply ignore those errors.
# Note: We should eventually refactor this code to make runner standalone and not
# depend on a db connection (as it was in the past) - this param should be passed
# to the runner by the action runner container
pack_common_libs_path = None
else:
pack_common_libs_path = None
if self._enable_common_pack_libs and pack_common_libs_path:
env['PYTHONPATH'] = pack_common_libs_path + ':' + sandbox_python_path
else:
env['PYTHONPATH'] = sandbox_python_path
# Include user provided environment variables (if any)
user_env_vars = self._get_env_vars()
env.update(user_env_vars)
# Include common st2 environment variables
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
datastore_env_vars = self._get_datastore_access_env_vars()
env.update(datastore_env_vars)
stdout = StringIO()
stderr = StringIO()
store_execution_stdout_line = functools.partial(
store_execution_output_data, output_type='stdout')
store_execution_stderr_line = functools.partial(
store_execution_output_data, output_type='stderr')
read_and_store_stdout = make_read_and_store_stream_func(
execution_db=self.execution,
action_db=self.action,
store_data_func=store_execution_stdout_line)
read_and_store_stderr = make_read_and_store_stream_func(
execution_db=self.execution,
action_db=self.action,
store_data_func=store_execution_stderr_line)
command_string = list2cmdline(args)
LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' %
(env['PATH'], env['PYTHONPATH'], command_string))
exit_code, stdout, stderr, timed_out = run_command(
cmd=args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
env=env,
timeout=self._timeout,
read_stdout_func=read_and_store_stdout,
read_stderr_func=read_and_store_stderr,
read_stdout_buffer=stdout,
read_stderr_buffer=stderr)
LOG.debug('Returning values: %s, %s, %s, %s' %
(exit_code, stdout, stderr, timed_out))
LOG.debug('Returning.')
return self._get_output_values(exit_code, stdout, stderr, timed_out)
def _run(self, action):
env_vars = self._env
if not self.entry_point:
script_action = False
else:
script_action = True
args = action.get_full_command_string()
sanitized_args = action.get_sanitized_full_command_string()
# For consistency with the old Fabric based runner, make sure the file is executable
if script_action:
script_local_path_abs = self.entry_point
args = "chmod +x %s ; %s" % (script_local_path_abs, args)
sanitized_args = "chmod +x %s ; %s" % (
script_local_path_abs,
sanitized_args,
)
env = os.environ.copy()
# Include user provided env vars (if any)
env.update(env_vars)
# Include common st2 env vars
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
LOG.info("Executing action via LocalRunner: %s", self.runner_id)
LOG.info(
"[Action info] name: %s, Id: %s, command: %s, user: %s, sudo: %s"
% (
action.name,
action.action_exec_id,
sanitized_args,
action.user,
action.sudo,
)
)
stdout = StringIO()
stderr = StringIO()
store_execution_stdout_line = functools.partial(
store_execution_output_data, output_type="stdout"
)
store_execution_stderr_line = functools.partial(
store_execution_output_data, output_type="stderr"
)
read_and_store_stdout = make_read_and_store_stream_func(
execution_db=self.execution,
action_db=self.action,
store_data_func=store_execution_stdout_line,
)
read_and_store_stderr = make_read_and_store_stream_func(
execution_db=self.execution,
action_db=self.action,
store_data_func=store_execution_stderr_line,
)
subprocess = concurrency.get_subprocess_module()
# If sudo password is provided, pass it to the subprocess via stdin>
# Note: We don't need to explicitly escape the argument because we pass command as a list
# to subprocess.Popen and all the arguments are escaped by the function.
if self._sudo_password:
LOG.debug("Supplying sudo password via stdin")
echo_process = concurrency.subprocess_popen(
["echo", self._sudo_password + "\n"], stdout=subprocess.PIPE
)
stdin = echo_process.stdout
else:
stdin = None
# Make sure os.setsid is called on each spawned process so that all processes
# are in the same group.
# Process is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
# bash means that multiple independent processes are spawned without them being
# children of the process we have access to and this requires use of pkill.
# Ideally os.killpg should have done the trick but for some reason that failed.
# Note: pkill will set the returncode to 143 so we don't need to explicitly set
# it to some non-zero value.
exit_code, stdout, stderr, timed_out = shell.run_command(
cmd=args,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=self._cwd,
env=env,
timeout=self._timeout,
preexec_func=os.setsid,
kill_func=kill_process,
read_stdout_func=read_and_store_stdout,
read_stderr_func=read_and_store_stderr,
read_stdout_buffer=stdout,
read_stderr_buffer=stderr,
)
error = None
if timed_out:
error = "Action failed to complete in %s seconds" % (self._timeout)
exit_code = -1 * exit_code_constants.SIGKILL_EXIT_CODE
# Detect if user provided an invalid sudo password or sudo is not configured for that user
if self._sudo_password:
if re.search(r"sudo: \d+ incorrect password attempts", stderr):
match = re.search(r"\[sudo\] password for (.+?)\:", stderr)
if match:
username = match.groups()[0]
else:
username = "******"
error = (
"Invalid sudo password provided or sudo is not configured for this user "
"(%s)" % (username)
)
exit_code = -1
succeeded = exit_code == exit_code_constants.SUCCESS_EXIT_CODE
result = {
"failed": not succeeded,
"succeeded": succeeded,
"return_code": exit_code,
"stdout": strip_shell_chars(stdout),
"stderr": strip_shell_chars(stderr),
}
if error:
result["error"] = error
status = PROC_EXIT_CODE_TO_LIVEACTION_STATUS_MAP.get(
str(exit_code), action_constants.LIVEACTION_STATUS_FAILED
)
return (
status,
jsonify.json_loads(result, BaseLocalShellRunner.KEYS_TO_TRANSFORM),
None,
)
def run(self, action_parameters):
pack = self.get_pack_name()
serialized_parameters = json.dumps(action_parameters) if action_parameters else ''
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
python_path = get_sandbox_python_binary_path(pack=pack)
if virtualenv_path and not os.path.isdir(virtualenv_path):
format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
raise Exception(msg)
if not self.entry_point:
raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))
args = [
python_path,
WRAPPER_SCRIPT_PATH,
'--pack=%s' % (pack),
'--file-path=%s' % (self.entry_point),
'--parameters=%s' % (serialized_parameters),
'--parent-args=%s' % (json.dumps(sys.argv[1:]))
]
# We need to ensure all the st2 dependencies are also available to the
# subprocess
env = os.environ.copy()
env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
env['PYTHONPATH'] = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=True)
# Include user provided environment variables (if any)
user_env_vars = self._get_env_vars()
env.update(user_env_vars)
# Include common st2 environment variables
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
env=env, timeout=self._timeout)
if timed_out:
error = 'Action failed to complete in %s seconds' % (self._timeout)
else:
error = None
if ACTION_OUTPUT_RESULT_DELIMITER in stdout:
split = stdout.split(ACTION_OUTPUT_RESULT_DELIMITER)
assert len(split) == 3
result = split[1].strip()
stdout = split[0] + split[2]
else:
result = None
try:
result = json.loads(result)
except:
pass
output = {
'stdout': stdout,
'stderr': stderr,
'exit_code': exit_code,
'result': result
}
if error:
output['error'] = error
if exit_code == 0:
status = LIVEACTION_STATUS_SUCCEEDED
elif timed_out:
status = LIVEACTION_STATUS_TIMED_OUT
else:
status = LIVEACTION_STATUS_FAILED
return (status, output, None)
def create_git_worktree(self, content_version):
"""
Create a git worktree for the provided git content version.
:return: Path to the created git worktree directory.
:rtype: ``str``
"""
pack_name = self.get_pack_name()
pack_directory = get_pack_directory(pack_name=pack_name)
worktree_path = tempfile.mkdtemp(prefix=self.WORKTREE_DIRECTORY_PREFIX)
# Set class variables
self.git_worktree_revision = content_version
self.git_worktree_path = worktree_path
extra = {
'pack_name': pack_name,
'pack_directory': pack_directory,
'content_version': content_version,
'worktree_path': worktree_path
}
if not os.path.isdir(pack_directory):
msg = (
'Failed to create git worktree for pack "%s". Pack directory "%s" doesn\'t '
'exist.' % (pack_name, pack_directory))
raise ValueError(msg)
args = [
'git', '-C', pack_directory, 'worktree', 'add', worktree_path,
content_version
]
cmd = list2cmdline(args)
LOG.debug(
'Creating git worktree for pack "%s", content version "%s" and execution '
'id "%s" in "%s"' %
(pack_name, content_version, self.execution_id, worktree_path),
extra=extra)
LOG.debug('Command: %s' % (cmd))
exit_code, stdout, stderr, timed_out = run_command(
cmd=cmd,
cwd=pack_directory,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if exit_code != 0:
self._handle_git_worktree_error(pack_name=pack_name,
pack_directory=pack_directory,
content_version=content_version,
exit_code=exit_code,
stdout=stdout,
stderr=stderr)
else:
LOG.debug('Git worktree created in "%s"' % (worktree_path),
extra=extra)
# Make sure system / action runner user can access that directory
args = ['chmod', '777', worktree_path]
cmd = list2cmdline(args)
run_command(cmd=cmd, shell=True)
return worktree_path
def run(self, action_parameters):
pack = self.action.pack if self.action else DEFAULT_PACK_NAME
serialized_parameters = json.dumps(action_parameters) if action_parameters else ''
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
python_path = get_sandbox_python_binary_path(pack=pack)
if virtualenv_path and not os.path.isdir(virtualenv_path):
msg = PACK_VIRTUALENV_DOESNT_EXIST % (pack, pack)
raise Exception(msg)
if not self.entry_point:
raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))
args = [
python_path,
WRAPPER_SCRIPT_PATH,
'--pack=%s' % (pack),
'--file-path=%s' % (self.entry_point),
'--parameters=%s' % (serialized_parameters),
'--parent-args=%s' % (json.dumps(sys.argv[1:]))
]
# We need to ensure all the st2 dependencies are also available to the
# subprocess
env = os.environ.copy()
env['PYTHONPATH'] = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=True)
# Include user provided environment variables (if any)
user_env_vars = self._get_env_vars()
env.update(user_env_vars)
exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
env=env, timeout=self._timeout)
if timed_out:
error = 'Action failed to complete in %s seconds' % (self._timeout)
else:
error = None
if ACTION_OUTPUT_RESULT_DELIMITER in stdout:
split = stdout.split(ACTION_OUTPUT_RESULT_DELIMITER)
assert len(split) == 3
result = split[1].strip()
stdout = split[0] + split[2]
else:
result = None
try:
result = json.loads(result)
except:
pass
output = {
'stdout': stdout,
'stderr': stderr,
'exit_code': exit_code,
'result': result
}
if error:
output['error'] = error
status = LIVEACTION_STATUS_SUCCEEDED if exit_code == 0 else LIVEACTION_STATUS_FAILED
self._log_action_completion(logger=LOG, result=output, status=status, exit_code=exit_code)
return (status, output, None)
def run(self, action_parameters):
env_vars = self._env
if not self.entry_point:
script_action = False
command = self.runner_parameters.get(RUNNER_COMMAND, None)
action = ShellCommandAction(name=self.action_name,
action_exec_id=str(self.liveaction_id),
command=command,
user=self._user,
env_vars=env_vars,
sudo=self._sudo,
timeout=self._timeout,
sudo_password=self._sudo_password)
else:
script_action = True
script_local_path_abs = self.entry_point
positional_args, named_args = self._get_script_args(action_parameters)
named_args = self._transform_named_args(named_args)
action = ShellScriptAction(name=self.action_name,
action_exec_id=str(self.liveaction_id),
script_local_path_abs=script_local_path_abs,
named_args=named_args,
positional_args=positional_args,
user=self._user,
env_vars=env_vars,
sudo=self._sudo,
timeout=self._timeout,
cwd=self._cwd,
sudo_password=self._sudo_password)
args = action.get_full_command_string()
sanitized_args = action.get_sanitized_full_command_string()
# For consistency with the old Fabric based runner, make sure the file is executable
if script_action:
args = 'chmod +x %s ; %s' % (script_local_path_abs, args)
sanitized_args = 'chmod +x %s ; %s' % (script_local_path_abs, sanitized_args)
env = os.environ.copy()
# Include user provided env vars (if any)
env.update(env_vars)
# Include common st2 env vars
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
LOG.info('Executing action via LocalRunner: %s', self.runner_id)
LOG.info('[Action info] name: %s, Id: %s, command: %s, user: %s, sudo: %s' %
(action.name, action.action_exec_id, sanitized_args, action.user, action.sudo))
stdout = StringIO()
stderr = StringIO()
store_execution_stdout_line = functools.partial(store_execution_output_data,
output_type='stdout')
store_execution_stderr_line = functools.partial(store_execution_output_data,
output_type='stderr')
read_and_store_stdout = make_read_and_store_stream_func(execution_db=self.execution,
action_db=self.action, store_data_func=store_execution_stdout_line)
read_and_store_stderr = make_read_and_store_stream_func(execution_db=self.execution,
action_db=self.action, store_data_func=store_execution_stderr_line)
# If sudo password is provided, pass it to the subprocess via stdin>
# Note: We don't need to explicitly escape the argument because we pass command as a list
# to subprocess.Popen and all the arguments are escaped by the function.
if self._sudo_password:
LOG.debug('Supplying sudo password via stdin')
echo_process = subprocess.Popen(['echo', self._sudo_password + '\n'],
stdout=subprocess.PIPE)
stdin = echo_process.stdout
else:
stdin = None
# Make sure os.setsid is called on each spawned process so that all processes
# are in the same group.
# Process is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
# bash means that multiple independent processes are spawned without them being
# children of the process we have access to and this requires use of pkill.
# Ideally os.killpg should have done the trick but for some reason that failed.
# Note: pkill will set the returncode to 143 so we don't need to explicitly set
# it to some non-zero value.
exit_code, stdout, stderr, timed_out = shell.run_command(cmd=args,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=self._cwd,
env=env,
timeout=self._timeout,
preexec_func=os.setsid,
kill_func=kill_process,
read_stdout_func=read_and_store_stdout,
read_stderr_func=read_and_store_stderr,
read_stdout_buffer=stdout,
read_stderr_buffer=stderr)
error = None
if timed_out:
error = 'Action failed to complete in %s seconds' % (self._timeout)
exit_code = -1 * exit_code_constants.SIGKILL_EXIT_CODE
# Detect if user provided an invalid sudo password or sudo is not configured for that user
if self._sudo_password:
if re.search('sudo: \d+ incorrect password attempts', stderr):
match = re.search('\[sudo\] password for (.+?)\:', stderr)
if match:
username = match.groups()[0]
else:
username = '******'
error = ('Invalid sudo password provided or sudo is not configured for this user '
'(%s)' % (username))
exit_code = -1
succeeded = (exit_code == exit_code_constants.SUCCESS_EXIT_CODE)
result = {
'failed': not succeeded,
'succeeded': succeeded,
'return_code': exit_code,
'stdout': strip_shell_chars(stdout),
'stderr': strip_shell_chars(stderr)
}
if error:
result['error'] = error
status = PROC_EXIT_CODE_TO_LIVEACTION_STATUS_MAP.get(
str(exit_code),
action_constants.LIVEACTION_STATUS_FAILED
)
return (status, jsonify.json_loads(result, LocalShellRunner.KEYS_TO_TRANSFORM), None)
def run(self, action_parameters):
LOG.debug('Running pythonrunner.')
LOG.debug('Getting pack name.')
pack = self.get_pack_name()
LOG.debug('Getting user.')
user = self.get_user()
LOG.debug('Serializing parameters.')
serialized_parameters = json.dumps(action_parameters) if action_parameters else ''
LOG.debug('Getting virtualenv_path.')
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
LOG.debug('Getting python path.')
python_path = get_sandbox_python_binary_path(pack=pack)
LOG.debug('Checking virtualenv path.')
if virtualenv_path and not os.path.isdir(virtualenv_path):
format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
LOG.error('virtualenv_path set but not a directory: %s', msg)
raise Exception(msg)
LOG.debug('Checking entry_point.')
if not self.entry_point:
LOG.error('Action "%s" is missing entry_point attribute' % (self.action.name))
raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))
LOG.debug('Setting args.')
args = [
python_path,
WRAPPER_SCRIPT_PATH,
'--pack=%s' % (pack),
'--file-path=%s' % (self.entry_point),
'--parameters=%s' % (serialized_parameters),
'--user=%s' % (user),
'--parent-args=%s' % (json.dumps(sys.argv[1:]))
]
# We need to ensure all the st2 dependencies are also available to the
# subprocess
LOG.debug('Setting env.')
env = os.environ.copy()
env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
env['PYTHONPATH'] = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=True)
# Include user provided environment variables (if any)
user_env_vars = self._get_env_vars()
env.update(user_env_vars)
# Include common st2 environment variables
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
datastore_env_vars = self._get_datastore_access_env_vars()
env.update(datastore_env_vars)
command_string = list2cmdline(args)
LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'],
command_string))
exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
env=env, timeout=self._timeout)
LOG.debug('Returning values: %s, %s, %s, %s' % (exit_code, stdout, stderr, timed_out))
LOG.debug('Returning.')
return self._get_output_values(exit_code, stdout, stderr, timed_out)
def create_git_worktree(self, content_version):
"""
Create a git worktree for the provided git content version.
:return: Path to the created git worktree directory.
:rtype: ``str``
"""
pack_name = self.get_pack_name()
pack_directory = get_pack_directory(pack_name=pack_name)
worktree_path = tempfile.mkdtemp(prefix=self.WORKTREE_DIRECTORY_PREFIX)
# Set class variables
self.git_worktree_revision = content_version
self.git_worktree_path = worktree_path
extra = {
'pack_name': pack_name,
'pack_directory': pack_directory,
'content_version': content_version,
'worktree_path': worktree_path
}
if not os.path.isdir(pack_directory):
msg = ('Failed to create git worktree for pack "%s". Pack directory "%s" doesn\'t '
'exist.' % (pack_name, pack_directory))
raise ValueError(msg)
args = [
'git',
'-C',
pack_directory,
'worktree',
'add',
worktree_path,
content_version
]
cmd = list2cmdline(args)
LOG.debug('Creating git worktree for pack "%s", content version "%s" and execution '
'id "%s" in "%s"' % (pack_name, content_version, self.execution_id,
worktree_path), extra=extra)
LOG.debug('Command: %s' % (cmd))
exit_code, stdout, stderr, timed_out = run_command(cmd=cmd,
cwd=pack_directory,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if exit_code != 0:
self._handle_git_worktree_error(pack_name=pack_name, pack_directory=pack_directory,
content_version=content_version,
exit_code=exit_code, stdout=stdout, stderr=stderr)
else:
LOG.debug('Git worktree created in "%s"' % (worktree_path), extra=extra)
# Make sure system / action runner user can access that directory
args = [
'chmod',
'777',
worktree_path
]
cmd = list2cmdline(args)
run_command(cmd=cmd, shell=True)
return worktree_path
def run(self, action_parameters):
env_vars = self._env
if not self.entry_point:
script_action = False
command = self.runner_parameters.get(RUNNER_COMMAND, None)
action = ShellCommandAction(name=self.action_name,
action_exec_id=str(self.liveaction_id),
command=command,
user=self._user,
env_vars=env_vars,
sudo=self._sudo,
timeout=self._timeout)
else:
script_action = True
script_local_path_abs = self.entry_point
positional_args, named_args = self._get_script_args(
action_parameters)
named_args = self._transform_named_args(named_args)
action = ShellScriptAction(
name=self.action_name,
action_exec_id=str(self.liveaction_id),
script_local_path_abs=script_local_path_abs,
named_args=named_args,
positional_args=positional_args,
user=self._user,
env_vars=env_vars,
sudo=self._sudo,
timeout=self._timeout,
cwd=self._cwd)
args = action.get_full_command_string()
# For consistency with the old Fabric based runner, make sure the file is executable
if script_action:
args = 'chmod +x %s ; %s' % (script_local_path_abs, args)
env = os.environ.copy()
# Include user provided env vars (if any)
env.update(env_vars)
# Include common st2 env vars
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
LOG.info('Executing action via LocalRunner: %s', self.runner_id)
LOG.info(
'[Action info] name: %s, Id: %s, command: %s, user: %s, sudo: %s' %
(action.name, action.action_exec_id, args, action.user,
action.sudo))
# Make sure os.setsid is called on each spawned process so that all processes
# are in the same group.
# Process is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
# bash means that multiple independent processes are spawned without them being
# children of the process we have access to and this requires use of pkill.
# Ideally os.killpg should have done the trick but for some reason that failed.
# Note: pkill will set the returncode to 143 so we don't need to explicitly set
# it to some non-zero value.
exit_code, stdout, stderr, timed_out = run_command(
cmd=args,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=self._cwd,
env=env,
timeout=self._timeout,
preexec_func=os.setsid,
kill_func=kill_process)
error = None
if timed_out:
error = 'Action failed to complete in %s seconds' % (self._timeout)
exit_code = -9
succeeded = (exit_code == 0)
result = {
'failed': not succeeded,
'succeeded': succeeded,
'return_code': exit_code,
'stdout': strip_shell_chars(stdout),
'stderr': strip_shell_chars(stderr)
}
if error:
result['error'] = error
if exit_code == 0:
status = LIVEACTION_STATUS_SUCCEEDED
elif timed_out:
status = LIVEACTION_STATUS_TIMED_OUT
else:
status = LIVEACTION_STATUS_FAILED
return (status,
jsonify.json_loads(result,
LocalShellRunner.KEYS_TO_TRANSFORM), None)
def run(self, action_parameters):
env_vars = self._env
if not self.entry_point:
script_action = False
command = self.runner_parameters.get(RUNNER_COMMAND, None)
action = ShellCommandAction(name=self.action_name,
action_exec_id=str(self.liveaction_id),
command=command,
user=self._user,
env_vars=env_vars,
sudo=self._sudo,
timeout=self._timeout)
else:
script_action = True
script_local_path_abs = self.entry_point
positional_args, named_args = self._get_script_args(action_parameters)
named_args = self._transform_named_args(named_args)
action = ShellScriptAction(name=self.action_name,
action_exec_id=str(self.liveaction_id),
script_local_path_abs=script_local_path_abs,
named_args=named_args,
positional_args=positional_args,
user=self._user,
env_vars=env_vars,
sudo=self._sudo,
timeout=self._timeout,
cwd=self._cwd)
args = action.get_full_command_string()
# For consistency with the old Fabric based runner, make sure the file is executable
if script_action:
args = 'chmod +x %s ; %s' % (script_local_path_abs, args)
env = os.environ.copy()
# Include user provided env vars (if any)
env.update(env_vars)
# Include common st2 env vars
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
LOG.info('Executing action via LocalRunner: %s', self.runner_id)
LOG.info('[Action info] name: %s, Id: %s, command: %s, user: %s, sudo: %s' %
(action.name, action.action_exec_id, args, action.user, action.sudo))
# Make sure os.setsid is called on each spawned process so that all processes
# are in the same group.
# Process is started as sudo -u {{system_user}} -- bash -c {{command}}. Introduction of the
# bash means that multiple independent processes are spawned without them being
# children of the process we have access to and this requires use of pkill.
# Ideally os.killpg should have done the trick but for some reason that failed.
# Note: pkill will set the returncode to 143 so we don't need to explicitly set
# it to some non-zero value.
exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
cwd=self._cwd,
env=env,
timeout=self._timeout,
preexec_func=os.setsid,
kill_func=kill_process)
error = None
if timed_out:
error = 'Action failed to complete in %s seconds' % (self._timeout)
exit_code = -9
succeeded = (exit_code == 0)
result = {
'failed': not succeeded,
'succeeded': succeeded,
'return_code': exit_code,
'stdout': strip_last_newline_char(stdout),
'stderr': strip_last_newline_char(stderr)
}
if error:
result['error'] = error
status = LIVEACTION_STATUS_SUCCEEDED if exit_code == 0 else LIVEACTION_STATUS_FAILED
return (status, jsonify.json_loads(result, LocalShellRunner.KEYS_TO_TRANSFORM), None)
def run(self, action_parameters):
LOG.debug('Running pythonrunner.')
LOG.debug('Getting pack name.')
pack = self.get_pack_ref()
LOG.debug('Getting user.')
user = self.get_user()
LOG.debug('Serializing parameters.')
serialized_parameters = json.dumps(action_parameters if action_parameters else {})
LOG.debug('Getting virtualenv_path.')
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
LOG.debug('Getting python path.')
if self._sandbox:
python_path = get_sandbox_python_binary_path(pack=pack)
else:
python_path = sys.executable
LOG.debug('Checking virtualenv path.')
if virtualenv_path and not os.path.isdir(virtualenv_path):
format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
LOG.error('virtualenv_path set but not a directory: %s', msg)
raise Exception(msg)
LOG.debug('Checking entry_point.')
if not self.entry_point:
LOG.error('Action "%s" is missing entry_point attribute' % (self.action.name))
raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))
# Note: We pass config as command line args so the actual wrapper process is standalone
# and doesn't need access to db
LOG.debug('Setting args.')
if self._use_parent_args:
parent_args = json.dumps(sys.argv[1:])
else:
parent_args = json.dumps([])
args = [
python_path,
'-u', # unbuffered mode so streaming mode works as expected
WRAPPER_SCRIPT_PATH,
'--pack=%s' % (pack),
'--file-path=%s' % (self.entry_point),
'--user=%s' % (user),
'--parent-args=%s' % (parent_args),
]
# If parameter size is larger than the maximum allowed by Linux kernel
# we need to swap to stdin to communicate parameters. This avoids a
# failure to fork the wrapper process when using large parameters.
stdin = None
stdin_params = None
if len(serialized_parameters) >= MAX_PARAM_LENGTH:
stdin = subprocess.PIPE
LOG.debug('Parameters are too big...changing to stdin')
stdin_params = '{"parameters": %s}\n' % (serialized_parameters)
args.append('--stdin-parameters')
else:
LOG.debug('Parameters are just right...adding them to arguments')
args.append('--parameters=%s' % (serialized_parameters))
if self._config:
args.append('--config=%s' % (json.dumps(self._config)))
if self._log_level != PYTHON_RUNNER_DEFAULT_LOG_LEVEL:
# We only pass --log-level parameter if non default log level value is specified
args.append('--log-level=%s' % (self._log_level))
# We need to ensure all the st2 dependencies are also available to the subprocess
LOG.debug('Setting env.')
env = os.environ.copy()
env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
sandbox_python_path = get_sandbox_python_path_for_python_action(
pack=pack,
inherit_from_parent=True,
inherit_parent_virtualenv=True)
if self._enable_common_pack_libs:
try:
pack_common_libs_path = self._get_pack_common_libs_path(pack_ref=pack)
except Exception as e:
LOG.debug('Failed to retrieve pack common lib path: %s' % (str(e)))
# There is no MongoDB connection available in Lambda and pack common lib
# functionality is not also mandatory for Lambda so we simply ignore those errors.
# Note: We should eventually refactor this code to make runner standalone and not
# depend on a db connection (as it was in the past) - this param should be passed
# to the runner by the action runner container
pack_common_libs_path = None
else:
pack_common_libs_path = None
# Remove leading : (if any)
if sandbox_python_path.startswith(':'):
sandbox_python_path = sandbox_python_path[1:]
if self._enable_common_pack_libs and pack_common_libs_path:
sandbox_python_path = pack_common_libs_path + ':' + sandbox_python_path
env['PYTHONPATH'] = sandbox_python_path
# Include user provided environment variables (if any)
user_env_vars = self._get_env_vars()
env.update(user_env_vars)
# Include common st2 environment variables
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
datastore_env_vars = self._get_datastore_access_env_vars()
env.update(datastore_env_vars)
stdout = StringIO()
stderr = StringIO()
store_execution_stdout_line = functools.partial(store_execution_output_data,
output_type='stdout')
store_execution_stderr_line = functools.partial(store_execution_output_data,
output_type='stderr')
read_and_store_stdout = make_read_and_store_stream_func(execution_db=self.execution,
action_db=self.action, store_data_func=store_execution_stdout_line)
read_and_store_stderr = make_read_and_store_stream_func(execution_db=self.execution,
action_db=self.action, store_data_func=store_execution_stderr_line)
command_string = list2cmdline(args)
if stdin_params:
command_string = 'echo %s | %s' % (quote_unix(stdin_params), command_string)
LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'],
command_string))
exit_code, stdout, stderr, timed_out = run_command(cmd=args,
stdin=stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
env=env,
timeout=self._timeout,
read_stdout_func=read_and_store_stdout,
read_stderr_func=read_and_store_stderr,
read_stdout_buffer=stdout,
read_stderr_buffer=stderr,
stdin_value=stdin_params)
LOG.debug('Returning values: %s, %s, %s, %s', exit_code, stdout, stderr, timed_out)
LOG.debug('Returning.')
return self._get_output_values(exit_code, stdout, stderr, timed_out)
def run(self, action_parameters):
LOG.debug('Running pythonrunner.')
LOG.debug('Getting pack name.')
pack = self.get_pack_name()
LOG.debug('Getting user.')
user = self.get_user()
LOG.debug('Serializing parameters.')
serialized_parameters = json.dumps(action_parameters) if action_parameters else ''
LOG.debug('Getting virtualenv_path.')
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
LOG.debug('Getting python path.')
python_path = get_sandbox_python_binary_path(pack=pack)
LOG.debug('Checking virtualenv path.')
if virtualenv_path and not os.path.isdir(virtualenv_path):
format_values = {'pack': pack, 'virtualenv_path': virtualenv_path}
msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values
LOG.error('virtualenv_path set but not a directory: %s', msg)
raise Exception(msg)
LOG.debug('Checking entry_point.')
if not self.entry_point:
LOG.error('Action "%s" is missing entry_point attribute' % (self.action.name))
raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name))
LOG.debug('Setting args.')
args = [
python_path,
'-u',
WRAPPER_SCRIPT_PATH,
'--pack=%s' % (pack),
'--file-path=%s' % (self.entry_point),
'--parameters=%s' % (serialized_parameters),
'--user=%s' % (user),
'--parent-args=%s' % (json.dumps(sys.argv[1:]))
]
# We need to ensure all the st2 dependencies are also available to the
# subprocess
LOG.debug('Setting env.')
env = os.environ.copy()
env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path)
env['PYTHONPATH'] = get_sandbox_python_path(inherit_from_parent=True,
inherit_parent_virtualenv=True)
# Include user provided environment variables (if any)
user_env_vars = self._get_env_vars()
env.update(user_env_vars)
# Include common st2 environment variables
st2_env_vars = self._get_common_action_env_variables()
env.update(st2_env_vars)
datastore_env_vars = self._get_datastore_access_env_vars()
env.update(datastore_env_vars)
stdout = StringIO()
stderr = StringIO()
store_execution_stdout_line = functools.partial(store_execution_output_data,
output_type='stdout')
store_execution_stderr_line = functools.partial(store_execution_output_data,
output_type='stderr')
read_and_store_stdout = make_read_and_store_stream_func(execution_db=self.execution,
action_db=self.action, store_data_func=store_execution_stdout_line)
read_and_store_stderr = make_read_and_store_stream_func(execution_db=self.execution,
action_db=self.action, store_data_func=store_execution_stderr_line)
command_string = list2cmdline(args)
LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'],
command_string))
exit_code, stdout, stderr, timed_out = run_command(cmd=args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=False,
env=env,
timeout=self._timeout,
read_stdout_func=read_and_store_stdout,
read_stderr_func=read_and_store_stderr,
read_stdout_buffer=stdout,
read_stderr_buffer=stderr)
LOG.debug('Returning values: %s, %s, %s, %s' % (exit_code, stdout, stderr, timed_out))
LOG.debug('Returning.')
return self._get_output_values(exit_code, stdout, stderr, timed_out)