def test_process_wrapper_exits_in_reasonable_timeframe(self):
# 1. Verify wrapper script path is correct and file exists
self.assertTrue(os.path.isfile(WRAPPER_SCRIPT_PATH))
# 2. First run it without time to verify path is valid
command_string = 'python %s' % (WRAPPER_SCRIPT_PATH)
_, _, stderr = run_command(command_string, shell=True)
self.assertTrue('usage: python_action_wrapper.py' in stderr)
self.assertTrue('python_action_wrapper.py: error: argument' in stderr)
# 3. Now time it
command_string = '%s -f "%%e" python %s' % (TIME_BINARY_PATH, WRAPPER_SCRIPT_PATH)
# Do multiple runs and average it
run_times = []
count = 8
for i in range(0, count):
_, _, stderr = run_command(command_string, shell=True)
stderr = stderr.strip().split('\n')[-1]
run_time_seconds = float(stderr)
run_times.append(run_time_seconds)
avg_run_time_seconds = (sum(run_times) / count)
assertion_msg = ASSERTION_ERROR_MESSAGE % (WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT,
avg_run_time_seconds)
self.assertTrue(avg_run_time_seconds <= WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT, assertion_msg)
def test_register_from_pack_action_metadata_fails_validation(self):
# No fail on failure flag, should succeed
pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_4')
runner_dirs = os.path.join(get_fixtures_packs_base_path(), 'runners')
opts = [
'--register-pack=%s' % (pack_dir),
'--register-no-fail-on-failure',
'--register-runner-dir=%s' % (runner_dirs),
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('Registered 0 actions.' in stderr)
self.assertEqual(exit_code, 0)
# Fail on failure, should fail
pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_4')
opts = [
'--register-pack=%s' % (pack_dir),
'--register-fail-on-failure',
'--register-runner-dir=%s' % (runner_dirs),
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('object has no attribute \'get\'' in stderr)
self.assertEqual(exit_code, 1)
def test_register_recreate_virtualenvs(self):
# 1. Register the pack and ensure it exists and doesn't rely on state from previous
# test methods
pack_dir = os.path.join(get_fixtures_packs_base_path(), "dummy_pack_1")
cmd = BASE_CMD_ARGS + [
"--register-pack=%s" % (pack_dir),
"--register-setup-virtualenvs",
"--register-no-fail-on-failure",
]
exit_code, stdout, stderr = run_command(cmd=cmd)
self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
self.assertIn("Setup virtualenv for 1 pack(s)", stderr)
self.assertEqual(exit_code, 0)
# 2. Run it again with --register-recreate-virtualenvs flag
pack_dir = os.path.join(get_fixtures_packs_base_path(), "dummy_pack_1")
cmd = BASE_CMD_ARGS + [
"--register-pack=%s" % (pack_dir),
"--register-recreate-virtualenvs",
"--register-no-fail-on-failure",
]
exit_code, stdout, stderr = run_command(cmd=cmd)
self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
self.assertIn("Virtualenv successfully removed.", stderr)
self.assertIn("Setup virtualenv for 1 pack(s)", stderr)
self.assertEqual(exit_code, 0)
def test_process_wrapper_exits_in_reasonable_timeframe(self):
# 1. Verify wrapper script path is correct and file exists
self.assertTrue(os.path.isfile(WRAPPER_SCRIPT_PATH))
# 2. First run it without time to verify path is valid
command_string = 'python %s --file-path=foo.py' % (WRAPPER_SCRIPT_PATH)
_, _, stderr = run_command(command_string, shell=True)
self.assertTrue('usage: python_action_wrapper.py' in stderr)
expected_msg_1 = 'python_action_wrapper.py: error: argument --pack is required'
expected_msg_2 = ('python_action_wrapper.py: error: the following arguments are '
'required: --pack')
self.assertTrue(expected_msg_1 in stderr or expected_msg_2 in stderr)
# 3. Now time it
command_string = '%s -f "%%e" python %s' % (TIME_BINARY_PATH, WRAPPER_SCRIPT_PATH)
# Do multiple runs and average it
run_times = []
count = 8
for i in range(0, count):
_, _, stderr = run_command(command_string, shell=True)
stderr = stderr.strip().split('\n')[-1]
run_time_seconds = float(stderr)
run_times.append(run_time_seconds)
avg_run_time_seconds = (sum(run_times) / count)
assertion_msg = ASSERTION_ERROR_MESSAGE % (WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT,
avg_run_time_seconds)
self.assertTrue(avg_run_time_seconds <= WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT, assertion_msg)
def test_register_from_packs_doesnt_throw_on_missing_pack_resource_folder(
self):
# dummy_pack_4 only has actions folder, make sure it doesn't throw when
# sensors and other resource folders are missing
# Note: We want to use a different config which sets fixtures/packs_1/
# dir as packs_base_paths
cmd = [
sys.executable,
SCRIPT_PATH,
"--config-file=conf/st2.tests1.conf",
"-v",
"--register-sensors",
]
exit_code, _, stderr = run_command(cmd=cmd)
self.assertIn("Registered 0 sensors.", stderr,
"Actual stderr: %s" % (stderr))
self.assertEqual(exit_code, 0)
cmd = [
sys.executable,
SCRIPT_PATH,
"--config-file=conf/st2.tests1.conf",
"-v",
"--register-all",
"--register-no-fail-on-failure",
]
exit_code, _, stderr = run_command(cmd=cmd)
self.assertIn("Registered 0 actions.", stderr)
self.assertIn("Registered 0 sensors.", stderr)
self.assertIn("Registered 0 rules.", stderr)
self.assertEqual(exit_code, 0)
def test_register_from_pack_action_metadata_fails_validation(self):
# No fail on failure flag, should succeed
pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_4')
runner_dirs = os.path.join(get_fixtures_packs_base_path(), 'runners')
opts = [
'--register-pack=%s' % (pack_dir),
'--register-no-fail-on-failure',
'--register-runner-dir=%s' % (runner_dirs),
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('Registered 0 actions.' in stderr)
self.assertEqual(exit_code, 0)
# Fail on failure, should fail
pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_4')
opts = [
'--register-pack=%s' % (pack_dir),
'--register-fail-on-failure',
'--register-runner-dir=%s' % (runner_dirs),
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('object has no attribute \'get\'' in stderr)
self.assertEqual(exit_code, 1)
def test_register_from_pack_action_metadata_fails_validation(self):
# No fail on failure flag, should succeed
pack_dir = os.path.join(get_fixtures_packs_base_path(), "dummy_pack_4")
runner_dirs = os.path.join(get_fixtures_packs_base_path(), "runners")
opts = [
"--register-pack=%s" % (pack_dir),
"--register-no-fail-on-failure",
"--register-runner-dir=%s" % (runner_dirs),
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, stderr = run_command(cmd=cmd)
self.assertIn("Registered 0 actions.", stderr)
self.assertEqual(exit_code, 0)
# Fail on failure, should fail
pack_dir = os.path.join(get_fixtures_packs_base_path(), "dummy_pack_4")
opts = [
"--register-pack=%s" % (pack_dir),
"--register-fail-on-failure",
"--register-runner-dir=%s" % (runner_dirs),
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, stderr = run_command(cmd=cmd)
self.assertIn("object has no attribute 'get'", stderr)
self.assertEqual(exit_code, 1)
def test_register_from_pack_fail_on_failure_pack_dir_doesnt_exist(self):
# No fail on failure flag, should succeed
pack_dir = "doesntexistblah"
cmd = BASE_CMD_ARGS + ["--register-pack=%s" % (pack_dir)]
exit_code, _, _ = run_command(cmd=cmd)
self.assertEqual(exit_code, 0)
# Fail on failure, should fail
cmd = BASE_CMD_ARGS + ["--register-pack=%s" % (pack_dir), "--register-fail-on-failure"]
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('Directory "doesntexistblah" doesn\'t exist' in stderr)
self.assertEqual(exit_code, 1)
def test_register_from_pack_fail_on_failure_pack_dir_doesnt_exist(self):
# No fail on failure flag, should succeed
pack_dir = 'doesntexistblah'
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + ['--register-pack=%s' % (pack_dir)]
exit_code, _, _ = run_command(cmd=cmd)
self.assertEqual(exit_code, 0)
# Fail on failure, should fail
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + ['--register-pack=%s' % (pack_dir),
'--register-fail-on-failure']
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('Directory "doesntexistblah" doesn\'t exist' in stderr)
self.assertEqual(exit_code, 1)
def test_register_from_pack_action_metadata_fails_validation(self):
# No fail on failure flag, should succeed
pack_dir = os.path.join(get_fixtures_base_path(), "dummy_pack_4")
cmd = BASE_CMD_ARGS + ["--register-pack=%s" % (pack_dir)]
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue("Registered 0 actions." in stderr)
self.assertEqual(exit_code, 0)
# Fail on failure, should fail
pack_dir = os.path.join(get_fixtures_base_path(), "dummy_pack_4")
cmd = BASE_CMD_ARGS + ["--register-pack=%s" % (pack_dir), "--register-fail-on-failure"]
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue("object has no attribute 'get'" in stderr)
self.assertEqual(exit_code, 1)
def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=None):
"""
Install a single requirement.
:param requirement: Requirement specifier.
"""
logger = logger or LOG
pip_path = os.path.join(virtualenv_path, 'bin/pip')
cmd = [pip_path]
if proxy_config:
cert = proxy_config.get('proxy_ca_bundle_path', None)
https_proxy = proxy_config.get('https_proxy', None)
http_proxy = proxy_config.get('http_proxy', None)
if http_proxy:
cmd.extend(['--proxy', http_proxy])
if https_proxy:
cmd.extend(['--proxy', https_proxy])
if cert:
cmd.extend(['--cert', cert])
cmd.extend(['install', requirement])
env = get_env_for_subprocess_command()
logger.debug('Installing requirement %s with command %s.',
requirement, ' '.join(cmd))
exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
if exit_code != 0:
raise Exception('Failed to install requirement "%s": %s' %
(requirement, stdout))
return True
def create_virtualenv(virtualenv_path, logger=None):
logger = logger or LOG
python_binary = cfg.CONF.actionrunner.python_binary
virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts
if not os.path.isfile(python_binary):
raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))
if not os.path.isfile(virtualenv_binary):
raise Exception('Virtualenv binary "%s" doesn\'t exist.' %
(virtualenv_binary))
logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
(virtualenv_path, python_binary))
cmd = [virtualenv_binary, '-p', python_binary]
cmd.extend(virtualenv_opts)
cmd.extend([virtualenv_path])
logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd))
try:
exit_code, _, stderr = run_command(cmd=cmd)
except OSError as e:
raise Exception('Error executing command %s. %s.' %
(' '.join(cmd), e.message))
if exit_code != 0:
raise Exception('Failed to create virtualenv in "%s": %s' %
(virtualenv_path, stderr))
return True
def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=None):
"""
Install a single requirement.
:param requirement: Requirement specifier.
"""
logger = logger or LOG
pip_path = os.path.join(virtualenv_path, 'bin/pip')
cmd = [pip_path]
if proxy_config:
cert = proxy_config.get('proxy_ca_bundle_path', None)
https_proxy = proxy_config.get('https_proxy', None)
http_proxy = proxy_config.get('http_proxy', None)
if http_proxy:
cmd.extend(['--proxy', http_proxy])
if https_proxy:
cmd.extend(['--proxy', https_proxy])
if cert:
cmd.extend(['--cert', cert])
cmd.extend(['install', requirement])
env = get_env_for_subprocess_command()
logger.debug('Installing requirement %s with command %s.',
requirement, ' '.join(cmd))
exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
if exit_code != 0:
raise Exception('Failed to install requirement "%s": %s' %
(requirement, stdout))
return True
def test_register_from_pack_success(self):
pack_dir = os.path.join(get_fixtures_base_path(), "dummy_pack_1")
cmd = BASE_CMD_ARGS + ["--register-pack=%s" % (pack_dir)]
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue("Registered 1 actions." in stderr)
self.assertEqual(exit_code, 0)
def get_deb_package_list(name_startswith):
cmd = 'dpkg -l | grep %s' % (quote_unix(name_startswith))
exit_code, stdout, _ = run_command(cmd=cmd, shell=True)
lines = stdout.split('\n')
packages = []
for line in lines:
line = line.strip()
if not line:
continue
split = re.split(r'\s+', line)
name = split[1]
version = split[2]
if not name.startswith(name_startswith):
continue
item = {
'name': name,
'version': version
}
packages.append(item)
return packages
def get_rpm_package_list(name_startswith):
cmd = 'rpm -qa | grep %s' % (quote_unix(name_startswith))
exit_code, stdout, _ = run_command(cmd=cmd, shell=True)
lines = stdout.split('\n')
packages = []
for line in lines:
line = line.strip()
if not line:
continue
split = line.rsplit('.', 1)
split = split[0].split('-', 1)
name = split[0]
version = split[1]
if not name.startswith(name_startswith):
continue
item = {
'name': name,
'version': version
}
packages.append(item)
return packages
def run(self, hostname, port, bean_name, command, arguments=None,
username=None, password=None):
args = self._get_args(hostname=hostname, port=port,
bean_name=bean_name, command=command,
arguments=arguments, username=username,
password=password)
command = ' '.join(args)
self.logger.debug('Running command: "%s"' % (command))
exit_code, stdout, stderr = run_command(cmd=args)
if exit_code != 0:
msg = 'Failed to invoke command: %s' % (stderr)
raise Exception(msg)
if re.match('.*Operation .*? not found.*', stderr):
msg = 'Failed to invoke command: %s' % (stderr)
raise Exception(msg)
if 'Passed param count does not match signature count' in stderr:
msg = 'Failed to invoke command: %s' % (stderr)
raise Exception(msg)
self.logger.debug('Command successfully finished. Output: %s' % (stdout))
return True
def test_register_from_pack_success(self):
pack_dir = os.path.join(get_fixtures_base_path(), 'dummy_pack_1')
cmd = BASE_CMD_ARGS + ['--register-pack=%s' % (pack_dir)]
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('Registered 1 actions.' in stderr)
self.assertEqual(exit_code, 0)
def test_register_setup_virtualenvs(self):
# Single pack
pack_dir = os.path.join(get_fixtures_base_path(), 'dummy_pack_1')
cmd = BASE_CMD_ARGS + ['--register-pack=%s' % (pack_dir), '--register-setup-virtualenvs']
exit_code, stdout, stderr = run_command(cmd=cmd)
self.assertTrue('Setting up virtualenv for pack "dummy_pack_1"' in stderr)
self.assertTrue('Setup virtualenv for 1 pack(s)' in stderr)
self.assertEqual(exit_code, 0)
# All packs
cmd = BASE_CMD_ARGS + ['--register-setup-virtualenvs']
exit_code, stdout, stderr = run_command(cmd=cmd)
self.assertTrue('Setup virtualenv for 4 pack(s)' in stderr)
self.assertEqual(exit_code, 0)
def create_virtualenv(virtualenv_path, logger=None):
logger = logger or LOG
python_binary = cfg.CONF.actionrunner.python_binary
virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts
if not os.path.isfile(python_binary):
raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))
if not os.path.isfile(virtualenv_binary):
raise Exception('Virtualenv binary "%s" doesn\'t exist.' % (virtualenv_binary))
logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
(virtualenv_path, python_binary))
cmd = [virtualenv_binary, '-p', python_binary]
cmd.extend(virtualenv_opts)
cmd.extend([virtualenv_path])
logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd))
try:
exit_code, _, stderr = run_command(cmd=cmd)
except OSError as e:
raise Exception('Error executing command %s. %s.' % (' '.join(cmd),
e.message))
if exit_code != 0:
raise Exception('Failed to create virtualenv in "%s": %s' %
(virtualenv_path, stderr))
return True
def run(self,
hostname,
port,
bean_name,
command,
arguments=None,
username=None,
password=None):
args = self._get_args(hostname=hostname,
port=port,
bean_name=bean_name,
command=command,
arguments=arguments,
username=username,
password=password)
command = ' '.join(args)
self.logger.debug('Running command: "%s"' % (command))
exit_code, stdout, stderr = run_command(cmd=args)
if exit_code != 0:
msg = 'Failed to invoke command: %s' % (stderr)
raise Exception(msg)
if re.match('.*Operation .*? not found.*', stderr):
msg = 'Failed to invoke command: %s' % (stderr)
raise Exception(msg)
if 'Passed param count does not match signature count' in stderr:
msg = 'Failed to invoke command: %s' % (stderr)
raise Exception(msg)
self.logger.debug('Command successfully finished. Output: %s' %
(stdout))
return True
def get_rpm_package_list(name_startswith):
cmd = 'rpm -qa | grep %s' % (pipes.quote(name_startswith))
exit_code, stdout, _ = run_command(cmd=cmd, shell=True)
lines = stdout.split('\n')
packages = []
for line in lines:
line = line.strip()
if not line:
continue
split = line.rsplit('.', 1)
split = split[0].split('-', 1)
name = split[0]
version = split[1]
if not name.startswith(name_startswith):
continue
item = {'name': name, 'version': version}
packages.append(item)
return packages
def test_register_from_pack_success(self):
pack_dir = os.path.join(get_fixtures_base_path(), 'dummy_pack_1')
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + ['--register-pack=%s' % (pack_dir)]
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('Registered 1 actions.' in stderr)
self.assertEqual(exit_code, 0)
def test_config_with_a_lot_of_items_and_a_lot_of_parameters_work_fine(
self):
# Test case which verifies that actions with large configs and a lot of parameters work
# fine. Config and parameters are passed to wrapper as command line argument so there is an
# upper limit on the size.
config = {}
for index in range(0, 50):
config["key_%s" % (index)] = "value value foo %s" % (index)
config = json.dumps(config)
parameters = {}
for index in range(0, 30):
parameters["param_foo_%s" %
(index)] = "some param value %s" % (index)
parameters = json.dumps(parameters)
file_path = os.path.join(BASE_DIR,
"../../../../examples/actions/noop.py")
command_string = (
"python %s --pack=dummy --file-path=%s --config='%s' "
"--parameters='%s'" %
(WRAPPER_SCRIPT_PATH, file_path, config, parameters))
exit_code, stdout, stderr = run_command(command_string, shell=True)
self.assertEqual(exit_code, 0)
self.assertIn('"status"', stdout)
def create_virtualenv(virtualenv_path,
logger=None,
include_pip=True,
include_setuptools=True,
include_wheel=True):
"""
:param include_pip: Include pip binary and package in the newely created virtual environment.
:type include_pip: ``bool``
:param include_setuptools: Include setuptools binary and package in the newely created virtual
environment.
:type include_setuptools: ``bool``
:param include_wheel: Include wheel in the newely created virtual environment.
:type include_wheel : ``bool``
"""
logger = logger or LOG
python_binary = cfg.CONF.actionrunner.python_binary
virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts
if not os.path.isfile(python_binary):
raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))
if not os.path.isfile(virtualenv_binary):
raise Exception('Virtualenv binary "%s" doesn\'t exist.' %
(virtualenv_binary))
logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
(virtualenv_path, python_binary))
cmd = [virtualenv_binary, '-p', python_binary]
cmd.extend(virtualenv_opts)
if not include_pip:
cmd.append('--no-pip')
if not include_setuptools:
cmd.append('--no-setuptools')
if not include_wheel:
cmd.append('--no-wheel')
cmd.extend([virtualenv_path])
logger.debug('Running command "%s" to create virtualenv.', ' '.join(cmd))
try:
exit_code, _, stderr = run_command(cmd=cmd)
except OSError as e:
raise Exception('Error executing command %s. %s.' %
(' '.join(cmd), e.message))
if exit_code != 0:
raise Exception('Failed to create virtualenv in "%s": %s' %
(virtualenv_path, stderr))
return True
def test_register_all_and_register_setup_virtualenvs(self):
# Verify that --register-all works in combinations with --register-setuo-virtualenvs
cmd = BASE_CMD_ARGS + ['--register-all', '--register-setup-virtualenvs']
exit_code, stdout, stderr = run_command(cmd=cmd)
self.assertTrue('Registering actions' in stderr)
self.assertTrue('Registering rules' in stderr)
self.assertTrue('Setup virtualenv for 5 pack(s)' in stderr)
self.assertEqual(exit_code, 0)
def test_register_from_packs_doesnt_throw_on_missing_pack_resource_folder(self):
# dummy_pack_4 only has actions folder, make sure it doesn't throw when
# sensors and other resource folders are missing
# Note: We want to use a different config which sets fixtures/packs_1/
# dir as packs_base_paths
cmd = [SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', '--register-sensors']
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('Registered 0 sensors.' in stderr)
self.assertEqual(exit_code, 0)
cmd = [SCRIPT_PATH, '--config-file=conf/st2.tests1.conf', '-v', '--register-all']
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('Registered 0 actions.' in stderr)
self.assertTrue('Registered 0 sensors.' in stderr)
self.assertTrue('Registered 0 rules.' in stderr)
self.assertEqual(exit_code, 0)
def test_register_all_and_register_setup_virtualenvs(self):
# Verify that --register-all works in combinations with --register-setup-virtualenvs
cmd = BASE_CMD_ARGS + ['--register-all', '--register-setup-virtualenvs',
'--register-no-fail-on-failure']
exit_code, stdout, stderr = run_command(cmd=cmd)
self.assertTrue('Registering actions' in stderr)
self.assertTrue('Registering rules' in stderr)
self.assertTrue('Setup virtualenv for %s pack(s)' % (PACKS_COUNT) in stderr)
self.assertEqual(exit_code, 0)
def test_process_wrapper_exits_in_reasonable_timeframe(self):
_, _, stderr = run_command('%s -f "%%e" python %s --is-subprocess' %
(TIME_BINARY_PATH, WRAPPER_SCRIPT_PATH), shell=True)
stderr = stderr.strip().split('\n')[-1]
run_time_seconds = float(stderr)
assertion_msg = ASSERTION_ERROR_MESSAGE % (WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT,
run_time_seconds)
self.assertTrue(run_time_seconds <= WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT, assertion_msg)
def test_register_from_pack_success(self):
pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_1')
runner_dirs = os.path.join(get_fixtures_packs_base_path(), 'runners')
opts = [
'--register-pack=%s' % (pack_dir),
'--register-runner-dir=%s' % (runner_dirs),
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('Registered 1 actions.' in stderr)
self.assertEqual(exit_code, 0)
def test_register_from_pack_success(self):
pack_dir = os.path.join(get_fixtures_packs_base_path(), "dummy_pack_1")
runner_dirs = os.path.join(get_fixtures_packs_base_path(), "runners")
opts = [
"--register-pack=%s" % (pack_dir),
"--register-runner-dir=%s" % (runner_dirs),
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, stderr = run_command(cmd=cmd)
self.assertIn("Registered 1 actions.", stderr)
self.assertEqual(exit_code, 0)
def test_process_wrapper_exits_in_reasonable_timeframe(self):
# 1. Verify wrapper script path is correct and file exists
self.assertTrue(os.path.isfile(WRAPPER_SCRIPT_PATH))
# 2. First run it without time to verify path is valid
command_string = "python %s --file-path=foo.py" % (WRAPPER_SCRIPT_PATH)
_, _, stderr = run_command(command_string, shell=True)
self.assertIn("usage: python_action_wrapper.py", stderr)
expected_msg_1 = "python_action_wrapper.py: error: argument --pack is required"
expected_msg_2 = (
"python_action_wrapper.py: error: the following arguments are "
"required: --pack")
self.assertTrue(expected_msg_1 in stderr or expected_msg_2 in stderr)
# 3. Now time it
command_string = '%s -f "%%e" python %s' % (
TIME_BINARY_PATH,
WRAPPER_SCRIPT_PATH,
)
# Do multiple runs and average it
run_times = []
count = 8
for i in range(0, count):
_, _, stderr = run_command(command_string, shell=True)
stderr = stderr.strip().split("\n")[-1]
run_time_seconds = float(stderr)
run_times.append(run_time_seconds)
avg_run_time_seconds = sum(run_times) / count
assertion_msg = ASSERTION_ERROR_MESSAGE % (
WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT,
avg_run_time_seconds,
)
self.assertTrue(
avg_run_time_seconds <= WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT,
assertion_msg)
def install_requirements(virtualenv_path,
requirements_file_path,
proxy_config=None,
logger=None):
"""
Install requirements from a file.
"""
logger = logger or LOG
pip_path = os.path.join(virtualenv_path, "bin/pip")
pip_opts = cfg.CONF.actionrunner.pip_opts or []
cmd = [pip_path]
if proxy_config:
cert = proxy_config.get("proxy_ca_bundle_path", None)
https_proxy = proxy_config.get("https_proxy", None)
http_proxy = proxy_config.get("http_proxy", None)
if http_proxy:
cmd.extend(["--proxy", http_proxy])
if https_proxy:
cmd.extend(["--proxy", https_proxy])
if cert:
cmd.extend(["--cert", cert])
cmd.append("install")
cmd.extend(pip_opts)
cmd.extend(["-U", "-r", requirements_file_path])
env = get_env_for_subprocess_command()
logger.debug(
"Installing requirements from file %s with command %s.",
requirements_file_path,
" ".join(cmd),
)
exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
# Normally we don't want this, even in debug logs. But it is useful to
# investigate pip behavior changes & broken virtualenv integration tests.
# logger.debug(f"\npip stdout=\n{stdout}")
if exit_code != 0:
stdout = to_ascii(stdout)
stderr = to_ascii(stderr)
raise Exception(
'Failed to install requirements from "%s": %s (stderr: %s)' %
(requirements_file_path, stdout, stderr))
return True
def test_stdin_params_invalid_format_friendly_error(self):
config = {}
file_path = os.path.join(BASE_DIR, '../../../contrib/examples/actions/noop.py')
# Not a valid JSON string
command_string = ('echo "invalid" | python %s --pack=dummy --file-path=%s --config=\'%s\' '
'--stdin-parameters' %
(WRAPPER_SCRIPT_PATH, file_path, config))
exit_code, stdout, stderr = run_command(command_string, shell=True)
expected_msg = ('ValueError: Failed to parse parameters from stdin. Expected a JSON '
'object with "parameters" attribute')
self.assertEqual(exit_code, 1)
self.assertIn(expected_msg, stderr)
def _install_requirement(self, virtualenv_path, requirement):
"""
Install a single requirement.
"""
pip_path = os.path.join(virtualenv_path, "bin/pip")
cmd = [pip_path, "install", requirement]
env = self._get_env_for_subprocess_command()
exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
if exit_code != 0:
raise Exception('Failed to install requirement "%s": %s' % (requirement, stdout))
return True
def get_package_list(name_startswith):
"""
Retrieve system packages which name matches the provided startswith filter.
Note: This function only supports Debian and RedHat based systems.
:param name_startswith: Package name startswith filter string.
:type name_startswith: ``str``
:rtype: ``list`` of ``dict``
"""
dpkg_exit_code, _, _ = run_command(cmd='dpkg', shell=True)
rpm_exit_code, _, _ = run_command(cmd='rpm', shell=True)
if dpkg_exit_code != 127:
result = get_deb_package_list(name_startswith=name_startswith)
elif rpm_exit_code != 127:
result = get_rpm_package_list(name_startswith=name_startswith)
else:
raise Exception('Unsupported platform (dpkg or rpm binary not available)')
return result
def test_stdin_params_timeout_no_stdin_data_provided(self):
config = {}
file_path = os.path.join(BASE_DIR, '../../../../examples/actions/noop.py')
command_string = ('python %s --pack=dummy --file-path=%s --config=\'%s\' '
'--stdin-parameters' %
(WRAPPER_SCRIPT_PATH, file_path, config))
exit_code, stdout, stderr = run_command(command_string, shell=True)
expected_msg = ('ValueError: No input received and timed out while waiting for parameters '
'from stdin')
self.assertEqual(exit_code, 1)
self.assertTrue(expected_msg in stderr)
def test_stdin_params_timeout_no_stdin_data_provided(self):
config = {}
file_path = os.path.join(BASE_DIR, '../../../../examples/actions/noop.py')
command_string = ('python %s --pack=dummy --file-path=%s --config=\'%s\' '
'--stdin-parameters' %
(WRAPPER_SCRIPT_PATH, file_path, config))
exit_code, stdout, stderr = run_command(command_string, shell=True)
expected_msg = ('ValueError: No input received and timed out while waiting for parameters '
'from stdin')
self.assertEqual(exit_code, 1)
self.assertTrue(expected_msg in stderr)
def test_register_all_and_register_setup_virtualenvs(self):
# Verify that --register-all works in combinations with --register-setup-virtualenvs
# Single pack
pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_1')
cmd = BASE_CMD_ARGS + [
'--register-pack=%s' % (pack_dir), '--register-all',
'--register-setup-virtualenvs', '--register-no-fail-on-failure'
]
exit_code, stdout, stderr = run_command(cmd=cmd)
self.assertIn('Registering actions', stderr,
'Actual stderr: %s' % (stderr))
self.assertIn('Registering rules', stderr)
self.assertIn('Setup virtualenv for %s pack(s)' % ('1'), stderr)
self.assertEqual(exit_code, 0)
def _install_requirement(self, virtualenv_path, requirement):
"""
Install a single requirement.
"""
pip_path = os.path.join(virtualenv_path, 'bin/pip')
cmd = [pip_path, 'install', requirement]
env = self._get_env_for_subprocess_command()
exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
if exit_code != 0:
raise Exception('Failed to install requirement "%s": %s' %
(requirement, stdout))
return True
def test_register_recreate_virtualenvs(self):
# Single pack
pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_1')
cmd = BASE_CMD_ARGS + [
'--register-pack=%s' % (pack_dir),
'--register-recreate-virtualenvs', '--register-no-fail-on-failure'
]
exit_code, stdout, stderr = run_command(cmd=cmd)
self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
self.assertIn('Setup virtualenv for 1 pack(s)', stderr)
self.assertIn('Virtualenv successfull removed.', stderr)
self.assertEqual(exit_code, 0)
def get_package_list(name_startswith):
"""
Retrieve system packages which name matches the provided startswith filter.
Note: This function only supports Debian and RedHat based systems.
:param name_startswith: Package name startswith filter string.
:type name_startswith: ``str``
:rtype: ``list`` of ``dict``
"""
dpkg_exit_code, _, _ = run_command(cmd='dpkg', shell=True)
rpm_exit_code, _, _ = run_command(cmd='rpm', shell=True)
if dpkg_exit_code != 127:
result = get_deb_package_list(name_startswith=name_startswith)
elif rpm_exit_code != 127:
result = get_rpm_package_list(name_startswith=name_startswith)
else:
raise Exception(
'Unsupported platform (dpkg or rpm binary not available)')
return result
def test_stdin_params_invalid_format_friendly_error(self):
config = {}
file_path = os.path.join(BASE_DIR, '../../../contrib/examples/actions/noop.py')
# Not a valid JSON string
command_string = ('echo "invalid" | python %s --pack=dummy --file-path=%s --config=\'%s\' '
'--stdin-parameters' %
(WRAPPER_SCRIPT_PATH, file_path, config))
exit_code, stdout, stderr = run_command(command_string, shell=True)
expected_msg = ('ValueError: Failed to parse parameters from stdin. Expected a JSON '
'object with "parameters" attribute:')
self.assertEqual(exit_code, 1)
self.assertTrue(expected_msg in stderr)
def test_register_setup_virtualenvs(self):
# Single pack
pack_dir = os.path.join(get_fixtures_packs_base_path(), "dummy_pack_1")
cmd = BASE_CMD_ARGS + [
"--register-pack=%s" % (pack_dir),
"--register-setup-virtualenvs",
"--register-no-fail-on-failure",
]
exit_code, stdout, stderr = run_command(cmd=cmd)
self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
self.assertIn("Setup virtualenv for 1 pack(s)", stderr)
self.assertEqual(exit_code, 0)
def install_requirements(virtualenv_path, requirements_file_path):
"""
Install requirements from a file.
"""
pip_path = os.path.join(virtualenv_path, 'bin/pip')
cmd = [pip_path, 'install', '-U', '-r', requirements_file_path]
env = get_env_for_subprocess_command()
exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
if exit_code != 0:
raise Exception('Failed to install requirements from "%s": %s (stderr: %s)' %
(requirements_file_path, stdout, stderr))
return True
def _install_requirements(self, virtualenv_path, requirements_file_path):
"""
Install requirements from a file.
"""
pip_path = os.path.join(virtualenv_path, 'bin/pip')
cmd = [pip_path, 'install', '-U', '-r', requirements_file_path]
env = self._get_env_for_subprocess_command()
exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
if exit_code != 0:
raise Exception('Failed to install requirements from "%s": %s' %
(requirements_file_path, stdout))
return True
def poll(self):
command = ' '.join(self._args)
self._logger.debug('Running command: "%s"' % (command))
exit_code, stdout, _ = run_command(cmd=self._args)
if 'status err' in stdout:
split = stdout.split(' ', 3)
error = split[-1]
self._logger.warn('Failed to retrieve metrics: %s' % (error))
else:
metrics = self._parse_output(output=stdout)
for metric in metrics:
self._dispatch_trigger_for_metric(metric=metric)
def test_register_all_and_register_setup_virtualenvs(self):
# Verify that --register-all works in combinations with --register-setup-virtualenvs
# Single pack
pack_dir = os.path.join(get_fixtures_packs_base_path(), 'dummy_pack_1')
cmd = BASE_CMD_ARGS + [
'--register-pack=%s' % (pack_dir),
'--register-all',
'--register-setup-virtualenvs',
'--register-no-fail-on-failure'
]
exit_code, stdout, stderr = run_command(cmd=cmd)
self.assertTrue('Registering actions' in stderr, 'Actual stderr: %s' % (stderr))
self.assertTrue('Registering rules' in stderr)
self.assertTrue('Setup virtualenv for %s pack(s)' % ('1') in stderr)
self.assertEqual(exit_code, 0)
def test_register_from_pack_fail_on_failure_pack_dir_doesnt_exist(self):
# No fail on failure flag, should succeed
pack_dir = 'doesntexistblah'
runner_dirs = os.path.join(get_fixtures_packs_base_path(), 'runners')
opts = [
'--register-pack=%s' % (pack_dir),
'--register-runner-dir=%s' % (runner_dirs),
'--register-no-fail-on-failure'
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, _ = run_command(cmd=cmd)
self.assertEqual(exit_code, 0)
# Fail on failure, should fail
opts = [
'--register-pack=%s' % (pack_dir),
'--register-runner-dir=%s' % (runner_dirs),
'--register-fail-on-failure'
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, stderr = run_command(cmd=cmd)
self.assertTrue('Directory "doesntexistblah" doesn\'t exist' in stderr)
self.assertEqual(exit_code, 1)
def get_commands_output():
""""
Get output of the required shell command and redirect the output to a file.
:return: output file paths
:rtype: ``list``
"""
commands_list = get_config_details('shell_commands')
output_files_list = []
for cmd in commands_list:
output_file = os.path.join('/tmp', '%s.txt' % format_output_filename(cmd))
exit_code, stdout, _ = run_command(cmd=cmd, shell=True)
with open(output_file, 'w') as fp:
fp.write(stdout)
output_files_list.append(output_file)
return output_files_list
def test_register_from_pack_fail_on_failure_pack_dir_doesnt_exist(self):
# No fail on failure flag, should succeed
pack_dir = "doesntexistblah"
runner_dirs = os.path.join(get_fixtures_packs_base_path(), "runners")
opts = [
"--register-pack=%s" % (pack_dir),
"--register-runner-dir=%s" % (runner_dirs),
"--register-no-fail-on-failure",
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, _ = run_command(cmd=cmd)
self.assertEqual(exit_code, 0)
# Fail on failure, should fail
opts = [
"--register-pack=%s" % (pack_dir),
"--register-runner-dir=%s" % (runner_dirs),
"--register-fail-on-failure",
]
cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
exit_code, _, stderr = run_command(cmd=cmd)
self.assertIn('Directory "doesntexistblah" doesn\'t exist', stderr)
self.assertEqual(exit_code, 1)
def install_requirement(virtualenv_path, requirement):
"""
Install a single requirement.
:param requirement: Requirement specifier.
"""
pip_path = os.path.join(virtualenv_path, 'bin/pip')
cmd = [pip_path, 'install', requirement]
env = get_env_for_subprocess_command()
exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
if exit_code != 0:
raise Exception('Failed to install requirement "%s": %s' %
(requirement, stdout))
return True
def _create_virtualenv(self, virtualenv_path):
python_binary = cfg.CONF.actionrunner.python_binary
if not os.path.isfile(python_binary):
raise Exception('Python binary "%s" doesn\'t exist' % (python_binary))
self.logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
(virtualenv_path, python_binary))
cmd = ['virtualenv', '-p', python_binary, '--system-site-packages', virtualenv_path]
exit_code, _, stderr = run_command(cmd=cmd)
if exit_code != 0:
raise Exception('Failed to create virtualenv in "%s": %s' %
(virtualenv_path, stderr))
return True
def add_shell_command_output(self, output_path):
""""
Get output of the required shell command and redirect the output to output path.
:param output_path: Directory where output files will be written
:param output_path: ``str``
"""
LOG.debug('Including the required shell commands output files')
for cmd in self.shell_commands:
output_file = os.path.join(output_path, '%s.txt' % self.format_output_filename(cmd))
exit_code, stdout, stderr = run_command(cmd=cmd, shell=True, cwd=output_path)
with open(output_file, 'w') as fp:
fp.write('[BEGIN STDOUT]\n')
fp.write(stdout)
fp.write('[END STDOUT]\n')
fp.write('[BEGIN STDERR]\n')
fp.write(stderr)
fp.write('[END STDERR]')
def get_repo_latest_revision_hash(repo_path):
"""
Return a hash to the latest revision in the provided repo.
:param repo_path: Path to the git repository.
:type repo_path: ``str``
:rtype: ``str``
"""
try:
exit_code, stdout, _ = run_command(cmd=["git", "rev-parse", "HEAD"], cwd=repo_path)
except Exception:
revision_hash = "unknown"
else:
if exit_code == 0:
revision_hash = stdout.strip()
else:
revision_hash = "unknown"
return revision_hash
def test_config_with_a_lot_of_items_and_a_lot_of_parameters_work_fine(self):
# Test case which verifies that actions with large configs and a lot of parameters work
# fine. Config and parameters are passed to wrapper as command line argument so there is an
# upper limit on the size.
config = {}
for index in range(0, 50):
config['key_%s' % (index)] = 'value value foo %s' % (index)
config = json.dumps(config)
parameters = {}
for index in range(0, 30):
parameters['param_foo_%s' % (index)] = 'some param value %s' % (index)
parameters = json.dumps(parameters)
file_path = os.path.join(BASE_DIR, '../../../../examples/actions/noop.py')
command_string = ('python %s --pack=dummy --file-path=%s --config=\'%s\' '
'--parameters=\'%s\'' %
(WRAPPER_SCRIPT_PATH, file_path, config, parameters))
exit_code, stdout, stderr = run_command(command_string, shell=True)
self.assertEqual(exit_code, 0)
self.assertTrue('"status"' in stdout)
def install_requirements(virtualenv_path, requirements_file_path, proxy_config=None, logger=None):
"""
Install requirements from a file.
"""
logger = logger or LOG
pip_path = os.path.join(virtualenv_path, 'bin/pip')
cmd = [pip_path]
if proxy_config:
cert = proxy_config.get('proxy_ca_bundle_path', None)
https_proxy = proxy_config.get('https_proxy', None)
http_proxy = proxy_config.get('http_proxy', None)
if http_proxy:
cmd.extend(['--proxy', http_proxy])
if https_proxy:
cmd.extend(['--proxy', https_proxy])
if cert:
cmd.extend(['--cert', cert])
cmd.extend(['install', '-U', '-r', requirements_file_path])
env = get_env_for_subprocess_command()
logger.debug('Installing requirements from file %s with command %s.',
requirements_file_path, ' '.join(cmd))
exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
if exit_code != 0:
stdout = to_ascii(stdout)
stderr = to_ascii(stderr)
raise Exception('Failed to install requirements from "%s": %s (stderr: %s)' %
(requirements_file_path, stdout, stderr))
return True