def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
'''
This test checks priority of loading library, whether the library which is specified in
the 'requirements.txt' of pack is loaded when a same name module is also specified in the
'requirements.txt' of st2, at a subprocess in ActionRunner.
To test above, this uses 'get_library_path.py' action in 'test_library_dependencies' pack.
This action returns file-path of imported module which is specified by 'module' parameter.
'''
pack_name = 'test_library_dependencies'
# Before calling action, this sets up virtualenv for test pack. This pack has
# requirements.txt wihch only writes 'six' module.
setup_pack_virtualenv(pack_name=pack_name)
self.assertTrue(os.path.exists(os.path.join(self.virtualenvs_path, pack_name)))
# This test suite expects that loaded six module is located under the virtualenv library,
# because 'six' is written in the requirements.txt of 'test_library_dependencies' pack.
(_, output, _) = self._run_action(pack_name, 'get_library_path.py', {'module': 'six'})
self.assertEqual(output['result'].find(self.virtualenvs_path), 0)
# Conversely, this expects that 'mock' module file-path is not under sandbox library,
# but the parent process's library path, because that is not under the pack's virtualenv.
(_, output, _) = self._run_action(pack_name, 'get_library_path.py', {'module': 'mock'})
self.assertEqual(output['result'].find(self.virtualenvs_path), -1)
# While a module which is in the pack's virtualenv library is specified at 'module'
# parameter of the action, this test suite expects that file-path under the parent's
# library is returned when 'sandbox' parameter of PythonRunner is False.
(_, output, _) = self._run_action(pack_name, 'get_library_path.py', {'module': 'six'},
{'_sandbox': False})
self.assertEqual(output['result'].find(self.virtualenvs_path), -1)
def main(argv):
_register_cli_opts()
# Parse CLI args, set up logging
common_setup(config=config,
setup_db=False,
register_mq_exchanges=False,
register_internal_trigger_types=False)
packs = cfg.CONF.pack
update = cfg.CONF.update
use_python3 = cfg.CONF.python3
proxy_config = get_and_set_proxy_config()
for pack in packs:
# Setup pack virtual environment
LOG.info('Setting up virtualenv for pack "%s"' % (pack))
setup_pack_virtualenv(pack_name=pack,
update=update,
logger=LOG,
proxy_config=proxy_config,
use_python3=use_python3,
no_download=True)
LOG.info('Successfully set up virtualenv for pack "%s"' % (pack))
return 0
def test_setup_pack_virtualenv_already_exists(self):
# Test a scenario where virtualenv already exists
pack_name = 'dummy_pack_1'
pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name)
# Verify virtualenv directory doesn't exist
self.assertFalse(os.path.exists(pack_virtualenv_dir))
# Create virtualenv
setup_pack_virtualenv(pack_name=pack_name,
update=False,
include_pip=False,
include_setuptools=False,
include_wheel=False)
# Verify that virtualenv has been created
self.assertVirtualenvExists(pack_virtualenv_dir)
# Re-create virtualenv
setup_pack_virtualenv(pack_name=pack_name,
update=False,
include_pip=False,
include_setuptools=False,
include_wheel=False)
# Verify virtrualenv is still there
self.assertVirtualenvExists(pack_virtualenv_dir)
def test_setup_virtualenv_update(self):
# Test a virtualenv update with pack which has requirements.txt
pack_name = 'dummy_pack_2'
pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name)
# Verify virtualenv directory doesn't exist
self.assertFalse(os.path.exists(pack_virtualenv_dir))
# Create virtualenv
setup_pack_virtualenv(pack_name=pack_name,
update=False,
include_setuptools=False,
include_wheel=False)
# Verify that virtualenv has been created
self.assertVirtualenvExists(pack_virtualenv_dir)
# Update it
setup_pack_virtualenv(pack_name=pack_name,
update=True,
include_setuptools=False,
include_wheel=False)
# Verify virtrualenv is still there
self.assertVirtualenvExists(pack_virtualenv_dir)
def setup_virtualenvs():
"""
Setup Python virtual environments for all the registered or the provided pack.
"""
pack_dir = cfg.CONF.register.pack
fail_on_failure = cfg.CONF.register.fail_on_failure
if pack_dir:
pack_name = os.path.basename(pack_dir)
pack_names = [pack_name]
else:
registrar = ResourceRegistrar()
pack_names = registrar.get_registered_packs()
setup_count = 0
for pack_name in pack_names:
try:
setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG)
except Exception as e:
exc_info = not fail_on_failure
LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e,
exc_info=exc_info)
if fail_on_failure:
raise e
else:
setup_count += 1
LOG.info('Setup virtualenv for %s pack.' % (setup_count))
def setup_virtualenvs():
"""
Setup Python virtual environments for all the registered or the provided pack.
"""
pack_dir = cfg.CONF.register.pack
fail_on_failure = cfg.CONF.register.fail_on_failure
if pack_dir:
pack_name = os.path.basename(pack_dir)
pack_names = [pack_name]
else:
registrar = ResourceRegistrar()
pack_names = registrar.get_registered_packs()
setup_count = 0
for pack_name in pack_names:
try:
setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG)
except Exception as e:
exc_info = not fail_on_failure
LOG.warning('Failed to setup virtualenv for pack "%s": %s',
pack_name,
e,
exc_info=exc_info)
if fail_on_failure:
raise e
else:
setup_count += 1
LOG.info('Setup virtualenv for %s pack.' % (setup_count))
def setup_virtualenvs(recreate_virtualenvs=False):
"""
Setup Python virtual environments for all the registered or the provided pack.
"""
LOG.info('=========================================================')
LOG.info('########### Setting up virtual environments #############')
LOG.info('=========================================================')
pack_dir = cfg.CONF.register.pack
fail_on_failure = not cfg.CONF.register.no_fail_on_failure
registrar = ResourceRegistrar()
if pack_dir:
pack_name = os.path.basename(pack_dir)
pack_names = [pack_name]
# 1. Register pack
registrar.register_pack(pack_name=pack_name, pack_dir=pack_dir)
else:
# 1. Register pack
base_dirs = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=base_dirs)
# 2. Retrieve available packs (aka packs which have been registered)
pack_names = registrar.get_registered_packs()
if recreate_virtualenvs:
"""
update = False:
this is more than an update of an existing virtualenv
the virtualenv itself will be removed & recreated
this is i.e. useful for updates to a newer Python release
"""
update = False
else:
"""
update = True:
only dependencies inside the virtualenv will be updated
"""
update = True
setup_count = 0
for pack_name in pack_names:
try:
setup_pack_virtualenv(pack_name=pack_name, update=update, logger=LOG)
except Exception as e:
exc_info = not fail_on_failure
LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e,
exc_info=exc_info)
if fail_on_failure:
raise e
else:
setup_count += 1
LOG.info('Setup virtualenv for %s pack(s).' % (setup_count))
def main(argv):
_register_cli_opts()
# Parse CLI args, set up logging
common_setup(
config=config,
setup_db=False,
register_mq_exchanges=False,
register_internal_trigger_types=False,
)
packs = cfg.CONF.pack
verify_ssl = cfg.CONF.verify_ssl
force = cfg.CONF.force
proxy_config = get_and_set_proxy_config()
for pack in packs:
# 1. Download the pack
LOG.info('Installing pack "%s"' % (pack))
result = download_pack(
pack=pack,
verify_ssl=verify_ssl,
force=force,
proxy_config=proxy_config,
force_permissions=True,
)
# Raw pack name excluding the version
pack_name = result[1]
success = result[2][0]
if success:
LOG.info('Successfully installed pack "%s"' % (pack_name))
else:
error = result[2][1]
LOG.error('Failed to install pack "%s": %s' % (pack_name, error))
sys.exit(2)
# 2. Setup pack virtual environment
LOG.info('Setting up virtualenv for pack "%s"' % (pack_name))
setup_pack_virtualenv(
pack_name=pack_name,
update=False,
logger=LOG,
proxy_config=proxy_config,
no_download=True,
)
LOG.info('Successfully set up virtualenv for pack "%s"' % (pack_name))
return 0
def test_setup_pack_virtualenv_doesnt_exist_yet(self):
# Test a fresh virtualenv creation
pack_name = 'dummy_pack_1'
pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name)
# Verify virtualenv directory doesn't exist
self.assertFalse(os.path.exists(pack_virtualenv_dir))
# Create virtualenv
# Note: This pack has no requirements
setup_pack_virtualenv(pack_name=pack_name, update=False)
# Verify that virtualenv has been created
self.assertVirtulenvExists(pack_virtualenv_dir)
def run(self, packs, update=False):
"""
:param packs: A list of packs to create the environment for.
:type: packs: ``list``
:param update: True to update dependencies inside the virtual environment.
:type update: ``bool``
"""
for pack_name in packs:
setup_pack_virtualenv(pack_name=pack_name, update=update, logger=self.logger)
message = ('Successfuly set up virtualenv for the following packs: %s' %
(', '.join(packs)))
return message
def test_setup_pack_virtualenv_doesnt_exist_yet(self):
# Test a fresh virtualenv creation
pack_name = 'dummy_pack_1'
pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name)
# Verify virtualenv directory doesn't exist
self.assertFalse(os.path.exists(pack_virtualenv_dir))
# Create virtualenv
# Note: This pack has no requirements
setup_pack_virtualenv(pack_name=pack_name, update=False)
# Verify that virtualenv has been created
self.assertVirtulenvExists(pack_virtualenv_dir)
def test_setup_virtualenv_invalid_dependency_in_requirements_file(self):
pack_name = 'pack_invalid_requirements'
pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name)
# Verify virtualenv directory doesn't exist
self.assertFalse(os.path.exists(pack_virtualenv_dir))
# Try to create virtualenv, assert that it fails
try:
setup_pack_virtualenv(pack_name=pack_name, update=False)
except Exception as e:
self.assertTrue('Failed to install requirements from' in str(e))
self.assertTrue('No matching distribution found for someinvalidname' in str(e))
else:
self.fail('Exception not thrown')
def test_setup_virtualenv_invalid_dependency_in_requirements_file(self):
pack_name = 'pack_invalid_requirements'
pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name)
# Verify virtualenv directory doesn't exist
self.assertFalse(os.path.exists(pack_virtualenv_dir))
# Try to create virtualenv, assert that it fails
try:
setup_pack_virtualenv(pack_name=pack_name, update=False,
include_setuptools=False, include_wheel=False)
except Exception as e:
self.assertTrue('Failed to install requirements from' in str(e))
self.assertTrue('No matching distribution found for someinvalidname' in str(e))
else:
self.fail('Exception not thrown')
def test_setup_pack_virtualenv_use_python3_binary(self, mock_run_command):
mock_run_command.return_value = 0, '', ''
cfg.CONF.set_override(name='python_binary',
group='actionrunner',
override='/usr/bin/python2.7')
cfg.CONF.set_override(name='python3_binary',
group='actionrunner',
override='/usr/bin/python3')
pack_name = 'dummy_pack_2'
# Python 2
setup_pack_virtualenv(pack_name=pack_name,
update=False,
include_setuptools=False,
include_wheel=False,
use_python3=False)
actual_cmd = mock_run_command.call_args_list[0][1]['cmd']
actual_cmd = ' '.join(actual_cmd)
self.assertEqual(mock_run_command.call_count, 2)
self.assertIn('-p /usr/bin/python2.7', actual_cmd)
mock_run_command.reset_mock()
# Python 3
setup_pack_virtualenv(pack_name=pack_name,
update=False,
include_setuptools=False,
include_wheel=False,
use_python3=True)
self.assertEqual(mock_run_command.call_count, 3)
actual_cmd = mock_run_command.call_args_list[0][1]['cmd']
actual_cmd = ' '.join(actual_cmd)
self.assertIn('-p /usr/bin/python3', actual_cmd)
actual_cmd = mock_run_command.call_args_list[1][1]['cmd']
actual_cmd = ' '.join(actual_cmd)
self.assertIn('pip install pyyaml', actual_cmd)
actual_cmd = mock_run_command.call_args_list[2][1]['cmd']
actual_cmd = ' '.join(actual_cmd)
self.assertIn('pip install', actual_cmd)
def run(self, packs, update=False):
"""
:param packs: A list of packs to create the environment for.
:type: packs: ``list``
:param update: True to update dependencies inside the virtual environment.
:type update: ``bool``
"""
for pack_name in packs:
setup_pack_virtualenv(pack_name=pack_name,
update=update,
logger=self.logger)
message = (
'Successfuly set up virtualenv for the following packs: %s' %
(', '.join(packs)))
return message
def run(self, packs, update=False, python3=False, no_download=True):
"""
:param packs: A list of packs to create the environment for.
:type: packs: ``list``
:param update: True to update dependencies inside the virtual environment.
:type update: ``bool``
"""
for pack_name in packs:
setup_pack_virtualenv(pack_name=pack_name, update=update, logger=self.logger,
proxy_config=self.proxy_config, use_python3=python3,
no_download=no_download)
message = ('Successfully set up virtualenv for the following packs: %s' %
(', '.join(packs)))
return message
def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
"""
This test checks priority of loading library, whether the library which is specified in
the 'requirements.txt' of pack is loaded when a same name module is also specified in the
'requirements.txt' of st2, at a subprocess in ActionRunner.
To test above, this uses 'get_library_path.py' action in 'test_library_dependencies' pack.
This action returns file-path of imported module which is specified by 'module' parameter.
"""
pack_name = "test_library_dependencies"
# Before calling action, this sets up virtualenv for test pack. This pack has
# requirements.txt wihch only writes 'six' module.
setup_pack_virtualenv(pack_name=pack_name)
self.assertTrue(
os.path.exists(os.path.join(self.virtualenvs_path, pack_name)))
# This test suite expects that loaded six module is located under the virtualenv library,
# because 'six' is written in the requirements.txt of 'test_library_dependencies' pack.
(_, output, _) = self._run_action(pack_name, "get_library_path.py",
{"module": "six"})
# FIXME: This test fails if system site-packages has six because
# it won't get installed in the virtualenv (w/ --system-site-packages)
# system site-packages is never from a virtualenv.
# Travis has python installed in /opt/python/3.6.7
# with a no-system-site-packages virtualenv at /home/travis/virtualenv/python3.6.7
# GitHub Actions python is in /opt/hostedtoolcache/Python/3.6.13/x64/
# But ther isn't a virtualenv, so when we pip installed `virtualenv`,
# (which depends on, and therefore installs `six`)
# we installed it in system-site-packages not an intermediate virtualenv
self.assertEqual(output["result"].find(self.virtualenvs_path), 0)
# Conversely, this expects that 'mock' module file-path is not under sandbox library,
# but the parent process's library path, because that is not under the pack's virtualenv.
(_, output, _) = self._run_action(pack_name, "get_library_path.py",
{"module": "mock"})
self.assertEqual(output["result"].find(self.virtualenvs_path), -1)
# While a module which is in the pack's virtualenv library is specified at 'module'
# parameter of the action, this test suite expects that file-path under the parent's
# library is returned when 'sandbox' parameter of PythonRunner is False.
(_, output, _) = self._run_action(pack_name, "get_library_path.py",
{"module": "six"},
{"_sandbox": False})
self.assertEqual(output["result"].find(self.virtualenvs_path), -1)
def setup_virtualenvs():
"""
Setup Python virtual environments for all the registered or the provided pack.
"""
LOG.info('=========================================================')
LOG.info('########### Setting up virtual environments #############')
LOG.info('=========================================================')
pack_dir = cfg.CONF.register.pack
fail_on_failure = cfg.CONF.register.fail_on_failure
registrar = ResourceRegistrar()
if pack_dir:
pack_name = os.path.basename(pack_dir)
pack_names = [pack_name]
# 1. Register pack
registrar.register_pack(pack_name=pack_name, pack_dir=pack_dir)
else:
# 1. Register pack
base_dirs = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=base_dirs)
# 2. Retrieve available packs (aka packs which have been registered)
pack_names = registrar.get_registered_packs()
setup_count = 0
for pack_name in pack_names:
try:
setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG)
except Exception as e:
exc_info = not fail_on_failure
LOG.warning('Failed to setup virtualenv for pack "%s": %s',
pack_name,
e,
exc_info=exc_info)
if fail_on_failure:
raise e
else:
setup_count += 1
LOG.info('Setup virtualenv for %s pack(s).' % (setup_count))
def test_setup_virtualenv_update(self):
# Test a virtualenv update with pack which has requirements.txt
pack_name = 'dummy_pack_2'
pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name)
# Verify virtualenv directory doesn't exist
self.assertFalse(os.path.exists(pack_virtualenv_dir))
# Create virtualenv
setup_pack_virtualenv(pack_name=pack_name, update=False)
# Verify that virtualenv has been created
self.assertVirtulenvExists(pack_virtualenv_dir)
# Update it
setup_pack_virtualenv(pack_name=pack_name, update=True)
# Verify virtrualenv is still there
self.assertVirtulenvExists(pack_virtualenv_dir)
def test_setup_pack_virtualenv_already_exists(self):
# Test a scenario where virtualenv already exists
pack_name = 'dummy_pack_1'
pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name)
# Verify virtualenv directory doesn't exist
self.assertFalse(os.path.exists(pack_virtualenv_dir))
# Create virtualenv
setup_pack_virtualenv(pack_name=pack_name, update=False)
# Verify that virtualenv has been created
self.assertVirtulenvExists(pack_virtualenv_dir)
# Re-create virtualenv
setup_pack_virtualenv(pack_name=pack_name, update=False)
# Verify virtrualenv is still there
self.assertVirtulenvExists(pack_virtualenv_dir)
def setup_virtualenvs():
"""
Setup Python virtual environments for all the registered or the provided pack.
"""
LOG.info('=========================================================')
LOG.info('########### Setting up virtual environments #############')
LOG.info('=========================================================')
pack_dir = cfg.CONF.register.pack
fail_on_failure = cfg.CONF.register.fail_on_failure
registrar = ResourceRegistrar()
if pack_dir:
pack_name = os.path.basename(pack_dir)
pack_names = [pack_name]
# 1. Register pack
registrar.register_pack(pack_name=pack_name, pack_dir=pack_dir)
else:
# 1. Register pack
base_dirs = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=base_dirs)
# 2. Retrieve available packs (aka packs which have been registered)
pack_names = registrar.get_registered_packs()
setup_count = 0
for pack_name in pack_names:
try:
setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG)
except Exception as e:
exc_info = not fail_on_failure
LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e,
exc_info=exc_info)
if fail_on_failure:
raise e
else:
setup_count += 1
LOG.info('Setup virtualenv for %s pack(s).' % (setup_count))
def test_setup_pack_virtualenv_use_python3_binary(self, mock_run_command):
mock_run_command.return_value = 0, '', ''
cfg.CONF.set_override(name='python_binary', group='actionrunner',
override='/usr/bin/python2.7')
cfg.CONF.set_override(name='python3_binary', group='actionrunner',
override='/usr/bin/python3')
pack_name = 'dummy_pack_2'
# Python 2
setup_pack_virtualenv(pack_name=pack_name, update=False,
include_setuptools=False, include_wheel=False,
use_python3=False)
actual_cmd = mock_run_command.call_args_list[0][1]['cmd']
actual_cmd = ' '.join(actual_cmd)
self.assertEqual(mock_run_command.call_count, 2)
self.assertTrue('-p /usr/bin/python2.7' in actual_cmd)
mock_run_command.reset_mock()
# Python 3
setup_pack_virtualenv(pack_name=pack_name, update=False,
include_setuptools=False, include_wheel=False,
use_python3=True)
self.assertEqual(mock_run_command.call_count, 3)
actual_cmd = mock_run_command.call_args_list[0][1]['cmd']
actual_cmd = ' '.join(actual_cmd)
self.assertTrue('-p /usr/bin/python3' in actual_cmd)
actual_cmd = mock_run_command.call_args_list[1][1]['cmd']
actual_cmd = ' '.join(actual_cmd)
self.assertTrue('pip install pyyaml' in actual_cmd)
actual_cmd = mock_run_command.call_args_list[2][1]['cmd']
actual_cmd = ' '.join(actual_cmd)
self.assertTrue('pip install' in actual_cmd)
def run(self, packs, update=False, no_download=True):
"""
:param packs: A list of packs to create the environment for.
:type: packs: ``list``
:param update: True to update dependencies inside the virtual environment.
:type update: ``bool``
"""
for pack_name in packs:
setup_pack_virtualenv(
pack_name=pack_name,
update=update,
logger=self.logger,
proxy_config=self.proxy_config,
no_download=no_download,
)
message = "Successfully set up virtualenv for the following packs: %s" % (
", ".join(packs))
return message
def main(argv):
_register_cli_opts()
# Parse CLI args, set up logging
common_setup(config=config, setup_db=False, register_mq_exchanges=False,
register_internal_trigger_types=False)
packs = cfg.CONF.pack
update = cfg.CONF.update
use_python3 = cfg.CONF.python3
proxy_config = get_and_set_proxy_config()
for pack in packs:
# Setup pack virtual environment
LOG.info('Setting up virtualenv for pack "%s"' % (pack))
setup_pack_virtualenv(pack_name=pack, update=update, logger=LOG,
proxy_config=proxy_config, use_python3=use_python3,
no_download=True)
LOG.info('Successfully set up virtualenv for pack "%s"' % (pack))
return 0
def main(argv):
_register_cli_opts()
# Parse CLI args, set up logging
common_setup(config=config, setup_db=False, register_mq_exchanges=False,
register_internal_trigger_types=False)
packs = cfg.CONF.pack
verify_ssl = cfg.CONF.verify_ssl
force = cfg.CONF.force
use_python3 = cfg.CONF.use_python3
proxy_config = get_and_set_proxy_config()
for pack in packs:
# 1. Download the pack
LOG.info('Installing pack "%s"' % (pack))
result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force,
proxy_config=proxy_config, force_permissions=True)
# Raw pack name excluding the version
pack_name = result[1]
success = result[2][0]
if success:
LOG.info('Successfully installed pack "%s"' % (pack_name))
else:
error = result[2][1]
LOG.error('Failed to install pack "%s": %s' % (pack_name, error))
sys.exit(2)
# 2. Setup pack virtual environment
LOG.info('Setting up virtualenv for pack "%s"' % (pack_name))
setup_pack_virtualenv(pack_name=pack_name, update=False, logger=LOG,
proxy_config=proxy_config, use_python3=use_python3,
no_download=True)
LOG.info('Successfully set up virtualenv for pack "%s"' % (pack_name))
return 0
def main():
common_setup(config=config, setup_db=False, register_mq_exchanges=False)
setup_pack_virtualenv(cfg.CONF.pack, logger=LOG)
common_teardown()