def test_priority_of_loading_library_after_setup_pack_virtualenv(self): ''' This test checks priority of loading library, whether the library which is specified in the 'requirements.txt' of pack is loaded when a same name module is also specified in the 'requirements.txt' of st2, at a subprocess in ActionRunner. To test above, this uses 'get_library_path.py' action in 'test_library_dependencies' pack. This action returns file-path of imported module which is specified by 'module' parameter. ''' pack_name = 'test_library_dependencies' # Before calling action, this sets up virtualenv for test pack. This pack has # requirements.txt wihch only writes 'six' module. setup_pack_virtualenv(pack_name=pack_name) self.assertTrue(os.path.exists(os.path.join(self.virtualenvs_path, pack_name))) # This test suite expects that loaded six module is located under the virtualenv library, # because 'six' is written in the requirements.txt of 'test_library_dependencies' pack. (_, output, _) = self._run_action(pack_name, 'get_library_path.py', {'module': 'six'}) self.assertEqual(output['result'].find(self.virtualenvs_path), 0) # Conversely, this expects that 'mock' module file-path is not under sandbox library, # but the parent process's library path, because that is not under the pack's virtualenv. (_, output, _) = self._run_action(pack_name, 'get_library_path.py', {'module': 'mock'}) self.assertEqual(output['result'].find(self.virtualenvs_path), -1) # While a module which is in the pack's virtualenv library is specified at 'module' # parameter of the action, this test suite expects that file-path under the parent's # library is returned when 'sandbox' parameter of PythonRunner is False. (_, output, _) = self._run_action(pack_name, 'get_library_path.py', {'module': 'six'}, {'_sandbox': False}) self.assertEqual(output['result'].find(self.virtualenvs_path), -1)
def main(argv): _register_cli_opts() # Parse CLI args, set up logging common_setup(config=config, setup_db=False, register_mq_exchanges=False, register_internal_trigger_types=False) packs = cfg.CONF.pack update = cfg.CONF.update use_python3 = cfg.CONF.python3 proxy_config = get_and_set_proxy_config() for pack in packs: # Setup pack virtual environment LOG.info('Setting up virtualenv for pack "%s"' % (pack)) setup_pack_virtualenv(pack_name=pack, update=update, logger=LOG, proxy_config=proxy_config, use_python3=use_python3, no_download=True) LOG.info('Successfully set up virtualenv for pack "%s"' % (pack)) return 0
def test_setup_pack_virtualenv_already_exists(self): # Test a scenario where virtualenv already exists pack_name = 'dummy_pack_1' pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name) # Verify virtualenv directory doesn't exist self.assertFalse(os.path.exists(pack_virtualenv_dir)) # Create virtualenv setup_pack_virtualenv(pack_name=pack_name, update=False, include_pip=False, include_setuptools=False, include_wheel=False) # Verify that virtualenv has been created self.assertVirtualenvExists(pack_virtualenv_dir) # Re-create virtualenv setup_pack_virtualenv(pack_name=pack_name, update=False, include_pip=False, include_setuptools=False, include_wheel=False) # Verify virtrualenv is still there self.assertVirtualenvExists(pack_virtualenv_dir)
def test_setup_virtualenv_update(self): # Test a virtualenv update with pack which has requirements.txt pack_name = 'dummy_pack_2' pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name) # Verify virtualenv directory doesn't exist self.assertFalse(os.path.exists(pack_virtualenv_dir)) # Create virtualenv setup_pack_virtualenv(pack_name=pack_name, update=False, include_setuptools=False, include_wheel=False) # Verify that virtualenv has been created self.assertVirtualenvExists(pack_virtualenv_dir) # Update it setup_pack_virtualenv(pack_name=pack_name, update=True, include_setuptools=False, include_wheel=False) # Verify virtrualenv is still there self.assertVirtualenvExists(pack_virtualenv_dir)
def setup_virtualenvs(): """ Setup Python virtual environments for all the registered or the provided pack. """ pack_dir = cfg.CONF.register.pack fail_on_failure = cfg.CONF.register.fail_on_failure if pack_dir: pack_name = os.path.basename(pack_dir) pack_names = [pack_name] else: registrar = ResourceRegistrar() pack_names = registrar.get_registered_packs() setup_count = 0 for pack_name in pack_names: try: setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG) except Exception as e: exc_info = not fail_on_failure LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e, exc_info=exc_info) if fail_on_failure: raise e else: setup_count += 1 LOG.info('Setup virtualenv for %s pack.' % (setup_count))
def setup_virtualenvs(): """ Setup Python virtual environments for all the registered or the provided pack. """ pack_dir = cfg.CONF.register.pack fail_on_failure = cfg.CONF.register.fail_on_failure if pack_dir: pack_name = os.path.basename(pack_dir) pack_names = [pack_name] else: registrar = ResourceRegistrar() pack_names = registrar.get_registered_packs() setup_count = 0 for pack_name in pack_names: try: setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG) except Exception as e: exc_info = not fail_on_failure LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e, exc_info=exc_info) if fail_on_failure: raise e else: setup_count += 1 LOG.info('Setup virtualenv for %s pack.' % (setup_count))
def setup_virtualenvs(recreate_virtualenvs=False): """ Setup Python virtual environments for all the registered or the provided pack. """ LOG.info('=========================================================') LOG.info('########### Setting up virtual environments #############') LOG.info('=========================================================') pack_dir = cfg.CONF.register.pack fail_on_failure = not cfg.CONF.register.no_fail_on_failure registrar = ResourceRegistrar() if pack_dir: pack_name = os.path.basename(pack_dir) pack_names = [pack_name] # 1. Register pack registrar.register_pack(pack_name=pack_name, pack_dir=pack_dir) else: # 1. Register pack base_dirs = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=base_dirs) # 2. Retrieve available packs (aka packs which have been registered) pack_names = registrar.get_registered_packs() if recreate_virtualenvs: """ update = False: this is more than an update of an existing virtualenv the virtualenv itself will be removed & recreated this is i.e. useful for updates to a newer Python release """ update = False else: """ update = True: only dependencies inside the virtualenv will be updated """ update = True setup_count = 0 for pack_name in pack_names: try: setup_pack_virtualenv(pack_name=pack_name, update=update, logger=LOG) except Exception as e: exc_info = not fail_on_failure LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e, exc_info=exc_info) if fail_on_failure: raise e else: setup_count += 1 LOG.info('Setup virtualenv for %s pack(s).' % (setup_count))
def main(argv): _register_cli_opts() # Parse CLI args, set up logging common_setup( config=config, setup_db=False, register_mq_exchanges=False, register_internal_trigger_types=False, ) packs = cfg.CONF.pack verify_ssl = cfg.CONF.verify_ssl force = cfg.CONF.force proxy_config = get_and_set_proxy_config() for pack in packs: # 1. Download the pack LOG.info('Installing pack "%s"' % (pack)) result = download_pack( pack=pack, verify_ssl=verify_ssl, force=force, proxy_config=proxy_config, force_permissions=True, ) # Raw pack name excluding the version pack_name = result[1] success = result[2][0] if success: LOG.info('Successfully installed pack "%s"' % (pack_name)) else: error = result[2][1] LOG.error('Failed to install pack "%s": %s' % (pack_name, error)) sys.exit(2) # 2. Setup pack virtual environment LOG.info('Setting up virtualenv for pack "%s"' % (pack_name)) setup_pack_virtualenv( pack_name=pack_name, update=False, logger=LOG, proxy_config=proxy_config, no_download=True, ) LOG.info('Successfully set up virtualenv for pack "%s"' % (pack_name)) return 0
def test_setup_pack_virtualenv_doesnt_exist_yet(self): # Test a fresh virtualenv creation pack_name = 'dummy_pack_1' pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name) # Verify virtualenv directory doesn't exist self.assertFalse(os.path.exists(pack_virtualenv_dir)) # Create virtualenv # Note: This pack has no requirements setup_pack_virtualenv(pack_name=pack_name, update=False) # Verify that virtualenv has been created self.assertVirtulenvExists(pack_virtualenv_dir)
def run(self, packs, update=False): """ :param packs: A list of packs to create the environment for. :type: packs: ``list`` :param update: True to update dependencies inside the virtual environment. :type update: ``bool`` """ for pack_name in packs: setup_pack_virtualenv(pack_name=pack_name, update=update, logger=self.logger) message = ('Successfuly set up virtualenv for the following packs: %s' % (', '.join(packs))) return message
def test_setup_pack_virtualenv_doesnt_exist_yet(self): # Test a fresh virtualenv creation pack_name = 'dummy_pack_1' pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name) # Verify virtualenv directory doesn't exist self.assertFalse(os.path.exists(pack_virtualenv_dir)) # Create virtualenv # Note: This pack has no requirements setup_pack_virtualenv(pack_name=pack_name, update=False) # Verify that virtualenv has been created self.assertVirtulenvExists(pack_virtualenv_dir)
def test_setup_virtualenv_invalid_dependency_in_requirements_file(self): pack_name = 'pack_invalid_requirements' pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name) # Verify virtualenv directory doesn't exist self.assertFalse(os.path.exists(pack_virtualenv_dir)) # Try to create virtualenv, assert that it fails try: setup_pack_virtualenv(pack_name=pack_name, update=False) except Exception as e: self.assertTrue('Failed to install requirements from' in str(e)) self.assertTrue('No matching distribution found for someinvalidname' in str(e)) else: self.fail('Exception not thrown')
def test_setup_virtualenv_invalid_dependency_in_requirements_file(self): pack_name = 'pack_invalid_requirements' pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name) # Verify virtualenv directory doesn't exist self.assertFalse(os.path.exists(pack_virtualenv_dir)) # Try to create virtualenv, assert that it fails try: setup_pack_virtualenv(pack_name=pack_name, update=False, include_setuptools=False, include_wheel=False) except Exception as e: self.assertTrue('Failed to install requirements from' in str(e)) self.assertTrue('No matching distribution found for someinvalidname' in str(e)) else: self.fail('Exception not thrown')
def test_setup_pack_virtualenv_use_python3_binary(self, mock_run_command): mock_run_command.return_value = 0, '', '' cfg.CONF.set_override(name='python_binary', group='actionrunner', override='/usr/bin/python2.7') cfg.CONF.set_override(name='python3_binary', group='actionrunner', override='/usr/bin/python3') pack_name = 'dummy_pack_2' # Python 2 setup_pack_virtualenv(pack_name=pack_name, update=False, include_setuptools=False, include_wheel=False, use_python3=False) actual_cmd = mock_run_command.call_args_list[0][1]['cmd'] actual_cmd = ' '.join(actual_cmd) self.assertEqual(mock_run_command.call_count, 2) self.assertIn('-p /usr/bin/python2.7', actual_cmd) mock_run_command.reset_mock() # Python 3 setup_pack_virtualenv(pack_name=pack_name, update=False, include_setuptools=False, include_wheel=False, use_python3=True) self.assertEqual(mock_run_command.call_count, 3) actual_cmd = mock_run_command.call_args_list[0][1]['cmd'] actual_cmd = ' '.join(actual_cmd) self.assertIn('-p /usr/bin/python3', actual_cmd) actual_cmd = mock_run_command.call_args_list[1][1]['cmd'] actual_cmd = ' '.join(actual_cmd) self.assertIn('pip install pyyaml', actual_cmd) actual_cmd = mock_run_command.call_args_list[2][1]['cmd'] actual_cmd = ' '.join(actual_cmd) self.assertIn('pip install', actual_cmd)
def run(self, packs, update=False): """ :param packs: A list of packs to create the environment for. :type: packs: ``list`` :param update: True to update dependencies inside the virtual environment. :type update: ``bool`` """ for pack_name in packs: setup_pack_virtualenv(pack_name=pack_name, update=update, logger=self.logger) message = ( 'Successfuly set up virtualenv for the following packs: %s' % (', '.join(packs))) return message
def run(self, packs, update=False, python3=False, no_download=True): """ :param packs: A list of packs to create the environment for. :type: packs: ``list`` :param update: True to update dependencies inside the virtual environment. :type update: ``bool`` """ for pack_name in packs: setup_pack_virtualenv(pack_name=pack_name, update=update, logger=self.logger, proxy_config=self.proxy_config, use_python3=python3, no_download=no_download) message = ('Successfully set up virtualenv for the following packs: %s' % (', '.join(packs))) return message
def test_priority_of_loading_library_after_setup_pack_virtualenv(self): """ This test checks priority of loading library, whether the library which is specified in the 'requirements.txt' of pack is loaded when a same name module is also specified in the 'requirements.txt' of st2, at a subprocess in ActionRunner. To test above, this uses 'get_library_path.py' action in 'test_library_dependencies' pack. This action returns file-path of imported module which is specified by 'module' parameter. """ pack_name = "test_library_dependencies" # Before calling action, this sets up virtualenv for test pack. This pack has # requirements.txt wihch only writes 'six' module. setup_pack_virtualenv(pack_name=pack_name) self.assertTrue( os.path.exists(os.path.join(self.virtualenvs_path, pack_name))) # This test suite expects that loaded six module is located under the virtualenv library, # because 'six' is written in the requirements.txt of 'test_library_dependencies' pack. (_, output, _) = self._run_action(pack_name, "get_library_path.py", {"module": "six"}) # FIXME: This test fails if system site-packages has six because # it won't get installed in the virtualenv (w/ --system-site-packages) # system site-packages is never from a virtualenv. # Travis has python installed in /opt/python/3.6.7 # with a no-system-site-packages virtualenv at /home/travis/virtualenv/python3.6.7 # GitHub Actions python is in /opt/hostedtoolcache/Python/3.6.13/x64/ # But ther isn't a virtualenv, so when we pip installed `virtualenv`, # (which depends on, and therefore installs `six`) # we installed it in system-site-packages not an intermediate virtualenv self.assertEqual(output["result"].find(self.virtualenvs_path), 0) # Conversely, this expects that 'mock' module file-path is not under sandbox library, # but the parent process's library path, because that is not under the pack's virtualenv. (_, output, _) = self._run_action(pack_name, "get_library_path.py", {"module": "mock"}) self.assertEqual(output["result"].find(self.virtualenvs_path), -1) # While a module which is in the pack's virtualenv library is specified at 'module' # parameter of the action, this test suite expects that file-path under the parent's # library is returned when 'sandbox' parameter of PythonRunner is False. (_, output, _) = self._run_action(pack_name, "get_library_path.py", {"module": "six"}, {"_sandbox": False}) self.assertEqual(output["result"].find(self.virtualenvs_path), -1)
def setup_virtualenvs(): """ Setup Python virtual environments for all the registered or the provided pack. """ LOG.info('=========================================================') LOG.info('########### Setting up virtual environments #############') LOG.info('=========================================================') pack_dir = cfg.CONF.register.pack fail_on_failure = cfg.CONF.register.fail_on_failure registrar = ResourceRegistrar() if pack_dir: pack_name = os.path.basename(pack_dir) pack_names = [pack_name] # 1. Register pack registrar.register_pack(pack_name=pack_name, pack_dir=pack_dir) else: # 1. Register pack base_dirs = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=base_dirs) # 2. Retrieve available packs (aka packs which have been registered) pack_names = registrar.get_registered_packs() setup_count = 0 for pack_name in pack_names: try: setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG) except Exception as e: exc_info = not fail_on_failure LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e, exc_info=exc_info) if fail_on_failure: raise e else: setup_count += 1 LOG.info('Setup virtualenv for %s pack(s).' % (setup_count))
def test_setup_virtualenv_update(self): # Test a virtualenv update with pack which has requirements.txt pack_name = 'dummy_pack_2' pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name) # Verify virtualenv directory doesn't exist self.assertFalse(os.path.exists(pack_virtualenv_dir)) # Create virtualenv setup_pack_virtualenv(pack_name=pack_name, update=False) # Verify that virtualenv has been created self.assertVirtulenvExists(pack_virtualenv_dir) # Update it setup_pack_virtualenv(pack_name=pack_name, update=True) # Verify virtrualenv is still there self.assertVirtulenvExists(pack_virtualenv_dir)
def test_setup_pack_virtualenv_already_exists(self): # Test a scenario where virtualenv already exists pack_name = 'dummy_pack_1' pack_virtualenv_dir = os.path.join(self.virtualenvs_path, pack_name) # Verify virtualenv directory doesn't exist self.assertFalse(os.path.exists(pack_virtualenv_dir)) # Create virtualenv setup_pack_virtualenv(pack_name=pack_name, update=False) # Verify that virtualenv has been created self.assertVirtulenvExists(pack_virtualenv_dir) # Re-create virtualenv setup_pack_virtualenv(pack_name=pack_name, update=False) # Verify virtrualenv is still there self.assertVirtulenvExists(pack_virtualenv_dir)
def setup_virtualenvs(): """ Setup Python virtual environments for all the registered or the provided pack. """ LOG.info('=========================================================') LOG.info('########### Setting up virtual environments #############') LOG.info('=========================================================') pack_dir = cfg.CONF.register.pack fail_on_failure = cfg.CONF.register.fail_on_failure registrar = ResourceRegistrar() if pack_dir: pack_name = os.path.basename(pack_dir) pack_names = [pack_name] # 1. Register pack registrar.register_pack(pack_name=pack_name, pack_dir=pack_dir) else: # 1. Register pack base_dirs = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=base_dirs) # 2. Retrieve available packs (aka packs which have been registered) pack_names = registrar.get_registered_packs() setup_count = 0 for pack_name in pack_names: try: setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG) except Exception as e: exc_info = not fail_on_failure LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e, exc_info=exc_info) if fail_on_failure: raise e else: setup_count += 1 LOG.info('Setup virtualenv for %s pack(s).' % (setup_count))
def test_setup_pack_virtualenv_use_python3_binary(self, mock_run_command): mock_run_command.return_value = 0, '', '' cfg.CONF.set_override(name='python_binary', group='actionrunner', override='/usr/bin/python2.7') cfg.CONF.set_override(name='python3_binary', group='actionrunner', override='/usr/bin/python3') pack_name = 'dummy_pack_2' # Python 2 setup_pack_virtualenv(pack_name=pack_name, update=False, include_setuptools=False, include_wheel=False, use_python3=False) actual_cmd = mock_run_command.call_args_list[0][1]['cmd'] actual_cmd = ' '.join(actual_cmd) self.assertEqual(mock_run_command.call_count, 2) self.assertTrue('-p /usr/bin/python2.7' in actual_cmd) mock_run_command.reset_mock() # Python 3 setup_pack_virtualenv(pack_name=pack_name, update=False, include_setuptools=False, include_wheel=False, use_python3=True) self.assertEqual(mock_run_command.call_count, 3) actual_cmd = mock_run_command.call_args_list[0][1]['cmd'] actual_cmd = ' '.join(actual_cmd) self.assertTrue('-p /usr/bin/python3' in actual_cmd) actual_cmd = mock_run_command.call_args_list[1][1]['cmd'] actual_cmd = ' '.join(actual_cmd) self.assertTrue('pip install pyyaml' in actual_cmd) actual_cmd = mock_run_command.call_args_list[2][1]['cmd'] actual_cmd = ' '.join(actual_cmd) self.assertTrue('pip install' in actual_cmd)
def run(self, packs, update=False, no_download=True): """ :param packs: A list of packs to create the environment for. :type: packs: ``list`` :param update: True to update dependencies inside the virtual environment. :type update: ``bool`` """ for pack_name in packs: setup_pack_virtualenv( pack_name=pack_name, update=update, logger=self.logger, proxy_config=self.proxy_config, no_download=no_download, ) message = "Successfully set up virtualenv for the following packs: %s" % ( ", ".join(packs)) return message
def main(argv): _register_cli_opts() # Parse CLI args, set up logging common_setup(config=config, setup_db=False, register_mq_exchanges=False, register_internal_trigger_types=False) packs = cfg.CONF.pack update = cfg.CONF.update use_python3 = cfg.CONF.python3 proxy_config = get_and_set_proxy_config() for pack in packs: # Setup pack virtual environment LOG.info('Setting up virtualenv for pack "%s"' % (pack)) setup_pack_virtualenv(pack_name=pack, update=update, logger=LOG, proxy_config=proxy_config, use_python3=use_python3, no_download=True) LOG.info('Successfully set up virtualenv for pack "%s"' % (pack)) return 0
def main(argv): _register_cli_opts() # Parse CLI args, set up logging common_setup(config=config, setup_db=False, register_mq_exchanges=False, register_internal_trigger_types=False) packs = cfg.CONF.pack verify_ssl = cfg.CONF.verify_ssl force = cfg.CONF.force use_python3 = cfg.CONF.use_python3 proxy_config = get_and_set_proxy_config() for pack in packs: # 1. Download the pack LOG.info('Installing pack "%s"' % (pack)) result = download_pack(pack=pack, verify_ssl=verify_ssl, force=force, proxy_config=proxy_config, force_permissions=True) # Raw pack name excluding the version pack_name = result[1] success = result[2][0] if success: LOG.info('Successfully installed pack "%s"' % (pack_name)) else: error = result[2][1] LOG.error('Failed to install pack "%s": %s' % (pack_name, error)) sys.exit(2) # 2. Setup pack virtual environment LOG.info('Setting up virtualenv for pack "%s"' % (pack_name)) setup_pack_virtualenv(pack_name=pack_name, update=False, logger=LOG, proxy_config=proxy_config, use_python3=use_python3, no_download=True) LOG.info('Successfully set up virtualenv for pack "%s"' % (pack_name)) return 0
def main(): common_setup(config=config, setup_db=False, register_mq_exchanges=False) setup_pack_virtualenv(cfg.CONF.pack, logger=LOG) common_teardown()