def test_register_pack_pack_stackstorm_version_and_future_parameters(self): # Verify DB is empty pack_dbs = Pack.get_all() self.assertEqual(len(pack_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_9': PACK_PATH_9} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Dependencies, stackstorm_version and future values pack_db = Pack.get_by_name('dummy_pack_9_deps') self.assertEqual(pack_db.dependencies, ['core=0.2.0']) self.assertEqual(pack_db.stackstorm_version, '>=1.6.0, <2.2.0') self.assertEqual(pack_db.system, {'centos': {'foo': '>= 1.0'}}) self.assertEqual(pack_db.python_versions, ['2', '3']) # Note: We only store parameters which are defined in the schema, all other custom user # defined attributes are ignored self.assertTrue(not hasattr(pack_db, 'future')) self.assertTrue(not hasattr(pack_db, 'this')) # Wrong characters in the required st2 version expected_msg = "'wrongstackstormversion' does not match" self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_10)
def setup_virtualenvs(): """ Setup Python virtual environments for all the registered or the provided pack. """ pack_dir = cfg.CONF.register.pack fail_on_failure = cfg.CONF.register.fail_on_failure if pack_dir: pack_name = os.path.basename(pack_dir) pack_names = [pack_name] else: registrar = ResourceRegistrar() pack_names = registrar.get_registered_packs() setup_count = 0 for pack_name in pack_names: try: setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG) except Exception as e: exc_info = not fail_on_failure LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e, exc_info=exc_info) if fail_on_failure: raise e else: setup_count += 1 LOG.info('Setup virtualenv for %s pack.' % (setup_count))
def _register_packs(self): """ Register all the packs inside the fixtures directory. """ registrar = ResourceRegistrar(use_pack_cache=False) registrar.register_packs(base_dirs=get_packs_base_paths())
def test_register_pack_pack_stackstorm_version_and_future_parameters(self): # Verify DB is empty pack_dbs = Pack.get_all() self.assertEqual(len(pack_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_9': PACK_PATH_9 } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Dependencies, stackstorm_version and future values pack_db = Pack.get_by_name('dummy_pack_9_deps') self.assertEqual(pack_db.dependencies, ['core=0.2.0']) self.assertEqual(pack_db.stackstorm_version, '>=1.6.0, <2.2.0') self.assertEqual(pack_db.system, {'centos': {'foo': '>= 1.0'}}) # Note: We only store paramters which are defined in the schema, all other custom user # defined attributes are ignored self.assertTrue(not hasattr(pack_db, 'future')) self.assertTrue(not hasattr(pack_db, 'this')) # Wrong characters in the required st2 version expected_msg = "'wrongstackstormversion' does not match" self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_10)
def test_register_pack_invalid_ref_name_friendly_error_message(self): registrar = ResourceRegistrar(use_pack_cache=False) # Invalid ref expected_msg = ( r'Pack ref / name can only contain valid word characters .*?,' ' dashes are not allowed.') self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_13) try: registrar._register_pack_db(pack_name=None, pack_dir=PACK_PATH_13) except ValidationError as e: self.assertTrue( "'invalid-has-dash' does not match '^[a-z0-9_]+$'" in str(e)) else: self.fail('Exception not thrown') # Pack ref not provided and name doesn't contain valid characters expected_msg = ( r'Pack name "dummy pack 14" contains invalid characters and "ref" ' 'attribute is not available. You either need to add') self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_14)
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) self.assertEqual(pack_dbs[0].name, 'dummy_pack_1') self.assertEqual(len(pack_dbs[0].contributors), 2) self.assertEqual(pack_dbs[0].contributors[0], 'John Doe1 <*****@*****.**>') self.assertEqual(pack_dbs[0].contributors[1], 'John Doe2 <*****@*****.**>') self.assertTrue('api_key' in config_schema_dbs[0].attributes) self.assertTrue('api_secret' in config_schema_dbs[0].attributes)
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_1': PACK_PATH } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) self.assertEqual(pack_dbs[0].name, 'dummy_pack_1') self.assertTrue('api_key' in config_schema_dbs[0].attributes) self.assertTrue('api_secret' in config_schema_dbs[0].attributes)
def test_register_pack_pack_ref(self): # Verify DB is empty pack_dbs = Pack.get_all() self.assertEqual(len(pack_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_1': PACK_PATH_1, 'dummy_pack_6': PACK_PATH_6 } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Ref is provided pack_db = Pack.get_by_name('dummy_pack_6') self.assertEqual(pack_db.ref, 'dummy_pack_6_ref') self.assertEqual(len(pack_db.contributors), 0) # Ref is not provided, directory name should be used pack_db = Pack.get_by_name('dummy_pack_1') self.assertEqual(pack_db.ref, 'dummy_pack_1') # "ref" is not provided, but "name" is registrar._register_pack_db(pack_name=None, pack_dir=PACK_PATH_7) pack_db = Pack.get_by_name('dummy_pack_7_name') self.assertEqual(pack_db.ref, 'dummy_pack_7_name') # "ref" is not provided and "name" contains invalid characters expected_msg = 'contains invalid characters' self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_8)
def test_register_pack_old_style_non_semver_version_is_normalized_to_valid_version(self): # Verify DB is empty pack_dbs = Pack.get_all() self.assertEqual(len(pack_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_11': PACK_PATH_11} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Non-semver valid version 0.2 should be normalize to 0.2.0 pack_db = Pack.get_by_name('dummy_pack_11') self.assertEqual(pack_db.version, '0.2.0')
def test_register_pack_arbitrary_properties_are_allowed(self): # Test registering a pack which has "arbitrary" properties in pack.yaml # We support this use-case (ignore properties which are not defined on the PackAPI model) # so we can add new attributes in a new version without breaking existing installations. registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_20': PACK_PATH_20, } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Ref is provided pack_db = Pack.get_by_name('dummy_pack_20') self.assertEqual(pack_db.ref, 'dummy_pack_20_ref') self.assertEqual(len(pack_db.contributors), 0)
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_1': PACK_PATH_1 } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) pack_db = pack_dbs[0] config_schema_db = config_schema_dbs[0] self.assertEqual(pack_db.name, 'dummy_pack_1') self.assertEqual(len(pack_db.contributors), 2) self.assertEqual(pack_db.contributors[0], 'John Doe1 <*****@*****.**>') self.assertEqual(pack_db.contributors[1], 'John Doe2 <*****@*****.**>') self.assertTrue('api_key' in config_schema_db.attributes) self.assertTrue('api_secret' in config_schema_db.attributes) # Verify pack_db.files is correct and doesn't contain excluded files (*.pyc, .git/*, etc.) # Note: We can't test that .git/* files are excluded since git doesn't allow you to add # .git directory to existing repo index :/ excluded_files = [ '__init__.pyc', 'actions/dummy1.pyc', 'actions/dummy2.pyc', ] for excluded_file in excluded_files: self.assertTrue(excluded_file not in pack_db.files)
def test_register_pack_empty_and_invalid_config_schema(self): registrar = ResourceRegistrar(use_pack_cache=False, fail_on_failure=True) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_17': PACK_PATH_17} packs_base_paths = content_utils.get_packs_base_paths() expected_msg = 'Config schema ".*?dummy_pack_17/config.schema.yaml" is empty and invalid.' self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_packs, base_dirs=packs_base_paths)
def test_register_pack_invalid_python_versions_attribute(self): registrar = ResourceRegistrar(use_pack_cache=False, fail_on_failure=True) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_21': PACK_PATH_21} packs_base_paths = content_utils.get_packs_base_paths() expected_msg = r"'4' is not one of \['2', '3'\]" self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_packs, base_dirs=packs_base_paths)
def test_register_pack_invalid_config_schema_invalid_attribute(self): registrar = ResourceRegistrar(use_pack_cache=False, fail_on_failure=True) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_18': PACK_PATH_18} packs_base_paths = content_utils.get_packs_base_paths() expected_msg = 'Additional properties are not allowed \(\'invalid\' was unexpected\)' self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_packs, base_dirs=packs_base_paths)
def test_register_packs(self): # Verify DB is empty pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 0) self.assertEqual(len(config_schema_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1} packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Verify pack and schema have been registered pack_dbs = Pack.get_all() config_schema_dbs = ConfigSchema.get_all() self.assertEqual(len(pack_dbs), 1) self.assertEqual(len(config_schema_dbs), 1) pack_db = pack_dbs[0] config_schema_db = config_schema_dbs[0] self.assertEqual(pack_db.name, 'dummy_pack_1') self.assertEqual(len(pack_db.contributors), 2) self.assertEqual(pack_db.contributors[0], 'John Doe1 <*****@*****.**>') self.assertEqual(pack_db.contributors[1], 'John Doe2 <*****@*****.**>') self.assertTrue('api_key' in config_schema_db.attributes) self.assertTrue('api_secret' in config_schema_db.attributes) # Verify pack_db.files is correct and doesn't contain excluded files (*.pyc, .git/*, etc.) # Note: We can't test that .git/* files are excluded since git doesn't allow you to add # .git directory to existing repo index :/ excluded_files = [ '__init__.pyc', 'actions/dummy1.pyc', 'actions/dummy2.pyc', ] for excluded_file in excluded_files: self.assertTrue(excluded_file not in pack_db.files)
def setup_virtualenvs(recreate_virtualenvs=False): """ Setup Python virtual environments for all the registered or the provided pack. """ LOG.info('=========================================================') LOG.info('########### Setting up virtual environments #############') LOG.info('=========================================================') pack_dir = cfg.CONF.register.pack fail_on_failure = not cfg.CONF.register.no_fail_on_failure registrar = ResourceRegistrar() if pack_dir: pack_name = os.path.basename(pack_dir) pack_names = [pack_name] # 1. Register pack registrar.register_pack(pack_name=pack_name, pack_dir=pack_dir) else: # 1. Register pack base_dirs = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=base_dirs) # 2. Retrieve available packs (aka packs which have been registered) pack_names = registrar.get_registered_packs() if recreate_virtualenvs: """ update = False: this is more than an update of an existing virtualenv the virtualenv itself will be removed & recreated this is i.e. useful for updates to a newer Python release """ update = False else: """ update = True: only dependencies inside the virtualenv will be updated """ update = True setup_count = 0 for pack_name in pack_names: try: setup_pack_virtualenv(pack_name=pack_name, update=update, logger=LOG) except Exception as e: exc_info = not fail_on_failure LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e, exc_info=exc_info) if fail_on_failure: raise e else: setup_count += 1 LOG.info('Setup virtualenv for %s pack(s).' % (setup_count))
def test_register_pack_invalid_ref_name_friendly_error_message(self): registrar = ResourceRegistrar(use_pack_cache=False) # Invalid ref expected_msg = (r'Pack ref / name can only contain valid word characters .*?,' ' dashes are not allowed.') self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_13) try: registrar._register_pack_db(pack_name=None, pack_dir=PACK_PATH_13) except ValidationError as e: self.assertTrue("'invalid-has-dash' does not match '^[a-z0-9_]+$'" in six.text_type(e)) else: self.fail('Exception not thrown') # Pack ref not provided and name doesn't contain valid characters expected_msg = (r'Pack name "dummy pack 14" contains invalid characters and "ref" ' 'attribute is not available. You either need to add') self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_14)
def test_register_pack_invalid_semver_version_friendly_error_message(self): registrar = ResourceRegistrar(use_pack_cache=False) expected_msg = ('Pack version "0.1.2.3.4" doesn\'t follow a valid semver format. Valid ' 'versions and formats include: 0.1.0, 0.2.1, 1.1.0, etc.') self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_12) expected_msg = ('Pack version "0.2" doesn\'t follow a valid semver format. Valid ' 'versions and formats include: 0.1.0, 0.2.1, 1.1.0, etc.') self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_11)
def test_register_pack_pack_ref(self): # Verify DB is empty pack_dbs = Pack.get_all() self.assertEqual(len(pack_dbs), 0) registrar = ResourceRegistrar(use_pack_cache=False) registrar._pack_loader.get_packs = mock.Mock() registrar._pack_loader.get_packs.return_value = { 'dummy_pack_1': PACK_PATH_1, 'dummy_pack_6': PACK_PATH_6 } packs_base_paths = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=packs_base_paths) # Ref is provided pack_db = Pack.get_by_name('dummy_pack_6') self.assertEqual(pack_db.ref, 'dummy_pack_6_ref') self.assertEqual(len(pack_dbs[0].contributors), 0) # Ref is not provided, directory name should be used pack_db = Pack.get_by_name('dummy_pack_1') self.assertEqual(pack_db.ref, 'dummy_pack_1') # "ref" is not provided, but "name" is registrar._register_pack_db(pack_name=None, pack_dir=PACK_PATH_7) pack_db = Pack.get_by_name('dummy_pack_7_name') self.assertEqual(pack_db.ref, 'dummy_pack_7_name') # "ref" is not provided and "name" contains invalid characters expected_msg = 'contains invalid characters' self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db, pack_name=None, pack_dir=PACK_PATH_8)
def setup_virtualenvs(): """ Setup Python virtual environments for all the registered or the provided pack. """ LOG.info('=========================================================') LOG.info('########### Setting up virtual environments #############') LOG.info('=========================================================') pack_dir = cfg.CONF.register.pack fail_on_failure = cfg.CONF.register.fail_on_failure registrar = ResourceRegistrar() if pack_dir: pack_name = os.path.basename(pack_dir) pack_names = [pack_name] # 1. Register pack registrar.register_pack(pack_name=pack_name, pack_dir=pack_dir) else: # 1. Register pack base_dirs = content_utils.get_packs_base_paths() registrar.register_packs(base_dirs=base_dirs) # 2. Retrieve available packs (aka packs which have been registered) pack_names = registrar.get_registered_packs() setup_count = 0 for pack_name in pack_names: try: setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG) except Exception as e: exc_info = not fail_on_failure LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e, exc_info=exc_info) if fail_on_failure: raise e else: setup_count += 1 LOG.info('Setup virtualenv for %s pack(s).' % (setup_count))
from st2common.constants.pack import DEFAULT_PACK_NAME from st2common.content.loader import ContentPackLoader from st2common.content.loader import MetaLoader from st2common.bootstrap.base import ResourceRegistrar import st2common.content.utils as content_utils from st2common.models.api.action import ActionAPI from st2common.models.api.sensor import SensorTypeAPI from st2common.models.api.rule import RuleAPI from st2common.service_setup import db_setup from st2common.service_setup import db_teardown from st2common.models.system.common import ResourceReference from st2common.persistence.rule import Rule from st2common.persistence.sensor import SensorType from st2common.persistence.action import Action registrar = ResourceRegistrar() registrar.ALLOWED_EXTENSIONS = [".yaml", ".yml", ".json"] meta_loader = MetaLoader() API_MODELS_ARTIFACT_TYPES = { "actions": ActionAPI, "sensors": SensorTypeAPI, "rules": RuleAPI, } API_MODELS_PERSISTENT_MODELS = { Action: ActionAPI, SensorType: SensorTypeAPI, Rule: RuleAPI, }
from st2common.constants.pack import DEFAULT_PACK_NAME from st2common.content.loader import ContentPackLoader from st2common.content.loader import MetaLoader from st2common.bootstrap.base import ResourceRegistrar import st2common.content.utils as content_utils from st2common.models.api.action import ActionAPI from st2common.models.api.sensor import SensorTypeAPI from st2common.models.api.rule import RuleAPI from st2common.service_setup import db_setup from st2common.service_setup import db_teardown from st2common.models.system.common import ResourceReference from st2common.persistence.rule import Rule from st2common.persistence.sensor import SensorType from st2common.persistence.action import Action registrar = ResourceRegistrar() registrar.ALLOWED_EXTENSIONS = ['.yaml', '.yml', '.json'] meta_loader = MetaLoader() API_MODELS_ARTIFACT_TYPES = { 'actions': ActionAPI, 'sensors': SensorTypeAPI, 'rules': RuleAPI } API_MODELS_PERSISTENT_MODELS = { Action: ActionAPI, SensorType: SensorTypeAPI, Rule: RuleAPI }
from st2common.constants.pack import DEFAULT_PACK_NAME from st2common.content.loader import ContentPackLoader from st2common.content.loader import MetaLoader from st2common.bootstrap.base import ResourceRegistrar import st2common.content.utils as content_utils from st2common.models.api.action import ActionAPI from st2common.models.api.sensor import SensorTypeAPI from st2common.models.api.rule import RuleAPI from st2common.models.db import db_setup from st2common.models.db import db_teardown from st2common.models.system.common import ResourceReference from st2common.persistence.rule import Rule from st2common.persistence.sensor import SensorType from st2common.persistence.action import Action registrar = ResourceRegistrar() registrar.ALLOWED_EXTENSIONS = [".yaml", ".yml", ".json"] meta_loader = MetaLoader() API_MODELS_ARTIFACT_TYPES = {"actions": ActionAPI, "sensors": SensorTypeAPI, "rules": RuleAPI} API_MODELS_PERSISTENT_MODELS = {Action: ActionAPI, SensorType: SensorTypeAPI, Rule: RuleAPI} def do_register_cli_opts(opts, ignore_errors=False): for opt in opts: try: cfg.CONF.register_cli_opt(opt) except: if not ignore_errors: