def test_get_pack_base_paths(self):
cfg.CONF.content.system_packs_base_path = ""
cfg.CONF.content.packs_base_paths = "/opt/path1"
result = get_packs_base_paths()
self.assertEqual(result, ["/opt/path1"])
# Multiple paths, no trailing colon
cfg.CONF.content.packs_base_paths = "/opt/path1:/opt/path2"
result = get_packs_base_paths()
self.assertEqual(result, ["/opt/path1", "/opt/path2"])
# Multiple paths, trailing colon
cfg.CONF.content.packs_base_paths = "/opt/path1:/opt/path2:"
result = get_packs_base_paths()
self.assertEqual(result, ["/opt/path1", "/opt/path2"])
# Multiple same paths
cfg.CONF.content.packs_base_paths = "/opt/path1:/opt/path2:/opt/path1:/opt/path2"
result = get_packs_base_paths()
self.assertEqual(result, ["/opt/path1", "/opt/path2"])
# Assert system path is always first
cfg.CONF.content.system_packs_base_path = "/opt/system"
cfg.CONF.content.packs_base_paths = "/opt/path2:/opt/path1"
result = get_packs_base_paths()
self.assertEqual(result, ["/opt/system", "/opt/path2", "/opt/path1"])
def test_get_pack_base_paths(self):
cfg.CONF.content.system_packs_base_path = ''
cfg.CONF.content.packs_base_paths = '/opt/path1'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1'])
# Multiple paths, no trailing colon
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Multiple paths, trailing colon
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Multiple same paths
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:/opt/path1:/opt/path2'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Assert system path is always first
cfg.CONF.content.system_packs_base_path = '/opt/system'
cfg.CONF.content.packs_base_paths = '/opt/path2:/opt/path1'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/system', '/opt/path2', '/opt/path1'])
def _register_packs(self):
"""
Register all the packs inside the fixtures directory.
"""
registrar = ResourceRegistrar(use_pack_cache=False)
registrar.register_packs(base_dirs=get_packs_base_paths())
def test_register_pack_pack_ref(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
self.assertEqual(len(pack_dbs), 0)
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {
'dummy_pack_1': PACK_PATH_1,
'dummy_pack_6': PACK_PATH_6
}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Ref is provided
pack_db = Pack.get_by_name('dummy_pack_6')
self.assertEqual(pack_db.ref, 'dummy_pack_6_ref')
self.assertEqual(len(pack_db.contributors), 0)
# Ref is not provided, directory name should be used
pack_db = Pack.get_by_name('dummy_pack_1')
self.assertEqual(pack_db.ref, 'dummy_pack_1')
# "ref" is not provided, but "name" is
registrar._register_pack_db(pack_name=None, pack_dir=PACK_PATH_7)
pack_db = Pack.get_by_name('dummy_pack_7_name')
self.assertEqual(pack_db.ref, 'dummy_pack_7_name')
# "ref" is not provided and "name" contains invalid characters
expected_msg = 'contains invalid characters'
self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db,
pack_name=None, pack_dir=PACK_PATH_8)
def _get_api_models_from_disk(artifact_type, pack_dir=None):
loader = ContentPackLoader()
artifacts = None
if pack_dir:
artifacts_dir = loader.get_content_from_pack(pack_dir, artifact_type)
pack_name = os.path.basename(os.path.normpath(pack_dir))
artifacts = {pack_name: artifacts_dir}
else:
packs_dirs = content_utils.get_packs_base_paths()
artifacts = loader.get_content(packs_dirs, artifact_type)
artifacts_dict = {}
for pack_name, pack_path in artifacts.items():
artifacts_paths = registrar.get_resources_from_pack(pack_path)
for artifact_path in artifacts_paths:
artifact = meta_loader.load(artifact_path)
if artifact_type == "sensors":
sensors_dir = os.path.dirname(artifact_path)
sensor_file_path = os.path.join(sensors_dir, artifact["entry_point"])
artifact["artifact_uri"] = "file://" + sensor_file_path
name = artifact.get("name", None) or artifact.get("class_name", None)
if not artifact.get("pack", None):
artifact["pack"] = pack_name
ref = ResourceReference.to_string_reference(name=name, pack=pack_name)
API_MODEL = API_MODELS_ARTIFACT_TYPES[artifact_type]
# Following conversions are required because we add some fields with
# default values in db model. If we don't do these conversions,
# we'll see a unnecessary diff for those fields.
artifact_api = API_MODEL(**artifact)
artifact_db = API_MODEL.to_model(artifact_api)
artifact_api = API_MODEL.from_model(artifact_db)
artifacts_dict[ref] = artifact_api
return artifacts_dict
def collect_pack_content(output_path):
"""
Copy pack contents to the output path.
:param output_path: Path where pack contents will be copied to.
:type output_path: ``str``
"""
LOG.debug('Including content')
packs_base_paths = get_packs_base_paths()
for index, packs_base_path in enumerate(packs_base_paths, 1):
dst = os.path.join(output_path, 'dir-%s' % index)
try:
shutil.copytree(src=packs_base_path, dst=dst)
except IOError:
continue
base_pack_dirs = get_dirs_in_path(file_path=output_path)
for base_pack_dir in base_pack_dirs:
pack_dirs = get_dirs_in_path(file_path=base_pack_dir)
for pack_dir in pack_dirs:
process_content_pack_dir(pack_dir=pack_dir)
def test_register_configs_for_all_packs(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_dbs), 0)
registrar = ConfigsRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_1_PATH}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_from_packs(base_dirs=packs_base_paths)
# Verify pack and schema have been registered
pack_dbs = Pack.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(config_dbs), 1)
config_db = config_dbs[0]
self.assertEqual(config_db.values['api_key'], '{{st2kv.user.api_key}}')
self.assertEqual(config_db.values['api_secret'], SUPER_SECRET_PARAMETER)
self.assertEqual(config_db.values['region'], 'us-west-1')
def test_register_packs(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Verify pack and schema have been registered
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(config_schema_dbs), 1)
self.assertEqual(pack_dbs[0].name, 'dummy_pack_1')
self.assertEqual(len(pack_dbs[0].contributors), 2)
self.assertEqual(pack_dbs[0].contributors[0], 'John Doe1 <*****@*****.**>')
self.assertEqual(pack_dbs[0].contributors[1], 'John Doe2 <*****@*****.**>')
self.assertTrue('api_key' in config_schema_dbs[0].attributes)
self.assertTrue('api_secret' in config_schema_dbs[0].attributes)
def test_register_pack_pack_stackstorm_version_and_future_parameters(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
self.assertEqual(len(pack_dbs), 0)
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_9': PACK_PATH_9}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Dependencies, stackstorm_version and future values
pack_db = Pack.get_by_name('dummy_pack_9_deps')
self.assertEqual(pack_db.dependencies, ['core=0.2.0'])
self.assertEqual(pack_db.stackstorm_version, '>=1.6.0, <2.2.0')
self.assertEqual(pack_db.system, {'centos': {'foo': '>= 1.0'}})
self.assertEqual(pack_db.python_versions, ['2', '3'])
# Note: We only store parameters which are defined in the schema, all other custom user
# defined attributes are ignored
self.assertTrue(not hasattr(pack_db, 'future'))
self.assertTrue(not hasattr(pack_db, 'this'))
# Wrong characters in the required st2 version
expected_msg = "'wrongstackstormversion' does not match"
self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db,
pack_name=None, pack_dir=PACK_PATH_10)
def test_register_all_configs_with_config_schema_validation_validation_failure_1(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_dbs), 0)
registrar = ConfigsRegistrar(use_pack_cache=False, fail_on_failure=True,
validate_configs=True)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_6': PACK_6_PATH}
# Register ConfigSchema for pack
registrar._register_pack_db = mock.Mock()
registrar._register_pack(pack_name='dummy_pack_5', pack_dir=PACK_6_PATH)
packs_base_paths = content_utils.get_packs_base_paths()
if six.PY3:
expected_msg = ('Failed validating attribute "regions" in config for pack '
'"dummy_pack_6" (.*?): 1000 is not of type \'array\'')
else:
expected_msg = ('Failed validating attribute "regions" in config for pack '
'"dummy_pack_6" (.*?): 1000 is not of type u\'array\'')
self.assertRaisesRegexp(ValueError, expected_msg,
registrar.register_from_packs,
base_dirs=packs_base_paths)
def test_register_configs_for_all_packs(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_dbs), 0)
registrar = ConfigsRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {"dummy_pack_1": PACK_PATH}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_configs_for_all_packs(base_dirs=packs_base_paths)
# Verify pack and schema have been registered
pack_dbs = Pack.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(config_dbs), 1)
config_db = config_dbs[0]
self.assertEqual(config_db.values["api_key"], "{{user.api_key}}")
self.assertEqual(config_db.values["api_secret"], "{{user.api_secret}}")
self.assertEqual(config_db.values["region"], "us-west-1")
def _setup_pack_virtualenv(self, pack_name, update=False):
"""
Setup virtual environment for the provided pack.
:param pack_name: Pack name.
:type pack_name: ``str``
"""
# Prevent directory traversal by whitelisting allowed characters in the
# pack name
if not re.match(PACK_NAME_WHITELIST, pack_name):
raise ValueError('Invalid pack name "%s"' % (pack_name))
self.logger.debug('Setting up virtualenv for pack "%s"' % (pack_name))
virtualenv_path = os.path.join(self._base_virtualenvs_path, quote_unix(pack_name))
# Ensure pack directory exists in one of the search paths
pack_path = get_pack_directory(pack_name=pack_name)
if not pack_path:
packs_base_paths = get_packs_base_paths()
search_paths = ', '.join(packs_base_paths)
msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
raise Exception(msg)
if not os.path.exists(self._base_virtualenvs_path):
os.makedirs(self._base_virtualenvs_path)
# If we don't want to update, or if the virtualenv doesn't exist, let's create it.
if not update or not os.path.exists(virtualenv_path):
# 0. Delete virtual environment if it exists
self._remove_virtualenv(virtualenv_path=virtualenv_path)
# 1. Create virtual environment
self.logger.debug('Creating virtualenv for pack "%s" in "%s"' %
(pack_name, virtualenv_path))
self._create_virtualenv(virtualenv_path=virtualenv_path)
# 2. Install base requirements which are common to all the packs
self.logger.debug('Installing base requirements')
for requirement in BASE_PACK_REQUIREMENTS:
self._install_requirement(virtualenv_path=virtualenv_path,
requirement=requirement)
# 3. Install pack-specific requirements
requirements_file_path = os.path.join(pack_path, 'requirements.txt')
has_requirements = os.path.isfile(requirements_file_path)
if has_requirements:
self.logger.debug('Installing pack specific requirements from "%s"' %
(requirements_file_path))
self._install_requirements(virtualenv_path, requirements_file_path)
else:
self.logger.debug('No pack specific requirements found')
self.logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
(pack_name,
'updated' if update else 'created',
virtualenv_path))
def run(self, pack):
"""
:param pack: Installed Pack Name to get info about
:type pack: ``str``
"""
packs_base_paths = get_packs_base_paths()
pack_path = None
metadata_file = None
for packs_base_path in packs_base_paths:
pack_path = os.path.join(packs_base_path, pack)
pack_yaml_path = os.path.join(pack_path, MANIFEST_FILE_NAME)
if os.path.isfile(pack_yaml_path):
metadata_file = pack_yaml_path
break
# Pack doesn't exist, finish execution normally with empty metadata
if not os.path.isdir(pack_path):
return {
'pack': None,
'git_status': None
}
if not metadata_file:
error = ('Pack "%s" doesn\'t contain pack.yaml file.' % (pack))
raise Exception(error)
try:
details = self._parse_yaml_file(metadata_file)
except Exception as e:
error = ('Pack "%s" doesn\'t contain a valid pack.yaml file: %s' % (pack, str(e)))
raise Exception(error)
try:
repo = Repo(pack_path)
git_status = "Status:\n%s\n\nRemotes:\n%s" % (
repo.git.status().split('\n')[0],
"\n".join([remote.url for remote in repo.remotes])
)
ahead_behind = repo.git.rev_list(
'--left-right', '--count', 'HEAD...origin/master'
).split()
# Dear god.
if ahead_behind != [u'0', u'0']:
git_status += "\n\n"
git_status += "%s commits ahead " if ahead_behind[0] != u'0' else ""
git_status += "and " if u'0' not in ahead_behind else ""
git_status += "%s commits behind " if ahead_behind[1] != u'0' else ""
git_status += "origin/master."
except InvalidGitRepositoryError:
git_status = None
return {
'pack': details,
'git_status': git_status
}
def test_register_pack_empty_and_invalid_config_schema(self):
registrar = ResourceRegistrar(use_pack_cache=False, fail_on_failure=True)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_17': PACK_PATH_17}
packs_base_paths = content_utils.get_packs_base_paths()
expected_msg = 'Config schema ".*?dummy_pack_17/config.schema.yaml" is empty and invalid.'
self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_packs,
base_dirs=packs_base_paths)
def test_register_pack_invalid_config_schema_invalid_attribute(self):
registrar = ResourceRegistrar(use_pack_cache=False, fail_on_failure=True)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_18': PACK_PATH_18}
packs_base_paths = content_utils.get_packs_base_paths()
expected_msg = r'Additional properties are not allowed \(\'invalid\' was unexpected\)'
self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_packs,
base_dirs=packs_base_paths)
def test_register_pack_invalid_python_versions_attribute(self):
registrar = ResourceRegistrar(use_pack_cache=False, fail_on_failure=True)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_21': PACK_PATH_21}
packs_base_paths = content_utils.get_packs_base_paths()
expected_msg = r"'4' is not one of \['2', '3'\]"
self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_packs,
base_dirs=packs_base_paths)
def test_get_pack_base_paths(self):
cfg.CONF.content.system_packs_base_path = ''
cfg.CONF.content.packs_base_paths = '/opt/path1'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1'])
# Multiple paths, no trailing colon
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Multiple paths, trailing colon
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Multiple same paths
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:/opt/path1:/opt/path2'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Assert system path is always first
cfg.CONF.content.system_packs_base_path = '/opt/system'
cfg.CONF.content.packs_base_paths = '/opt/path2:/opt/path1'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/system', '/opt/path2', '/opt/path1'])
# More scenarios
orig_path = cfg.CONF.content.system_packs_base_path
cfg.CONF.content.system_packs_base_path = '/tests/packs'
names = [
'test_pack_1',
'test_pack_2',
'ma_pack'
]
for name in names:
actual = get_pack_base_path(pack_name=name)
expected = os.path.join(cfg.CONF.content.system_packs_base_path,
name)
self.assertEqual(actual, expected)
cfg.CONF.content.system_packs_base_path = orig_path
def post(self, pack_register_request):
if pack_register_request and hasattr(pack_register_request, 'types'):
types = pack_register_request.types
if 'all' in types:
types = PackRegisterController.CONTENT_TYPES
else:
types = PackRegisterController.CONTENT_TYPES
if pack_register_request and hasattr(pack_register_request, 'packs'):
packs = list(set(pack_register_request.packs))
else:
packs = None
result = defaultdict(int)
# Register depended resources (actions depend on runners, rules depend on rule types, etc)
if ('runner' in types or 'runners' in types) or ('action' in types or 'actions' in types):
result['runners'] = runners_registrar.register_runners(experimental=True)
if ('rule_type' in types or 'rule_types' in types) or \
('rule' in types or 'rules' in types):
result['rule_types'] = rule_types_registrar.register_rule_types()
if ('policy_type' in types or 'policy_types' in types) or \
('policy' in types or 'policies' in types):
result['policy_types'] = policies_registrar.register_policy_types(st2common)
use_pack_cache = False
fail_on_failure = getattr(pack_register_request, 'fail_on_failure', True)
for type, (Registrar, name) in six.iteritems(ENTITIES):
if type in types or name in types:
registrar = Registrar(use_pack_cache=use_pack_cache,
use_runners_cache=True,
fail_on_failure=fail_on_failure)
if packs:
for pack in packs:
pack_path = content_utils.get_pack_base_path(pack)
try:
registered_count = registrar.register_from_pack(pack_dir=pack_path)
result[name] += registered_count
except ValueError as e:
# Throw more user-friendly exception if requsted pack doesn't exist
if re.match('Directory ".*?" doesn\'t exist', six.text_type(e)):
msg = 'Pack "%s" not found on disk: %s' % (pack, six.text_type(e))
raise ValueError(msg)
raise e
else:
packs_base_paths = content_utils.get_packs_base_paths()
registered_count = registrar.register_from_packs(base_dirs=packs_base_paths)
result[name] += registered_count
return result
def validate_action(action_api):
runner_db = _get_runner_model(action_api)
# Check if pack is valid.
if not _is_valid_pack(action_api.pack):
packs_base_paths = get_packs_base_paths()
packs_base_paths = ','.join(packs_base_paths)
msg = ('Content pack "%s" is not found or doesn\'t contain actions directory. '
'Searched in: %s' %
(action_api.pack, packs_base_paths))
raise ValueValidationException(msg)
# Check if parameters defined are valid.
_validate_parameters(action_api.parameters, runner_db.runner_parameters)
def test_register_pack_old_style_non_semver_version_is_normalized_to_valid_version(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
self.assertEqual(len(pack_dbs), 0)
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_11': PACK_PATH_11}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Non-semver valid version 0.2 should be normalize to 0.2.0
pack_db = Pack.get_by_name('dummy_pack_11')
self.assertEqual(pack_db.version, '0.2.0')
def register_sensors(packs_base_paths=None, pack_dir=None, use_pack_cache=True):
if packs_base_paths:
assert isinstance(packs_base_paths, list)
if not packs_base_paths:
packs_base_paths = content_utils.get_packs_base_paths()
registrar = SensorsRegistrar(use_pack_cache=use_pack_cache)
if pack_dir:
result = registrar.register_sensors_from_pack(pack_dir=pack_dir)
else:
result = registrar.register_sensors_from_packs(base_dirs=packs_base_paths)
return result
def register_policies(packs_base_paths=None, pack_dir=None):
if packs_base_paths:
assert(isinstance(packs_base_paths, list))
if not packs_base_paths:
packs_base_paths = content_utils.get_packs_base_paths()
registrar = PolicyRegistrar()
if pack_dir:
result = registrar.register_policies_from_pack(pack_dir=pack_dir)
else:
result = registrar.register_policies_from_packs(base_dirs=packs_base_paths)
return result
def register_aliases(packs_base_paths=None, pack_dir=None, use_pack_cache=True, fail_on_failure=False):
if packs_base_paths:
assert isinstance(packs_base_paths, list)
if not packs_base_paths:
packs_base_paths = content_utils.get_packs_base_paths()
registrar = AliasesRegistrar(use_pack_cache=use_pack_cache, fail_on_failure=fail_on_failure)
if pack_dir:
result = registrar.register_aliases_from_pack(pack_dir=pack_dir)
else:
result = registrar.register_aliases_from_packs(base_dirs=packs_base_paths)
return result
def test_register_pack_arbitrary_properties_are_allowed(self):
# Test registering a pack which has "arbitrary" properties in pack.yaml
# We support this use-case (ignore properties which are not defined on the PackAPI model)
# so we can add new attributes in a new version without breaking existing installations.
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {
'dummy_pack_20': PACK_PATH_20,
}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Ref is provided
pack_db = Pack.get_by_name('dummy_pack_20')
self.assertEqual(pack_db.ref, 'dummy_pack_20_ref')
self.assertEqual(len(pack_db.contributors), 0)
def setup_virtualenvs():
"""
Setup Python virtual environments for all the registered or the provided pack.
"""
LOG.info('=========================================================')
LOG.info('########### Setting up virtual environments #############')
LOG.info('=========================================================')
pack_dir = cfg.CONF.register.pack
fail_on_failure = not cfg.CONF.register.no_fail_on_failure
registrar = ResourceRegistrar()
if pack_dir:
pack_name = os.path.basename(pack_dir)
pack_names = [pack_name]
# 1. Register pack
registrar.register_pack(pack_name=pack_name, pack_dir=pack_dir)
else:
# 1. Register pack
base_dirs = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=base_dirs)
# 2. Retrieve available packs (aka packs which have been registered)
pack_names = registrar.get_registered_packs()
setup_count = 0
for pack_name in pack_names:
try:
setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG)
except Exception as e:
exc_info = not fail_on_failure
LOG.warning('Failed to setup virtualenv for pack "%s": %s',
pack_name,
e,
exc_info=exc_info)
if fail_on_failure:
raise e
else:
setup_count += 1
LOG.info('Setup virtualenv for %s pack(s).' % (setup_count))
def _get_pack_dependencies(self, pack):
packs_base_paths = get_packs_base_paths()
pack_path = None
metadata_file = None
for packs_base_path in packs_base_paths:
pack_path = os.path.join(packs_base_path, pack)
pack_yaml_path = os.path.join(pack_path, MANIFEST_FILE_NAME)
if os.path.isfile(pack_yaml_path):
metadata_file = pack_yaml_path
break
# Pack doesn't exist, finish execution normally with empty metadata
if not os.path.isdir(pack_path):
return []
if not metadata_file:
error = ('Pack "%s" doesn\'t contain pack.yaml file.' % (pack))
raise Exception(error)
try:
details = self._parse_yaml_file(metadata_file)
except Exception as e:
error = ('Pack "%s" doesn\'t contain a valid pack.yaml file: %s' %
(pack, str(e)))
raise Exception(error)
# dev mode
# if dev_dependencies are specified, use them
# else use dependencies
# else
# use dependencies
deps = []
if self.stackstorm_environment == "dev":
if 'dev_dependencies' in details:
deps = details["dev_dependencies"]
elif 'dependencies' in details:
deps = details["dependencies"]
else:
if 'dependencies' in details:
deps = details["dependencies"]
return deps
def register_policies(packs_base_paths=None,
pack_dir=None,
use_pack_cache=True,
fail_on_failure=False):
if packs_base_paths:
assert isinstance(packs_base_paths, list)
if not packs_base_paths:
packs_base_paths = content_utils.get_packs_base_paths()
registrar = PolicyRegistrar(use_pack_cache=use_pack_cache,
fail_on_failure=fail_on_failure)
if pack_dir:
result = registrar.register_from_pack(pack_dir=pack_dir)
else:
result = registrar.register_from_packs(base_dirs=packs_base_paths)
return result
def setup_virtualenvs():
"""
Setup Python virtual environments for all the registered or the provided pack.
"""
LOG.info('=========================================================')
LOG.info('########### Setting up virtual environments #############')
LOG.info('=========================================================')
pack_dir = cfg.CONF.register.pack
fail_on_failure = cfg.CONF.register.fail_on_failure
registrar = ResourceRegistrar()
if pack_dir:
pack_name = os.path.basename(pack_dir)
pack_names = [pack_name]
# 1. Register pack
registrar.register_pack(pack_name=pack_name, pack_dir=pack_dir)
else:
# 1. Register pack
base_dirs = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=base_dirs)
# 2. Retrieve available packs (aka packs which have been registered)
pack_names = registrar.get_registered_packs()
setup_count = 0
for pack_name in pack_names:
try:
setup_pack_virtualenv(pack_name=pack_name, update=True, logger=LOG)
except Exception as e:
exc_info = not fail_on_failure
LOG.warning('Failed to setup virtualenv for pack "%s": %s', pack_name, e,
exc_info=exc_info)
if fail_on_failure:
raise e
else:
setup_count += 1
LOG.info('Setup virtualenv for %s pack(s).' % (setup_count))
def register_configs(packs_base_paths=None, pack_dir=None, use_pack_cache=True,
fail_on_failure=False, validate_configs=True):
if packs_base_paths:
assert isinstance(packs_base_paths, list)
if not packs_base_paths:
packs_base_paths = content_utils.get_packs_base_paths()
registrar = ConfigsRegistrar(use_pack_cache=use_pack_cache,
fail_on_failure=fail_on_failure,
validate_configs=validate_configs)
if pack_dir:
result = registrar.register_config_for_pack(pack_dir=pack_dir)
else:
result = registrar.register_configs_for_all_packs(base_dirs=packs_base_paths)
return result
def register_configs(packs_base_paths=None, pack_dir=None, use_pack_cache=True,
fail_on_failure=False, validate_configs=True):
if packs_base_paths:
assert isinstance(packs_base_paths, list)
if not packs_base_paths:
packs_base_paths = content_utils.get_packs_base_paths()
registrar = ConfigsRegistrar(use_pack_cache=use_pack_cache,
fail_on_failure=fail_on_failure,
validate_configs=validate_configs)
if pack_dir:
result = registrar.register_from_pack(pack_dir=pack_dir)
else:
result = registrar.register_from_packs(base_dirs=packs_base_paths)
return result
def test_register_packs(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Verify pack and schema have been registered
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(config_schema_dbs), 1)
pack_db = pack_dbs[0]
config_schema_db = config_schema_dbs[0]
self.assertEqual(pack_db.name, 'dummy_pack_1')
self.assertEqual(len(pack_db.contributors), 2)
self.assertEqual(pack_db.contributors[0], 'John Doe1 <*****@*****.**>')
self.assertEqual(pack_db.contributors[1], 'John Doe2 <*****@*****.**>')
self.assertTrue('api_key' in config_schema_db.attributes)
self.assertTrue('api_secret' in config_schema_db.attributes)
# Verify pack_db.files is correct and doesn't contain excluded files (*.pyc, .git/*, etc.)
# Note: We can't test that .git/* files are excluded since git doesn't allow you to add
# .git directory to existing repo index :/
excluded_files = [
'__init__.pyc',
'actions/dummy1.pyc',
'actions/dummy2.pyc',
]
for excluded_file in excluded_files:
self.assertTrue(excluded_file not in pack_db.files)
def post(self, pack_register_request):
if pack_register_request and hasattr(pack_register_request, 'types'):
types = pack_register_request.types
else:
types = [
'runner', 'action', 'trigger', 'sensor', 'rule', 'rule_type',
'alias', 'policy_type', 'policy', 'config'
]
if pack_register_request and hasattr(pack_register_request, 'packs'):
packs = pack_register_request.packs
else:
packs = None
result = {}
if 'runner' in types or 'action' in types:
result['runners'] = runners_registrar.register_runners(
experimental=True)
if 'rule_type' in types or 'rule' in types:
result['rule_types'] = rule_types_registrar.register_rule_types()
if 'policy_type' in types or 'policy' in types:
result['policy_types'] = policies_registrar.register_policy_types(
st2common)
use_pack_cache = False
for type, (Registrar, name) in six.iteritems(ENTITIES):
if type in types:
registrar = Registrar(use_pack_cache=use_pack_cache,
fail_on_failure=False)
if packs:
for pack in packs:
pack_path = content_utils.get_pack_base_path(pack)
result[name] = registrar.register_from_pack(
pack_dir=pack_path)
else:
packs_base_paths = content_utils.get_packs_base_paths()
result[name] = registrar.register_from_packs(
base_dirs=packs_base_paths)
return result
def test_register_packs(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_schema_dbs), 0)
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Verify pack and schema have been registered
pack_dbs = Pack.get_all()
config_schema_dbs = ConfigSchema.get_all()
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(config_schema_dbs), 1)
pack_db = pack_dbs[0]
config_schema_db = config_schema_dbs[0]
self.assertEqual(pack_db.name, 'dummy_pack_1')
self.assertEqual(len(pack_db.contributors), 2)
self.assertEqual(pack_db.contributors[0], 'John Doe1 <*****@*****.**>')
self.assertEqual(pack_db.contributors[1], 'John Doe2 <*****@*****.**>')
self.assertTrue('api_key' in config_schema_db.attributes)
self.assertTrue('api_secret' in config_schema_db.attributes)
# Verify pack_db.files is correct and doesn't contain excluded files (*.pyc, .git/*, etc.)
# Note: We can't test that .git/* files are excluded since git doesn't allow you to add
# .git directory to existing repo index :/
excluded_files = [
'__init__.pyc',
'actions/dummy1.pyc',
'actions/dummy2.pyc',
]
for excluded_file in excluded_files:
self.assertTrue(excluded_file not in pack_db.files)
def test_register_all_configs_with_config_schema_validation_validation_failure_2(
self, ):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_dbs), 0)
registrar = ConfigsRegistrar(use_pack_cache=False,
fail_on_failure=True,
validate_configs=True)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {
"dummy_pack_19": PACK_19_PATH
}
# Register ConfigSchema for pack
registrar._register_pack_db = mock.Mock()
registrar._register_pack(pack_name="dummy_pack_19",
pack_dir=PACK_19_PATH)
packs_base_paths = content_utils.get_packs_base_paths()
if six.PY3:
expected_msg = (
'Failed validating attribute "instances.0.alias" in config for pack '
"\"dummy_pack_19\" (.*?): {'not': 'string'} is not of type "
"'string'")
else:
expected_msg = (
'Failed validating attribute "instances.0.alias" in config for pack '
"\"dummy_pack_19\" (.*?): {'not': 'string'} is not of type "
"u'string'")
self.assertRaisesRegexp(
ValueError,
expected_msg,
registrar.register_from_packs,
base_dirs=packs_base_paths,
)
def test_register_pack_pack_ref(self):
# Verify DB is empty
pack_dbs = Pack.get_all()
self.assertEqual(len(pack_dbs), 0)
registrar = ResourceRegistrar(use_pack_cache=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {
"dummy_pack_1": PACK_PATH_1,
"dummy_pack_6": PACK_PATH_6,
}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=packs_base_paths)
# Ref is provided
pack_db = Pack.get_by_name("dummy_pack_6")
self.assertEqual(pack_db.ref, "dummy_pack_6_ref")
self.assertEqual(len(pack_db.contributors), 0)
# Ref is not provided, directory name should be used
pack_db = Pack.get_by_name("dummy_pack_1")
self.assertEqual(pack_db.ref, "dummy_pack_1")
# "ref" is not provided, but "name" is
registrar._register_pack_db(pack_name=None, pack_dir=PACK_PATH_7)
pack_db = Pack.get_by_name("dummy_pack_7_name")
self.assertEqual(pack_db.ref, "dummy_pack_7_name")
# "ref" is not provided and "name" contains invalid characters
expected_msg = "contains invalid characters"
self.assertRaisesRegexp(
ValueError,
expected_msg,
registrar._register_pack_db,
pack_name=None,
pack_dir=PACK_PATH_8,
)
def run(self, pack):
packs_base_paths = get_packs_base_paths()
pack_git_info_path = None
for packs_base_path in packs_base_paths:
git_info_path = os.path.join(packs_base_path, pack, GITINFO_FILE)
if os.path.isfile(git_info_path):
pack_git_info_path = git_info_path
break
if not pack_git_info_path:
error = ('Pack "%s" doesn\'t exist or it doesn\'t contain a .gitinfo file' % (pack))
raise Exception(error)
try:
details = self._parse_git_info_file(git_info_path)
except Exception as e:
error = ('Pack "%s" doesn\'t contain a valid .gitinfo file: %s' % (pack, str(e)))
raise Exception(error)
return details
def register_policies(packs_base_paths=None,
pack_dir=None,
use_pack_cache=True,
fail_on_failure=False):
if packs_base_paths:
if not isinstance(packs_base_paths, list):
raise TypeError(
"The pack base paths has a value that is not a list"
f" (was {type(packs_base_paths)}).")
if not packs_base_paths:
packs_base_paths = content_utils.get_packs_base_paths()
registrar = PolicyRegistrar(use_pack_cache=use_pack_cache,
fail_on_failure=fail_on_failure)
if pack_dir:
result = registrar.register_from_pack(pack_dir=pack_dir)
else:
result = registrar.register_from_packs(base_dirs=packs_base_paths)
return result
def _get_api_models_from_disk(artifact_type, pack_dir=None):
loader = ContentPackLoader()
artifacts = None
if pack_dir:
artifacts_dir = loader.get_content_from_pack(pack_dir, artifact_type)
pack_name = os.path.basename(os.path.normpath(pack_dir))
artifacts = {pack_name: artifacts_dir}
else:
packs_dirs = content_utils.get_packs_base_paths()
artifacts = loader.get_content(packs_dirs, artifact_type)
artifacts_dict = {}
for pack_name, pack_path in artifacts.items():
artifacts_paths = registrar.get_resources_from_pack(pack_path)
for artifact_path in artifacts_paths:
artifact = meta_loader.load(artifact_path)
if artifact_type == 'sensors':
sensors_dir = os.path.dirname(artifact_path)
sensor_file_path = os.path.join(sensors_dir,
artifact['entry_point'])
artifact['artifact_uri'] = 'file://' + sensor_file_path
name = artifact.get('name', None) or artifact.get(
'class_name', None)
if not artifact.get('pack', None):
artifact['pack'] = pack_name
ref = ResourceReference.to_string_reference(name=name,
pack=pack_name)
API_MODEL = API_MODELS_ARTIFACT_TYPES[artifact_type]
# Following conversions are required because we add some fields with
# default values in db model. If we don't do these conversions,
# we'll see a unnecessary diff for those fields.
artifact_api = API_MODEL(**artifact)
artifact_db = API_MODEL.to_model(artifact_api)
artifact_api = API_MODEL.from_model(artifact_db)
artifacts_dict[ref] = artifact_api
return artifacts_dict
def test_register_all_configs_with_config_schema_validation_validation_failure_4(
self, ):
# This test checks for default values containing "decrypt_kv" jinja filter for
# keys which have "secret: True" set.
# Verify DB is empty
pack_dbs = Pack.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_dbs), 0)
registrar = ConfigsRegistrar(use_pack_cache=False,
fail_on_failure=True,
validate_configs=True)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {
"dummy_pack_22": PACK_22_PATH
}
# Register ConfigSchema for pack
registrar._register_pack_db = mock.Mock()
registrar._register_pack(pack_name="dummy_pack_22",
pack_dir=PACK_22_PATH)
packs_base_paths = content_utils.get_packs_base_paths()
expected_msg = (
'Values specified as "secret: True" in config schema are automatically '
'decrypted by default. Use of "decrypt_kv" jinja filter is not allowed '
"for such values. Please check the specified values in the config or "
"the default values in the schema.")
self.assertRaisesRegexp(
ValueError,
expected_msg,
registrar.register_from_packs,
base_dirs=packs_base_paths,
)
def test_register_all_configs_invalid_config_no_config_schema(self):
# verify_ configs is on, but ConfigSchema for the pack doesn't exist so
# validation should proceed normally
# Verify DB is empty
pack_dbs = Pack.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_dbs), 0)
registrar = ConfigsRegistrar(use_pack_cache=False, validate_configs=False)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {'dummy_pack_6': PACK_6_PATH}
packs_base_paths = content_utils.get_packs_base_paths()
registrar.register_from_packs(base_dirs=packs_base_paths)
# Verify pack and schema have been registered
pack_dbs = Pack.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 1)
self.assertEqual(len(config_dbs), 1)
def validate_action(action_api, runner_type_db=None):
"""
:param runner_type_db: RunnerTypeDB object belonging to this action. If not provided, it's
retrieved from the database.
:type runner_type_db: :class:`RunnerTypeDB`
"""
if not runner_type_db:
runner_db = get_runner_model(action_api)
else:
runner_db = runner_type_db
# Check if pack is valid.
if not _is_valid_pack(action_api.pack):
packs_base_paths = get_packs_base_paths()
packs_base_paths = ','.join(packs_base_paths)
msg = ('Content pack "%s" is not found or doesn\'t contain actions directory. '
'Searched in: %s' %
(action_api.pack, packs_base_paths))
raise ValueValidationException(msg)
# Check if parameters defined are valid.
action_ref = ResourceReference.to_string_reference(pack=action_api.pack, name=action_api.name)
_validate_parameters(action_ref, action_api.parameters, runner_db.runner_parameters)
def test_register_all_configs_with_config_schema_validation_validation_failure_1(
self):
# Verify DB is empty
pack_dbs = Pack.get_all()
config_dbs = Config.get_all()
self.assertEqual(len(pack_dbs), 0)
self.assertEqual(len(config_dbs), 0)
registrar = ConfigsRegistrar(use_pack_cache=False,
fail_on_failure=True,
validate_configs=True)
registrar._pack_loader.get_packs = mock.Mock()
registrar._pack_loader.get_packs.return_value = {
'dummy_pack_6': PACK_6_PATH
}
# Register ConfigSchema for pack
registrar._register_pack_db = mock.Mock()
registrar._register_pack(pack_name='dummy_pack_5',
pack_dir=PACK_6_PATH)
packs_base_paths = content_utils.get_packs_base_paths()
if six.PY3:
expected_msg = (
'Failed validating attribute "regions" in config for pack '
'"dummy_pack_6" (.*?): 1000 is not of type \'array\'')
else:
expected_msg = (
'Failed validating attribute "regions" in config for pack '
'"dummy_pack_6" (.*?): 1000 is not of type u\'array\'')
self.assertRaisesRegexp(ValueError,
expected_msg,
registrar.register_from_packs,
base_dirs=packs_base_paths)
def create_archive(include_logs, include_configs, include_content, include_system_info,
debug=False):
"""
Create an archive with debugging information.
:return: Path to the generated archive.
:rtype: ``str``
"""
date = date_utils.get_datetime_utc_now().strftime('%Y-%m-%d-%H:%M:%S')
values = {'hostname': socket.gethostname(), 'date': date}
output_file_name = OUTPUT_FILENAME_TEMPLATE % values
output_file_path = os.path.join('/tmp', output_file_name)
# 1. Create temporary directory with the final directory structure where we will move files
# which will be processed and included in the tarball
temp_dir_path = tempfile.mkdtemp()
output_paths = {
'logs': os.path.join(temp_dir_path, 'logs/'),
'configs': os.path.join(temp_dir_path, 'configs/'),
'content': os.path.join(temp_dir_path, 'content/'),
'system_info': os.path.join(temp_dir_path, 'system_info.yaml')
}
for directory_name in DIRECTORY_STRUCTURE:
full_path = os.path.join(temp_dir_path, directory_name)
os.mkdir(full_path)
# 2. Moves all the files to the temporary directory
LOG.info('Collecting files...')
# Logs
if include_logs:
LOG.debug('Including log files')
for file_path_glob in LOG_FILE_PATHS:
log_file_list = get_full_file_list(file_path_glob=file_path_glob)
copy_files(file_paths=log_file_list, destination=output_paths['logs'])
# Config files
if include_configs:
LOG.debug('Including config files')
copy_files(file_paths=CONFIG_FILE_PATHS, destination=output_paths['configs'])
# Content
if include_content:
LOG.debug('Including content')
packs_base_paths = get_packs_base_paths()
for index, packs_base_path in enumerate(packs_base_paths, 1):
dst = os.path.join(output_paths['content'], 'dir-%s' % (index))
try:
shutil.copytree(src=packs_base_path, dst=dst)
except IOError:
continue
# System information
if include_system_info:
LOG.debug('Including system info')
system_information = get_system_information()
system_information = yaml.dump(system_information, default_flow_style=False)
with open(output_paths['system_info'], 'w') as fp:
fp.write(system_information)
# Configs
st2_config_path = os.path.join(output_paths['configs'], ST2_CONFIG_FILE_NAME)
process_st2_config(config_path=st2_config_path)
mistral_config_path = os.path.join(output_paths['configs'], MISTRAL_CONFIG_FILE_NAME)
process_mistral_config(config_path=mistral_config_path)
# Content
base_pack_dirs = get_dirs_in_path(file_path=output_paths['content'])
for base_pack_dir in base_pack_dirs:
pack_dirs = get_dirs_in_path(file_path=base_pack_dir)
for pack_dir in pack_dirs:
process_content_pack_dir(pack_dir=pack_dir)
# 4. Create a tarball
LOG.info('Creating tarball...')
with tarfile.open(output_file_path, 'w:gz') as tar:
for file_path in output_paths.values():
file_path = os.path.normpath(file_path)
source_dir = file_path
if '.' in file_path:
arcname = os.path.basename(file_path)
else:
arcname = os.path.split(file_path)[-1]
tar.add(source_dir, arcname=arcname)
return output_file_path
def setup_virtualenvs(recreate_virtualenvs=False):
"""
Setup Python virtual environments for all the registered or the provided pack.
"""
LOG.info("=========================================================")
LOG.info("########### Setting up virtual environments #############")
LOG.info("=========================================================")
pack_dir = cfg.CONF.register.pack
fail_on_failure = not cfg.CONF.register.no_fail_on_failure
registrar = ResourceRegistrar()
if pack_dir:
pack_name = os.path.basename(pack_dir)
pack_names = [pack_name]
# 1. Register pack
registrar.register_pack(pack_name=pack_name, pack_dir=pack_dir)
else:
# 1. Register pack
base_dirs = content_utils.get_packs_base_paths()
registrar.register_packs(base_dirs=base_dirs)
# 2. Retrieve available packs (aka packs which have been registered)
pack_names = registrar.get_registered_packs()
if recreate_virtualenvs:
"""
update = False:
this is more than an update of an existing virtualenv
the virtualenv itself will be removed & recreated
this is i.e. useful for updates to a newer Python release
"""
update = False
else:
"""
update = True:
only dependencies inside the virtualenv will be updated
"""
update = True
setup_count = 0
for pack_name in pack_names:
try:
setup_pack_virtualenv(pack_name=pack_name,
update=update,
logger=LOG)
except Exception as e:
exc_info = not fail_on_failure
LOG.warning(
'Failed to setup virtualenv for pack "%s": %s',
pack_name,
e,
exc_info=exc_info,
)
if fail_on_failure:
raise e
else:
setup_count += 1
LOG.info("Setup virtualenv for %s pack(s)." % (setup_count))
def _register_pack_configs(self, validate_configs=False):
"""
Register all the packs inside the fixtures directory.
"""
registrar = ConfigsRegistrar(use_pack_cache=False, validate_configs=validate_configs)
registrar.register_from_packs(base_dirs=get_packs_base_paths())
def setup_pack_virtualenv(pack_name,
update=False,
logger=None,
include_pip=True,
include_setuptools=True,
include_wheel=True,
proxy_config=None,
no_download=True,
force_owner_group=True):
"""
Setup virtual environment for the provided pack.
:param pack_name: Name of the pack to setup the virtualenv for.
:type pack_name: ``str``
:param update: True to update dependencies inside the virtual environment.
:type update: ``bool``
:param logger: Optional logger instance to use. If not provided it defaults to the module
level logger.
:param no_download: Do not download and install latest version of pre-installed packages such
as pip and distutils.
:type no_download: ``bool``
"""
logger = logger or LOG
if not re.match(PACK_REF_WHITELIST_REGEX, pack_name):
raise ValueError('Invalid pack name "%s"' % (pack_name))
base_virtualenvs_path = os.path.join(cfg.CONF.system.base_path,
'virtualenvs/')
virtualenv_path = os.path.join(base_virtualenvs_path,
quote_unix(pack_name))
# Ensure pack directory exists in one of the search paths
pack_path = get_pack_directory(pack_name=pack_name)
logger.debug('Setting up virtualenv for pack "%s" (%s)' %
(pack_name, pack_path))
if not pack_path:
packs_base_paths = get_packs_base_paths()
search_paths = ', '.join(packs_base_paths)
msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name,
search_paths)
raise Exception(msg)
# 1. Create virtualenv if it doesn't exist
if not update or not os.path.exists(virtualenv_path):
# 0. Delete virtual environment if it exists
remove_virtualenv(virtualenv_path=virtualenv_path, logger=logger)
# 1. Create virtual environment
logger.debug('Creating virtualenv for pack "%s" in "%s"' %
(pack_name, virtualenv_path))
create_virtualenv(virtualenv_path=virtualenv_path,
logger=logger,
include_pip=include_pip,
include_setuptools=include_setuptools,
include_wheel=include_wheel,
no_download=no_download)
# 2. Install base requirements which are common to all the packs
logger.debug('Installing base requirements')
for requirement in BASE_PACK_REQUIREMENTS:
install_requirement(virtualenv_path=virtualenv_path,
requirement=requirement,
proxy_config=proxy_config,
logger=logger)
# 3. Install pack-specific requirements
requirements_file_path = os.path.join(pack_path, 'requirements.txt')
has_requirements = os.path.isfile(requirements_file_path)
if has_requirements:
logger.debug('Installing pack specific requirements from "%s"' %
(requirements_file_path))
install_requirements(virtualenv_path=virtualenv_path,
requirements_file_path=requirements_file_path,
proxy_config=proxy_config,
logger=logger)
else:
logger.debug('No pack specific requirements found')
# 4. Set the owner group
if force_owner_group:
apply_pack_owner_group(pack_path=virtualenv_path)
action = 'updated' if update else 'created'
logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
(pack_name, action, virtualenv_path))
def post(self, pack_register_request):
if pack_register_request and hasattr(pack_register_request, "types"):
types = pack_register_request.types
if "all" in types:
types = PackRegisterController.CONTENT_TYPES
else:
types = PackRegisterController.CONTENT_TYPES
if pack_register_request and hasattr(pack_register_request, "packs"):
packs = list(set(pack_register_request.packs))
else:
packs = None
result = defaultdict(int)
# Register depended resources (actions depend on runners, rules depend on rule types, etc)
if ("runner" in types or "runners" in types) or (
"action" in types or "actions" in types
):
result["runners"] = runners_registrar.register_runners(experimental=True)
if ("rule_type" in types or "rule_types" in types) or (
"rule" in types or "rules" in types
):
result["rule_types"] = rule_types_registrar.register_rule_types()
if ("policy_type" in types or "policy_types" in types) or (
"policy" in types or "policies" in types
):
result["policy_types"] = policies_registrar.register_policy_types(st2common)
use_pack_cache = False
fail_on_failure = getattr(pack_register_request, "fail_on_failure", True)
for type, (Registrar, name) in six.iteritems(ENTITIES):
if type in types or name in types:
registrar = Registrar(
use_pack_cache=use_pack_cache,
use_runners_cache=True,
fail_on_failure=fail_on_failure,
)
if packs:
for pack in packs:
pack_path = content_utils.get_pack_base_path(pack)
try:
registered_count = registrar.register_from_pack(
pack_dir=pack_path
)
result[name] += registered_count
except ValueError as e:
# Throw more user-friendly exception if requsted pack doesn't exist
if re.match(
'Directory ".*?" doesn\'t exist', six.text_type(e)
):
msg = 'Pack "%s" not found on disk: %s' % (
pack,
six.text_type(e),
)
raise ValueError(msg)
raise e
else:
packs_base_paths = content_utils.get_packs_base_paths()
registered_count = registrar.register_from_packs(
base_dirs=packs_base_paths
)
result[name] += registered_count
return result
def _setup_pack_virtualenv(self, pack_name, update=False):
"""
Setup virtual environment for the provided pack.
:param pack_name: Pack name.
:type pack_name: ``str``
"""
# Prevent directory traversal by whitelisting allowed characters in the
# pack name
if not re.match(PACK_NAME_WHITELIST, pack_name):
raise ValueError('Invalid pack name "%s"' % (pack_name))
self.logger.debug('Setting up virtualenv for pack "%s"' % (pack_name))
virtualenv_path = os.path.join(self._base_virtualenvs_path,
quote_unix(pack_name))
# Ensure pack directory exists in one of the search paths
pack_path = get_pack_directory(pack_name=pack_name)
if not pack_path:
packs_base_paths = get_packs_base_paths()
search_paths = ', '.join(packs_base_paths)
msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name,
search_paths)
raise Exception(msg)
if not os.path.exists(self._base_virtualenvs_path):
os.makedirs(self._base_virtualenvs_path)
# If we don't want to update, or if the virtualenv doesn't exist, let's create it.
if not update or not os.path.exists(virtualenv_path):
# 0. Delete virtual environment if it exists
self._remove_virtualenv(virtualenv_path=virtualenv_path)
# 1. Create virtual environment
self.logger.debug('Creating virtualenv for pack "%s" in "%s"' %
(pack_name, virtualenv_path))
self._create_virtualenv(virtualenv_path=virtualenv_path)
# 2. Install base requirements which are common to all the packs
self.logger.debug('Installing base requirements')
for requirement in BASE_PACK_REQUIREMENTS:
self._install_requirement(virtualenv_path=virtualenv_path,
requirement=requirement)
# 3. Install pack-specific requirements
requirements_file_path = os.path.join(pack_path, 'requirements.txt')
has_requirements = os.path.isfile(requirements_file_path)
if has_requirements:
self.logger.debug(
'Installing pack specific requirements from "%s"' %
(requirements_file_path))
self._install_requirements(virtualenv_path, requirements_file_path)
else:
self.logger.debug('No pack specific requirements found')
self.logger.debug(
'Virtualenv for pack "%s" successfully %s in "%s"' %
(pack_name, 'updated' if update else 'created', virtualenv_path))
def post(self, pack_register_request):
if pack_register_request and hasattr(pack_register_request, 'types'):
types = pack_register_request.types
else:
types = [
'runner', 'action', 'trigger', 'sensor', 'rule', 'rule_type',
'alias', 'policy_type', 'policy', 'config'
]
if pack_register_request and hasattr(pack_register_request, 'packs'):
packs = list(set(pack_register_request.packs))
else:
packs = None
result = defaultdict(int)
# Register depended resources (actions depend on runners, rules depend on rule types, etc)
if ('runner' in types or 'runners' in types) or ('action' in types or
'actions' in types):
result['runners'] = runners_registrar.register_runners(
experimental=True)
if ('rule_type' in types or 'rule_types' in types) or \
('rule' in types or 'rules' in types):
result['rule_types'] = rule_types_registrar.register_rule_types()
if ('policy_type' in types or 'policy_types' in types) or \
('policy' in types or 'policies' in types):
result['policy_types'] = policies_registrar.register_policy_types(
st2common)
use_pack_cache = False
fail_on_failure = getattr(pack_register_request, 'fail_on_failure',
True)
for type, (Registrar, name) in six.iteritems(ENTITIES):
if type in types or name in types:
registrar = Registrar(use_pack_cache=use_pack_cache,
fail_on_failure=fail_on_failure)
if packs:
for pack in packs:
pack_path = content_utils.get_pack_base_path(pack)
try:
registered_count = registrar.register_from_pack(
pack_dir=pack_path)
result[name] += registered_count
except ValueError as e:
# Throw more user-friendly exception if requsted pack doesn't exist
if re.match('Directory ".*?" doesn\'t exist',
str(e)):
msg = 'Pack "%s" not found on disk: %s' % (
pack, str(e))
raise ValueError(msg)
raise e
else:
packs_base_paths = content_utils.get_packs_base_paths()
registered_count = registrar.register_from_packs(
base_dirs=packs_base_paths)
result[name] += registered_count
return result
def _register_pack_configs(self):
"""
Register all the packs inside the fixtures directory.
"""
registrar = ConfigsRegistrar(use_pack_cache=False)
registrar.register_configs_for_all_packs(base_dirs=get_packs_base_paths())
def _register_packs(self):
"""
Register all the packs inside the fixtures directory.
"""
registrar = ResourceRegistrar(use_pack_cache=False)
registrar.register_packs(base_dirs=get_packs_base_paths())
def post(self, pack_register_request):
if pack_register_request and hasattr(pack_register_request, "types"):
types = pack_register_request.types
if "all" in types:
types = PackRegisterController.CONTENT_TYPES
else:
types = PackRegisterController.CONTENT_TYPES
if pack_register_request and hasattr(pack_register_request, "packs"):
packs = list(set(pack_register_request.packs))
else:
packs = None
result = defaultdict(int)
# Register depended resources (actions depend on runners, rules depend on rule types, etc)
if ("runner" in types or "runners" in types) or ("action" in types or
"actions" in types):
result["runners"] = runners_registrar.register_runners(
experimental=True)
if ("rule_type" in types or "rule_types"
in types) or ("rule" in types or "rules" in types):
result["rule_types"] = rule_types_registrar.register_rule_types()
if ("policy_type" in types or "policy_types"
in types) or ("policy" in types or "policies" in types):
result["policy_types"] = policies_registrar.register_policy_types(
st2common)
use_pack_cache = False
# TODO: To speed up this operation since it's mostli IO bound we could use green thread
# pool here and register different resources concurrently
fail_on_failure = getattr(pack_register_request, "fail_on_failure",
True)
for type, (Registrar, name) in six.iteritems(ENTITIES):
if type in types or name in types:
registrar = Registrar(
use_pack_cache=use_pack_cache,
use_runners_cache=True,
fail_on_failure=fail_on_failure,
)
if packs:
for pack in packs:
pack_path = content_utils.get_pack_base_path(pack)
try:
res = registrar.register_from_pack(
pack_dir=pack_path)
# Where overridding is supported return is tuple of
# (registered,overridden) else its just registered
# count return
if isinstance(res, tuple):
result[name] += res[0]
if res[1] != 0:
result[f"{name}(overridden)"] = res[1]
else:
result[name] += res
except ValueError as e:
# Throw more user-friendly exception if requsted pack doesn't exist
if re.match('Directory ".*?" doesn\'t exist',
six.text_type(e)):
msg = 'Pack "%s" not found on disk: %s' % (
pack,
six.text_type(e),
)
raise ValueError(msg)
raise e
else:
packs_base_paths = content_utils.get_packs_base_paths()
res = registrar.register_from_packs(
base_dirs=packs_base_paths)
# Where overridding is supported return is tuple of
# (registered,overridden) else its just registered
# count return
if isinstance(res, tuple):
result[name] += res[0]
if res[1] != 0:
result[f"{name}(overridden)"] = res[1]
else:
result[name] += res
return result
