def _get_pack_common_libs_path(self, pack_ref):
"""
Retrieve path to the pack common lib/ directory taking git work tree path into account
(if used).
"""
worktree_path = self.git_worktree_path
pack_common_libs_path = get_pack_common_libs_path_for_pack_ref(pack_ref=pack_ref)
if not worktree_path:
return pack_common_libs_path
# Modify the path so it uses git worktree directory
pack_base_path = get_pack_base_path(pack_name=pack_ref)
new_pack_common_libs_path = pack_common_libs_path.replace(pack_base_path, '')
# Remove leading slash (if any)
if new_pack_common_libs_path.startswith('/'):
new_pack_common_libs_path = new_pack_common_libs_path[1:]
new_pack_common_libs_path = os.path.join(worktree_path, new_pack_common_libs_path)
# Check to prevent directory traversal
common_prefix = os.path.commonprefix([worktree_path, new_pack_common_libs_path])
if common_prefix != worktree_path:
raise ValueError('pack libs path is not located inside the pack directory')
return new_pack_common_libs_path
def _register_triggers_from_pack(self, pack, triggers):
registered_count = 0
pack_base_path = content_utils.get_pack_base_path(
pack_name=pack, include_trailing_slash=True)
for trigger in triggers:
try:
self._register_trigger_from_pack(pack_base_path=pack_base_path,
pack=pack,
trigger=trigger)
except Exception as e:
if self._fail_on_failure:
msg = (
'Failed to register trigger "%s" from pack "%s": %s' %
(trigger, pack, six.text_type(e)))
raise ValueError(msg)
LOG.debug('Failed to register trigger "%s": %s', trigger,
six.text_type(e))
else:
LOG.debug('Trigger "%s" successfully registered', trigger)
registered_count += 1
return registered_count
def _get_entry_point_for_worktree_path(self, pack_name, entry_point,
worktree_path):
"""
Method which returns path to an action entry point which is located inside the git
worktree directory.
:rtype: ``str``
"""
pack_base_path = get_pack_base_path(pack_name=pack_name)
new_entry_point = entry_point.replace(pack_base_path, '')
# Remove leading slash (if any)
if new_entry_point.startswith('/'):
new_entry_point = new_entry_point[1:]
new_entry_point = os.path.join(worktree_path, new_entry_point)
# Check to prevent directory traversal
common_prefix = os.path.commonprefix([worktree_path, new_entry_point])
if common_prefix != worktree_path:
raise ValueError(
'entry_point is not located inside the pack directory')
return new_entry_point
def post(self, pack_register_request):
if pack_register_request and hasattr(pack_register_request, 'types'):
types = pack_register_request.types
if 'all' in types:
types = PackRegisterController.CONTENT_TYPES
else:
types = PackRegisterController.CONTENT_TYPES
if pack_register_request and hasattr(pack_register_request, 'packs'):
packs = list(set(pack_register_request.packs))
else:
packs = None
result = defaultdict(int)
# Register depended resources (actions depend on runners, rules depend on rule types, etc)
if ('runner' in types or 'runners' in types) or ('action' in types or
'actions' in types):
result['runners'] = runners_registrar.register_runners(
experimental=True)
if ('rule_type' in types or 'rule_types' in types) or \
('rule' in types or 'rules' in types):
result['rule_types'] = rule_types_registrar.register_rule_types()
if ('policy_type' in types or 'policy_types' in types) or \
('policy' in types or 'policies' in types):
result['policy_types'] = policies_registrar.register_policy_types(
st2common)
use_pack_cache = False
fail_on_failure = getattr(pack_register_request, 'fail_on_failure',
True)
for type, (Registrar, name) in six.iteritems(ENTITIES):
if type in types or name in types:
registrar = Registrar(use_pack_cache=use_pack_cache,
fail_on_failure=fail_on_failure)
if packs:
for pack in packs:
pack_path = content_utils.get_pack_base_path(pack)
try:
registered_count = registrar.register_from_pack(
pack_dir=pack_path)
result[name] += registered_count
except ValueError as e:
# Throw more user-friendly exception if requsted pack doesn't exist
if re.match('Directory ".*?" doesn\'t exist',
str(e)):
msg = 'Pack "%s" not found on disk: %s' % (
pack, str(e))
raise ValueError(msg)
raise e
else:
packs_base_paths = content_utils.get_packs_base_paths()
registered_count = registrar.register_from_packs(
base_dirs=packs_base_paths)
result[name] += registered_count
return result
def _get_pack_common_libs_path(self, pack_ref):
"""
Retrieve path to the pack common lib/ directory taking git work tree path into account
(if used).
"""
worktree_path = self.git_worktree_path
pack_common_libs_path = get_pack_common_libs_path_for_pack_ref(pack_ref=pack_ref)
if not worktree_path:
return pack_common_libs_path
# Modify the path so it uses git worktree directory
pack_base_path = get_pack_base_path(pack_name=pack_ref)
new_pack_common_libs_path = pack_common_libs_path.replace(pack_base_path, '')
# Remove leading slash (if any)
if new_pack_common_libs_path.startswith('/'):
new_pack_common_libs_path = new_pack_common_libs_path[1:]
new_pack_common_libs_path = os.path.join(worktree_path, new_pack_common_libs_path)
# Check to prevent directory traversal
common_prefix = os.path.commonprefix([worktree_path, new_pack_common_libs_path])
if common_prefix != worktree_path:
raise ValueError('pack libs path is not located inside the pack directory')
return new_pack_common_libs_path
def get_pack_version(pack=None):
pack_path = get_pack_base_path(pack)
try:
pack_metadata = get_pack_metadata(pack_dir=pack_path)
result = pack_metadata.get("version", None)
except Exception:
result = None
finally:
return result
def get_warnings(pack=None):
result = None
pack_path = get_pack_base_path(pack)
try:
pack_metadata = get_pack_metadata(pack_dir=pack_path)
result = get_pack_warnings(pack_metadata)
except Exception:
print("Could not open pack.yaml at location %s" % pack_path)
finally:
return result
def clone_action_files(source_action_db, dest_action_db, dest_pack_base_path):
"""
Prepares the path for entry point and metadata files for source and destination.
Clones the content from source action files to destination action files.
"""
source_pack = source_action_db["pack"]
source_entry_point = source_action_db["entry_point"]
source_metadata_file = source_action_db["metadata_file"]
source_pack_base_path = get_pack_base_path(pack_name=source_pack)
source_metadata_file_path = os.path.join(source_pack_base_path,
source_metadata_file)
dest_metadata_file_name = dest_action_db["metadata_file"]
dest_metadata_file_path = os.path.join(dest_pack_base_path,
dest_metadata_file_name)
# creating actions directory in destination pack if doesn't exist
ac_dir_path = os.path.join(dest_pack_base_path, "actions")
if not os.path.isdir(ac_dir_path):
os.mkdir(path=ac_dir_path)
_clone_content_to_destination_file(
source_file=source_metadata_file_path,
destination_file=dest_metadata_file_path)
dest_entry_point = dest_action_db["entry_point"]
dest_runner_type = dest_action_db["runner_type"]["name"]
if dest_entry_point:
if dest_runner_type in ["orquesta", "action-chain"]:
# creating workflows directory if doesn't exist
wf_dir_path = os.path.join(dest_pack_base_path, "actions",
"workflows")
if not os.path.isdir(wf_dir_path):
os.mkdir(path=wf_dir_path)
source_entry_point_file_path = os.path.join(source_pack_base_path,
"actions",
source_entry_point)
dest_entrypoint_file_path = os.path.join(dest_pack_base_path,
"actions", dest_entry_point)
_clone_content_to_destination_file(
source_file=source_entry_point_file_path,
destination_file=dest_entrypoint_file_path,
)
with open(dest_metadata_file_path) as df:
doc = yaml.load(df, Loader=yaml.FullLoader)
doc["name"] = dest_action_db["name"]
if "pack" in doc:
doc["pack"] = dest_action_db["pack"]
doc["entry_point"] = dest_entry_point
with open(dest_metadata_file_path, "w") as df:
yaml.dump(doc, df, default_flow_style=False, sort_keys=False)
def get_dependency_list(pack=None):
pack_path = get_pack_base_path(pack)
try:
pack_metadata = get_pack_metadata(pack_dir=pack_path)
result = pack_metadata.get("dependencies", None)
except Exception:
print("Could not open pack.yaml at location %s" % pack_path)
result = None
finally:
return result
def post(self, pack_register_request):
if pack_register_request and hasattr(pack_register_request, 'types'):
types = pack_register_request.types
if 'all' in types:
types = PackRegisterController.CONTENT_TYPES
else:
types = PackRegisterController.CONTENT_TYPES
if pack_register_request and hasattr(pack_register_request, 'packs'):
packs = list(set(pack_register_request.packs))
else:
packs = None
result = defaultdict(int)
# Register depended resources (actions depend on runners, rules depend on rule types, etc)
if ('runner' in types or 'runners' in types) or ('action' in types or 'actions' in types):
result['runners'] = runners_registrar.register_runners(experimental=True)
if ('rule_type' in types or 'rule_types' in types) or \
('rule' in types or 'rules' in types):
result['rule_types'] = rule_types_registrar.register_rule_types()
if ('policy_type' in types or 'policy_types' in types) or \
('policy' in types or 'policies' in types):
result['policy_types'] = policies_registrar.register_policy_types(st2common)
use_pack_cache = False
fail_on_failure = getattr(pack_register_request, 'fail_on_failure', True)
for type, (Registrar, name) in six.iteritems(ENTITIES):
if type in types or name in types:
registrar = Registrar(use_pack_cache=use_pack_cache,
use_runners_cache=True,
fail_on_failure=fail_on_failure)
if packs:
for pack in packs:
pack_path = content_utils.get_pack_base_path(pack)
try:
registered_count = registrar.register_from_pack(pack_dir=pack_path)
result[name] += registered_count
except ValueError as e:
# Throw more user-friendly exception if requsted pack doesn't exist
if re.match('Directory ".*?" doesn\'t exist', six.text_type(e)):
msg = 'Pack "%s" not found on disk: %s' % (pack, six.text_type(e))
raise ValueError(msg)
raise e
else:
packs_base_paths = content_utils.get_packs_base_paths()
registered_count = registrar.register_from_packs(base_dirs=packs_base_paths)
result[name] += registered_count
return result
def get_sandbox_python_path_for_python_action(pack,
inherit_from_parent=True,
inherit_parent_virtualenv=True):
"""
Return sandbox PYTHONPATH for a particular Python runner action.
Same as get_sandbox_python_path() function, but it's intended to be used for Python runner
actions.
"""
sandbox_python_path = get_sandbox_python_path(
inherit_from_parent=inherit_from_parent,
inherit_parent_virtualenv=inherit_parent_virtualenv)
pack_base_path = get_pack_base_path(pack_name=pack)
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
if virtualenv_path and os.path.isdir(virtualenv_path):
pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib')
virtualenv_directories = os.listdir(pack_virtualenv_lib_path)
virtualenv_directories = [
dir_name for dir_name in virtualenv_directories
if fnmatch.fnmatch(dir_name, 'python*')
]
# Add the pack's lib directory (lib/python3.x) in front of the PYTHONPATH.
pack_actions_lib_paths = os.path.join(pack_base_path, 'actions', 'lib')
pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib')
pack_venv_lib_directory = os.path.join(pack_virtualenv_lib_path,
virtualenv_directories[0])
# Add the pack's site-packages directory (lib/python3.x/site-packages)
# in front of the Python system site-packages This is important because
# we want Python 3 compatible libraries to be used from the pack virtual
# environment and not system ones.
pack_venv_site_packages_directory = os.path.join(
pack_virtualenv_lib_path, virtualenv_directories[0],
'site-packages')
full_sandbox_python_path = [
# NOTE: Order here is very important for imports to function correctly
pack_venv_lib_directory,
pack_venv_site_packages_directory,
pack_actions_lib_paths,
sandbox_python_path,
]
sandbox_python_path = ':'.join(full_sandbox_python_path)
return sandbox_python_path
def _write_data_file(self, pack_name, file_path, content):
"""
Write data file on disk.
"""
# Throw if pack directory doesn't exist
pack_base_path = get_pack_base_path(pack_name=pack_name)
if not os.path.isdir(pack_base_path):
raise ValueError('Directory for pack "%s" doesn\'t exist' % (pack_name))
# Create pack sub-directory tree if it doesn't exist
directory = os.path.dirname(file_path)
if not os.path.isdir(directory):
os.makedirs(directory)
with open(file_path, 'w') as fp:
fp.write(content)
def _write_data_file(self, pack_name, file_path, content):
"""
Write data file on disk.
"""
# Throw if pack directory doesn't exist
pack_base_path = get_pack_base_path(pack_name=pack_name)
if not os.path.isdir(pack_base_path):
raise ValueError('Directory for pack "%s" doesn\'t exist' % (pack_name))
# Create pack sub-directory tree if it doesn't exist
directory = os.path.dirname(file_path)
if not os.path.isdir(directory):
os.makedirs(directory)
with open(file_path, 'w') as fp:
fp.write(content)
def test_get_pack_base_paths(self):
cfg.CONF.content.system_packs_base_path = ''
cfg.CONF.content.packs_base_paths = '/opt/path1'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1'])
# Multiple paths, no trailing colon
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Multiple paths, trailing colon
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Multiple same paths
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:/opt/path1:/opt/path2'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Assert system path is always first
cfg.CONF.content.system_packs_base_path = '/opt/system'
cfg.CONF.content.packs_base_paths = '/opt/path2:/opt/path1'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/system', '/opt/path2', '/opt/path1'])
# More scenarios
orig_path = cfg.CONF.content.system_packs_base_path
cfg.CONF.content.system_packs_base_path = '/tests/packs'
names = [
'test_pack_1',
'test_pack_2',
'ma_pack'
]
for name in names:
actual = get_pack_base_path(pack_name=name)
expected = os.path.join(cfg.CONF.content.system_packs_base_path,
name)
self.assertEqual(actual, expected)
cfg.CONF.content.system_packs_base_path = orig_path
def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True,
inherit_parent_virtualenv=True):
"""
Same as get_sandbox_python_path function, but it's intended to be used for Python runner actions
and also takes into account if a pack virtual environment uses Python 3.
"""
sandbox_python_path = get_sandbox_python_path(
inherit_from_parent=inherit_from_parent,
inherit_parent_virtualenv=inherit_parent_virtualenv)
# If python3.? directory exists in pack virtualenv lib/ path it means Python 3 is used by
# that virtual environment and we take that in to account when constructing PYTHONPATH
pack_base_path = get_pack_base_path(pack_name=pack)
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
if virtualenv_path and os.path.isdir(virtualenv_path):
pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/')
pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib')
virtualenv_directories = os.listdir(pack_virtualenv_lib_path)
virtualenv_directories = [dir_name for dir_name in virtualenv_directories if
fnmatch.fnmatch(dir_name, 'python3*')]
uses_python3 = bool(virtualenv_directories)
else:
uses_python3 = False
if uses_python3:
# Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid
# issues with scripts trying to use packages / modules from Python 2.7 site-packages
# directory instead of the versions from Python 3 stdlib.
python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0])
# Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python
# 2.7 system site-packages This is important because we want Python 3 compatible libraries
# to be used from the pack virtual environment and not system ones.
python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path,
virtualenv_directories[0],
'site-packages')
sandbox_python_path = (python3_lib_directory + ':' + python3_site_packages_directory + ':' +
pack_actions_lib_paths + ':' + sandbox_python_path)
return sandbox_python_path
def get_sandbox_python_path_for_python_action(pack,
inherit_from_parent=True,
inherit_parent_virtualenv=True):
"""
Return sandbox PYTHONPATH for a particular Python runner action.
Same as get_sandbox_python_path() function, but it's intended to be used for Python runner
actions and also takes into account if a pack virtual environment uses Python 3.
"""
sandbox_python_path = get_sandbox_python_path(
inherit_from_parent=inherit_from_parent,
inherit_parent_virtualenv=inherit_parent_virtualenv)
pack_base_path = get_pack_base_path(pack_name=pack)
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
if not virtualenv_path:
return sandbox_python_path
uses_python3, virtualenv_directories = is_pack_virtualenv_using_python3(
pack=pack)
if uses_python3:
# Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid
# issues with scripts trying to use packages / modules from Python 2.7 site-packages
# directory instead of the versions from Python 3 stdlib.
pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/')
pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib')
python3_lib_directory = os.path.join(pack_virtualenv_lib_path,
virtualenv_directories[0])
# Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python
# 2.7 system site-packages This is important because we want Python 3 compatible libraries
# to be used from the pack virtual environment and not system ones.
python3_site_packages_directory = os.path.join(
pack_virtualenv_lib_path, virtualenv_directories[0],
'site-packages')
sandbox_python_path = (python3_lib_directory + ':' +
python3_site_packages_directory + ':' +
pack_actions_lib_paths + ':' +
sandbox_python_path)
return sandbox_python_path
def post(self, pack_register_request):
if pack_register_request and hasattr(pack_register_request, 'types'):
types = pack_register_request.types
else:
types = [
'runner', 'action', 'trigger', 'sensor', 'rule', 'rule_type',
'alias', 'policy_type', 'policy', 'config'
]
if pack_register_request and hasattr(pack_register_request, 'packs'):
packs = pack_register_request.packs
else:
packs = None
result = {}
if 'runner' in types or 'action' in types:
result['runners'] = runners_registrar.register_runners(
experimental=True)
if 'rule_type' in types or 'rule' in types:
result['rule_types'] = rule_types_registrar.register_rule_types()
if 'policy_type' in types or 'policy' in types:
result['policy_types'] = policies_registrar.register_policy_types(
st2common)
use_pack_cache = False
for type, (Registrar, name) in six.iteritems(ENTITIES):
if type in types:
registrar = Registrar(use_pack_cache=use_pack_cache,
fail_on_failure=False)
if packs:
for pack in packs:
pack_path = content_utils.get_pack_base_path(pack)
result[name] = registrar.register_from_pack(
pack_dir=pack_path)
else:
packs_base_paths = content_utils.get_packs_base_paths()
result[name] = registrar.register_from_packs(
base_dirs=packs_base_paths)
return result
def _write_data_file(self, pack_ref, file_path, content):
"""
Write data file on disk.
"""
# Throw if pack directory doesn't exist
pack_base_path = get_pack_base_path(pack_name=pack_ref)
if not os.path.isdir(pack_base_path):
raise ValueError('Directory for pack "%s" doesn\'t exist' % (pack_ref))
# Create pack sub-directory tree if it doesn't exist
directory = os.path.dirname(file_path)
if not os.path.isdir(directory):
# NOTE: We apply same permission bits as we do on pack install. If we don't do that,
# st2api won't be able to write to pack sub-directory
mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
os.makedirs(directory, mode)
with open(file_path, 'w') as fp:
fp.write(content)
def _write_data_file(self, pack_ref, file_path, content):
"""
Write data file on disk.
"""
# Throw if pack directory doesn't exist
pack_base_path = get_pack_base_path(pack_name=pack_ref)
if not os.path.isdir(pack_base_path):
raise ValueError('Directory for pack "%s" doesn\'t exist' % (pack_ref))
# Create pack sub-directory tree if it doesn't exist
directory = os.path.dirname(file_path)
if not os.path.isdir(directory):
# NOTE: We apply same permission bits as we do on pack install. If we don't do that,
# st2api won't be able to write to pack sub-directory
mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
os.makedirs(directory, mode)
with open(file_path, 'w') as fp:
fp.write(content)
def run(self, template_pack, template_path, context, include_execution,
include_six):
if include_six:
context['six'] = six
if include_execution:
execution = self._get_execution(include_execution)
context['__execution'] = execution
pack_path = content_utils.get_pack_base_path(template_pack)
abs_template_path = os.path.abspath(
os.path.join(pack_path, template_path))
if not abs_template_path.startswith(pack_path):
raise ValueError(
'Template_path points to a file outside pack directory.')
with open(abs_template_path, 'r') as f:
template = f.read()
return self.jinja.from_string(template).render(context)
def test_get_pack_base_paths(self):
cfg.CONF.content.system_packs_base_path = ''
cfg.CONF.content.packs_base_paths = '/opt/path1'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1'])
# Multiple paths, no trailing colon
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Multiple paths, trailing colon
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Multiple same paths
cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:/opt/path1:/opt/path2'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/path1', '/opt/path2'])
# Assert system path is always first
cfg.CONF.content.system_packs_base_path = '/opt/system'
cfg.CONF.content.packs_base_paths = '/opt/path2:/opt/path1'
result = get_packs_base_paths()
self.assertEqual(result, ['/opt/system', '/opt/path2', '/opt/path1'])
# More scenarios
orig_path = cfg.CONF.content.system_packs_base_path
cfg.CONF.content.system_packs_base_path = '/tests/packs'
names = ['test_pack_1', 'test_pack_2', 'ma_pack']
for name in names:
actual = get_pack_base_path(pack_name=name)
expected = os.path.join(cfg.CONF.content.system_packs_base_path,
name)
self.assertEqual(actual, expected)
cfg.CONF.content.system_packs_base_path = orig_path
def _register_triggers_from_pack(self, pack, triggers):
registered_count = 0
pack_base_path = content_utils.get_pack_base_path(pack_name=pack,
include_trailing_slash=True)
for trigger in triggers:
try:
self._register_trigger_from_pack(pack_base_path=pack_base_path, pack=pack,
trigger=trigger)
except Exception as e:
if self._fail_on_failure:
msg = ('Failed to register trigger "%s" from pack "%s": %s' % (trigger, pack,
six.text_type(e)))
raise ValueError(msg)
LOG.debug('Failed to register trigger "%s": %s', trigger, six.text_type(e))
else:
LOG.debug('Trigger "%s" successfully registered', trigger)
registered_count += 1
return registered_count
def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True,
inherit_parent_virtualenv=True):
"""
Same as get_sandbox_python_path function, but it's intended to be used for Python runner actions
and also takes into account if a pack virtual environment uses Python 3.
"""
sandbox_python_path = get_sandbox_python_path(
inherit_from_parent=inherit_from_parent,
inherit_parent_virtualenv=inherit_parent_virtualenv)
# If python3.? directory exists in pack virtualenv lib/ path it means Python 3 is used by
# that virtual environment and we take that in to account when constructing PYTHONPATH
pack_base_path = get_pack_base_path(pack_name=pack)
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
if virtualenv_path and os.path.isdir(virtualenv_path):
pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/')
pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib')
virtualenv_directories = os.listdir(pack_virtualenv_lib_path)
virtualenv_directories = [dir_name for dir_name in virtualenv_directories if
fnmatch.fnmatch(dir_name, 'python3*')]
uses_python3 = bool(virtualenv_directories)
else:
uses_python3 = False
if uses_python3:
# Add Python 3 lib/site-packages directory infront of the system site packages
# This is important because we want Python 3 compatible libraries to be used from
# the pack virtual environment and not system ones
python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path,
virtualenv_directories[0],
'site-packages')
sandbox_python_path = (python3_site_packages_directory + ':' + pack_actions_lib_paths +
':' + sandbox_python_path)
return sandbox_python_path
def _get_entry_point_for_worktree_path(self, pack_name, entry_point, worktree_path):
"""
Method which returns path to an action entry point which is located inside the git
worktree directory.
:rtype: ``str``
"""
pack_base_path = get_pack_base_path(pack_name=pack_name)
new_entry_point = entry_point.replace(pack_base_path, '')
# Remove leading slash (if any)
if new_entry_point.startswith('/'):
new_entry_point = new_entry_point[1:]
new_entry_point = os.path.join(worktree_path, new_entry_point)
# Check to prevent directory traversal
common_prefix = os.path.commonprefix([worktree_path, new_entry_point])
if common_prefix != worktree_path:
raise ValueError('entry_point is not located inside the pack directory')
return new_entry_point
def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True,
inherit_parent_virtualenv=True):
"""
Return sandbox PYTHONPATH for a particular Python runner action.
Same as get_sandbox_python_path() function, but it's intended to be used for Python runner
actions and also takes into account if a pack virtual environment uses Python 3.
"""
sandbox_python_path = get_sandbox_python_path(
inherit_from_parent=inherit_from_parent,
inherit_parent_virtualenv=inherit_parent_virtualenv)
pack_base_path = get_pack_base_path(pack_name=pack)
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
if not virtualenv_path:
return sandbox_python_path
uses_python3, virtualenv_directories = is_pack_virtualenv_using_python3(pack=pack)
if uses_python3:
# Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid
# issues with scripts trying to use packages / modules from Python 2.7 site-packages
# directory instead of the versions from Python 3 stdlib.
pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/')
pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib')
python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0])
# Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python
# 2.7 system site-packages This is important because we want Python 3 compatible libraries
# to be used from the pack virtual environment and not system ones.
python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path,
virtualenv_directories[0],
'site-packages')
sandbox_python_path = (python3_lib_directory + ':' + python3_site_packages_directory + ':' +
pack_actions_lib_paths + ':' + sandbox_python_path)
return sandbox_python_path
def post(self, pack_register_request):
if pack_register_request and hasattr(pack_register_request, 'types'):
types = pack_register_request.types
else:
types = ['runner', 'action', 'trigger', 'sensor', 'rule', 'rule_type', 'alias',
'policy_type', 'policy', 'config']
if pack_register_request and hasattr(pack_register_request, 'packs'):
packs = pack_register_request.packs
else:
packs = None
result = {}
if 'runner' in types or 'action' in types:
result['runners'] = runners_registrar.register_runners(experimental=True)
if 'rule_type' in types or 'rule' in types:
result['rule_types'] = rule_types_registrar.register_rule_types()
if 'policy_type' in types or 'policy' in types:
result['policy_types'] = policies_registrar.register_policy_types(st2common)
use_pack_cache = False
for type, (Registrar, name) in six.iteritems(ENTITIES):
if type in types:
registrar = Registrar(use_pack_cache=use_pack_cache,
fail_on_failure=False)
if packs:
for pack in packs:
pack_path = content_utils.get_pack_base_path(pack)
result[name] = registrar.register_from_pack(pack_dir=pack_path)
else:
packs_base_paths = content_utils.get_packs_base_paths()
result[name] = registrar.register_from_packs(base_dirs=packs_base_paths)
return result
def __init__(self, pack_name):
self.pack_name = pack_name
self.pack_path = utils.get_pack_base_path(pack_name=pack_name)
def __init__(self, pack_name):
self.pack_name = pack_name
self.pack_path = utils.get_pack_base_path(pack_name=pack_name)
def delete_action_files_from_pack(pack_name, entry_point, metadata_file):
"""
Prepares the path for entry_point file and metadata file of action and
deletes them from disk.
"""
pack_base_path = get_pack_base_path(pack_name=pack_name)
action_entrypoint_file_path = os.path.join(pack_base_path, "actions", entry_point)
action_metadata_file_path = os.path.join(pack_base_path, metadata_file)
if os.path.isfile(action_entrypoint_file_path):
try:
os.remove(action_entrypoint_file_path)
except PermissionError:
LOG.error(
'No permission to delete the "%s" file',
action_entrypoint_file_path,
)
msg = 'No permission to delete "%s" file from disk' % (
action_entrypoint_file_path
)
raise PermissionError(msg)
except Exception as e:
LOG.error(
'Unable to delete "%s" file. Exception was "%s"',
action_entrypoint_file_path,
e,
)
msg = (
'The action file "%s" could not be removed from disk, please '
"check the logs or ask your StackStorm administrator to check "
"and delete the actions files manually" % (action_entrypoint_file_path)
)
raise ResourceDiskFilesRemovalError(msg)
else:
LOG.warning(
'The action entry point file "%s" does not exists on disk.',
action_entrypoint_file_path,
)
if os.path.isfile(action_metadata_file_path):
try:
os.remove(action_metadata_file_path)
except PermissionError:
LOG.error(
'No permission to delete the "%s" file',
action_metadata_file_path,
)
msg = 'No permission to delete "%s" file from disk' % (
action_metadata_file_path
)
raise PermissionError(msg)
except Exception as e:
LOG.error(
'Could not delete "%s" file. Exception was "%s"',
action_metadata_file_path,
e,
)
msg = (
'The action file "%s" could not be removed from disk, please '
"check the logs or ask your StackStorm administrator to check "
"and delete the actions files manually" % (action_metadata_file_path)
)
raise ResourceDiskFilesRemovalError(msg)
else:
LOG.warning(
'The action metadata file "%s" does not exists on disk.',
action_metadata_file_path,
)
def __init__(self, pack_name, user=None):
self.pack_name = pack_name
self.user = user or cfg.CONF.system_user.user
self.pack_path = content_utils.get_pack_base_path(pack_name=pack_name)
self._config_parser = ContentPackConfigParser(pack_name=pack_name)
def __init__(self, pack_name, user=None):
self.pack_name = pack_name
self.user = user or cfg.CONF.system_user.user
self.pack_path = content_utils.get_pack_base_path(pack_name=pack_name)
self._config_parser = ContentPackConfigParser(pack_name=pack_name)
def post(self, pack_register_request):
if pack_register_request and hasattr(pack_register_request, "types"):
types = pack_register_request.types
if "all" in types:
types = PackRegisterController.CONTENT_TYPES
else:
types = PackRegisterController.CONTENT_TYPES
if pack_register_request and hasattr(pack_register_request, "packs"):
packs = list(set(pack_register_request.packs))
else:
packs = None
result = defaultdict(int)
# Register depended resources (actions depend on runners, rules depend on rule types, etc)
if ("runner" in types or "runners" in types) or ("action" in types or
"actions" in types):
result["runners"] = runners_registrar.register_runners(
experimental=True)
if ("rule_type" in types or "rule_types"
in types) or ("rule" in types or "rules" in types):
result["rule_types"] = rule_types_registrar.register_rule_types()
if ("policy_type" in types or "policy_types"
in types) or ("policy" in types or "policies" in types):
result["policy_types"] = policies_registrar.register_policy_types(
st2common)
use_pack_cache = False
# TODO: To speed up this operation since it's mostli IO bound we could use green thread
# pool here and register different resources concurrently
fail_on_failure = getattr(pack_register_request, "fail_on_failure",
True)
for type, (Registrar, name) in six.iteritems(ENTITIES):
if type in types or name in types:
registrar = Registrar(
use_pack_cache=use_pack_cache,
use_runners_cache=True,
fail_on_failure=fail_on_failure,
)
if packs:
for pack in packs:
pack_path = content_utils.get_pack_base_path(pack)
try:
registered_count = registrar.register_from_pack(
pack_dir=pack_path)
result[name] += registered_count
except ValueError as e:
# Throw more user-friendly exception if requsted pack doesn't exist
if re.match('Directory ".*?" doesn\'t exist',
six.text_type(e)):
msg = 'Pack "%s" not found on disk: %s' % (
pack,
six.text_type(e),
)
raise ValueError(msg)
raise e
else:
packs_base_paths = content_utils.get_packs_base_paths()
registered_count = registrar.register_from_packs(
base_dirs=packs_base_paths)
result[name] += registered_count
return result
def clone(self, dest_data, ref_or_id, requester_user):
"""
Clone an action from source pack to destination pack.
Handles requests:
POST /actions/{ref_or_id}/clone
"""
source_action_db = self._get_by_ref_or_id(ref_or_id=ref_or_id)
if not source_action_db:
msg = "The requested source for cloning operation doesn't exists"
abort(http_client.BAD_REQUEST, six.text_type(msg))
extra = {"action_db": source_action_db}
LOG.audit("Source action found. Action.id=%s" % (source_action_db.id),
extra=extra)
try:
permission_type = PermissionType.ACTION_VIEW
rbac_utils = get_rbac_backend().get_utils_class()
rbac_utils.assert_user_has_resource_db_permission(
user_db=requester_user,
resource_db=source_action_db,
permission_type=permission_type,
)
except ResourceAccessDeniedError as e:
abort(http_client.UNAUTHORIZED, six.text_type(e))
cloned_dest_action_db = clone_action_db(
source_action_db=source_action_db,
dest_pack=dest_data.dest_pack,
dest_action=dest_data.dest_action,
)
cloned_action_api = ActionAPI.from_model(cloned_dest_action_db)
try:
permission_type = PermissionType.ACTION_CREATE
rbac_utils.assert_user_has_resource_api_permission(
user_db=requester_user,
resource_api=cloned_action_api,
permission_type=permission_type,
)
except ResourceAccessDeniedError as e:
abort(http_client.UNAUTHORIZED, six.text_type(e))
dest_pack_base_path = get_pack_base_path(pack_name=dest_data.dest_pack)
if not os.path.isdir(dest_pack_base_path):
msg = "Destination pack '%s' doesn't exist" % (dest_data.dest_pack)
abort(http_client.BAD_REQUEST, six.text_type(msg))
dest_pack_base_path = get_pack_base_path(pack_name=dest_data.dest_pack)
dest_ref = ".".join([dest_data.dest_pack, dest_data.dest_action])
dest_action_db = self._get_by_ref(resource_ref=dest_ref)
try:
validate_not_part_of_system_pack_by_name(dest_data.dest_pack)
except ValueValidationException as e:
abort(http_client.BAD_REQUEST, six.text_type(e))
if dest_action_db:
if not dest_data.overwrite:
msg = "The requested destination action already exists"
abort(http_client.BAD_REQUEST, six.text_type(msg))
try:
permission_type = PermissionType.ACTION_DELETE
rbac_utils.assert_user_has_resource_db_permission(
user_db=requester_user,
resource_db=dest_action_db,
permission_type=permission_type,
)
options = GenericRequestParam(remove_files=True)
dest_metadata_file = dest_action_db["metadata_file"]
dest_entry_point = dest_action_db["entry_point"]
temp_sub_dir = str(uuid.uuid4())
temp_backup_action_files(
dest_pack_base_path,
dest_metadata_file,
dest_entry_point,
temp_sub_dir,
)
self.delete(options, dest_ref, requester_user)
except ResourceAccessDeniedError as e:
abort(http_client.UNAUTHORIZED, six.text_type(e))
except Exception as e:
LOG.debug(
"Exception encountered during deleting existing destination action. "
"Exception was: %s",
e,
)
abort(http_client.INTERNAL_SERVER_ERROR, six.text_type(e))
try:
post_response = self.post(cloned_action_api, requester_user)
if post_response.status_code != http_client.CREATED:
raise Exception("Could not add cloned action to database.")
cloned_dest_action_db["id"] = post_response.json["id"]
clone_action_files(
source_action_db=source_action_db,
dest_action_db=cloned_dest_action_db,
dest_pack_base_path=dest_pack_base_path,
)
extra = {"cloned_acion_db": cloned_dest_action_db}
LOG.audit("Action cloned. Action.id=%s" %
(cloned_dest_action_db.id),
extra=extra)
if dest_action_db:
remove_temp_action_files(temp_sub_dir)
return post_response
except PermissionError as e:
LOG.error("No permission to clone the action. Exception was %s", e)
delete_action_files_from_pack(
pack_name=cloned_dest_action_db["pack"],
entry_point=cloned_dest_action_db["entry_point"],
metadata_file=cloned_dest_action_db["metadata_file"],
)
if post_response.status_code == http_client.CREATED:
Action.delete(cloned_dest_action_db)
if dest_action_db:
self._restore_action(dest_action_db, dest_pack_base_path,
temp_sub_dir)
abort(http_client.FORBIDDEN, six.text_type(e))
except Exception as e:
LOG.error(
"Exception encountered during cloning action. Exception was %s",
e,
)
delete_action_files_from_pack(
pack_name=cloned_dest_action_db["pack"],
entry_point=cloned_dest_action_db["entry_point"],
metadata_file=cloned_dest_action_db["metadata_file"],
)
if post_response.status_code == http_client.CREATED:
Action.delete(cloned_dest_action_db)
if dest_action_db:
self._restore_action(dest_action_db, dest_pack_base_path,
temp_sub_dir)
abort(http_client.INTERNAL_SERVER_ERROR, six.text_type(e))
def get_pack_base_path(pack_name):
return utils.get_pack_base_path(pack_name)
def get_pack_base_path(pack_name):
return utils.get_pack_base_path(pack_name)
def get_sandbox_python_path_for_python_action(pack,
inherit_from_parent=True,
inherit_parent_virtualenv=True):
"""
Return sandbox PYTHONPATH for a particular Python runner action.
Same as get_sandbox_python_path() function, but it's intended to be used for Python runner
actions and also takes into account if a pack virtual environment uses Python 3.
"""
sandbox_python_path = get_sandbox_python_path(
inherit_from_parent=inherit_from_parent,
inherit_parent_virtualenv=inherit_parent_virtualenv)
pack_base_path = get_pack_base_path(pack_name=pack)
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
if not virtualenv_path:
return sandbox_python_path
uses_python3, virtualenv_directories = is_pack_virtualenv_using_python3(
pack=pack)
if uses_python3:
# Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid
# issues with scripts trying to use packages / modules from Python 2.7 site-packages
# directory instead of the versions from Python 3 stdlib.
pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/')
pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib')
python3_lib_directory = os.path.join(pack_virtualenv_lib_path,
virtualenv_directories[0])
# Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python
# 2.7 system site-packages This is important because we want Python 3 compatible libraries
# to be used from the pack virtual environment and not system ones.
python3_site_packages_directory = os.path.join(
pack_virtualenv_lib_path, virtualenv_directories[0],
'site-packages')
# Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv
# one (e.g. /usr/lib/python3.6)
# NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2
system_prefix_dirs = []
# Take custom prefix into account (if specified)
if cfg.CONF.actionrunner.python3_prefix:
system_prefix_dirs.append(cfg.CONF.actionrunner.python3_prefix)
# By default, Python libs are installed either in /usr/lib/python3.x or
# /usr/local/lib/python3.x
system_prefix_dirs.extend(['/usr/lib', '/usr/local/lib'])
for system_prefix_dir in system_prefix_dirs:
python3_system_lib_directory = os.path.join(
system_prefix_dir, virtualenv_directories[0])
if os.path.exists(python3_system_lib_directory):
break
if not python3_system_lib_directory or not os.path.exists(
python3_system_lib_directory):
python3_system_lib_directory = None
full_sandbox_python_path = []
# NOTE: Order here is very important for imports to function correctly
if python3_system_lib_directory:
full_sandbox_python_path.append(python3_system_lib_directory)
full_sandbox_python_path.append(python3_lib_directory)
full_sandbox_python_path.append(python3_site_packages_directory)
full_sandbox_python_path.append(pack_actions_lib_paths)
full_sandbox_python_path.append(sandbox_python_path)
sandbox_python_path = ':'.join(full_sandbox_python_path)
return sandbox_python_path
def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True,
inherit_parent_virtualenv=True):
"""
Return sandbox PYTHONPATH for a particular Python runner action.
Same as get_sandbox_python_path() function, but it's intended to be used for Python runner
actions and also takes into account if a pack virtual environment uses Python 3.
"""
sandbox_python_path = get_sandbox_python_path(
inherit_from_parent=inherit_from_parent,
inherit_parent_virtualenv=inherit_parent_virtualenv)
pack_base_path = get_pack_base_path(pack_name=pack)
virtualenv_path = get_sandbox_virtualenv_path(pack=pack)
if not virtualenv_path:
return sandbox_python_path
uses_python3, virtualenv_directories = is_pack_virtualenv_using_python3(pack=pack)
if uses_python3:
# Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid
# issues with scripts trying to use packages / modules from Python 2.7 site-packages
# directory instead of the versions from Python 3 stdlib.
pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/')
pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib')
python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0])
# Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python
# 2.7 system site-packages This is important because we want Python 3 compatible libraries
# to be used from the pack virtual environment and not system ones.
python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path,
virtualenv_directories[0],
'site-packages')
# Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv
# one (e.g. /usr/lib/python3.6)
# NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2
system_prefix_dirs = []
# Take custom prefix into account (if specified)
if cfg.CONF.actionrunner.python3_prefix:
system_prefix_dirs.append(cfg.CONF.actionrunner.python3_prefix)
# By default, Python libs are installed either in /usr/lib/python3.x or
# /usr/local/lib/python3.x
system_prefix_dirs.extend(['/usr/lib', '/usr/local/lib'])
for system_prefix_dir in system_prefix_dirs:
python3_system_lib_directory = os.path.join(system_prefix_dir,
virtualenv_directories[0])
if os.path.exists(python3_system_lib_directory):
break
if not python3_system_lib_directory or not os.path.exists(python3_system_lib_directory):
python3_system_lib_directory = None
full_sandbox_python_path = []
# NOTE: Order here is very important for imports to function correctly
if python3_system_lib_directory:
full_sandbox_python_path.append(python3_system_lib_directory)
full_sandbox_python_path.append(python3_lib_directory)
full_sandbox_python_path.append(python3_site_packages_directory)
full_sandbox_python_path.append(pack_actions_lib_paths)
full_sandbox_python_path.append(sandbox_python_path)
sandbox_python_path = ':'.join(full_sandbox_python_path)
return sandbox_python_path