def _get_pack_common_libs_path(self, pack_ref): """ Retrieve path to the pack common lib/ directory taking git work tree path into account (if used). """ worktree_path = self.git_worktree_path pack_common_libs_path = get_pack_common_libs_path_for_pack_ref(pack_ref=pack_ref) if not worktree_path: return pack_common_libs_path # Modify the path so it uses git worktree directory pack_base_path = get_pack_base_path(pack_name=pack_ref) new_pack_common_libs_path = pack_common_libs_path.replace(pack_base_path, '') # Remove leading slash (if any) if new_pack_common_libs_path.startswith('/'): new_pack_common_libs_path = new_pack_common_libs_path[1:] new_pack_common_libs_path = os.path.join(worktree_path, new_pack_common_libs_path) # Check to prevent directory traversal common_prefix = os.path.commonprefix([worktree_path, new_pack_common_libs_path]) if common_prefix != worktree_path: raise ValueError('pack libs path is not located inside the pack directory') return new_pack_common_libs_path
def _register_triggers_from_pack(self, pack, triggers): registered_count = 0 pack_base_path = content_utils.get_pack_base_path( pack_name=pack, include_trailing_slash=True) for trigger in triggers: try: self._register_trigger_from_pack(pack_base_path=pack_base_path, pack=pack, trigger=trigger) except Exception as e: if self._fail_on_failure: msg = ( 'Failed to register trigger "%s" from pack "%s": %s' % (trigger, pack, six.text_type(e))) raise ValueError(msg) LOG.debug('Failed to register trigger "%s": %s', trigger, six.text_type(e)) else: LOG.debug('Trigger "%s" successfully registered', trigger) registered_count += 1 return registered_count
def _get_entry_point_for_worktree_path(self, pack_name, entry_point, worktree_path): """ Method which returns path to an action entry point which is located inside the git worktree directory. :rtype: ``str`` """ pack_base_path = get_pack_base_path(pack_name=pack_name) new_entry_point = entry_point.replace(pack_base_path, '') # Remove leading slash (if any) if new_entry_point.startswith('/'): new_entry_point = new_entry_point[1:] new_entry_point = os.path.join(worktree_path, new_entry_point) # Check to prevent directory traversal common_prefix = os.path.commonprefix([worktree_path, new_entry_point]) if common_prefix != worktree_path: raise ValueError( 'entry_point is not located inside the pack directory') return new_entry_point
def post(self, pack_register_request): if pack_register_request and hasattr(pack_register_request, 'types'): types = pack_register_request.types if 'all' in types: types = PackRegisterController.CONTENT_TYPES else: types = PackRegisterController.CONTENT_TYPES if pack_register_request and hasattr(pack_register_request, 'packs'): packs = list(set(pack_register_request.packs)) else: packs = None result = defaultdict(int) # Register depended resources (actions depend on runners, rules depend on rule types, etc) if ('runner' in types or 'runners' in types) or ('action' in types or 'actions' in types): result['runners'] = runners_registrar.register_runners( experimental=True) if ('rule_type' in types or 'rule_types' in types) or \ ('rule' in types or 'rules' in types): result['rule_types'] = rule_types_registrar.register_rule_types() if ('policy_type' in types or 'policy_types' in types) or \ ('policy' in types or 'policies' in types): result['policy_types'] = policies_registrar.register_policy_types( st2common) use_pack_cache = False fail_on_failure = getattr(pack_register_request, 'fail_on_failure', True) for type, (Registrar, name) in six.iteritems(ENTITIES): if type in types or name in types: registrar = Registrar(use_pack_cache=use_pack_cache, fail_on_failure=fail_on_failure) if packs: for pack in packs: pack_path = content_utils.get_pack_base_path(pack) try: registered_count = registrar.register_from_pack( pack_dir=pack_path) result[name] += registered_count except ValueError as e: # Throw more user-friendly exception if requsted pack doesn't exist if re.match('Directory ".*?" doesn\'t exist', str(e)): msg = 'Pack "%s" not found on disk: %s' % ( pack, str(e)) raise ValueError(msg) raise e else: packs_base_paths = content_utils.get_packs_base_paths() registered_count = registrar.register_from_packs( base_dirs=packs_base_paths) result[name] += registered_count return result
def get_pack_version(pack=None): pack_path = get_pack_base_path(pack) try: pack_metadata = get_pack_metadata(pack_dir=pack_path) result = pack_metadata.get("version", None) except Exception: result = None finally: return result
def get_warnings(pack=None): result = None pack_path = get_pack_base_path(pack) try: pack_metadata = get_pack_metadata(pack_dir=pack_path) result = get_pack_warnings(pack_metadata) except Exception: print("Could not open pack.yaml at location %s" % pack_path) finally: return result
def clone_action_files(source_action_db, dest_action_db, dest_pack_base_path): """ Prepares the path for entry point and metadata files for source and destination. Clones the content from source action files to destination action files. """ source_pack = source_action_db["pack"] source_entry_point = source_action_db["entry_point"] source_metadata_file = source_action_db["metadata_file"] source_pack_base_path = get_pack_base_path(pack_name=source_pack) source_metadata_file_path = os.path.join(source_pack_base_path, source_metadata_file) dest_metadata_file_name = dest_action_db["metadata_file"] dest_metadata_file_path = os.path.join(dest_pack_base_path, dest_metadata_file_name) # creating actions directory in destination pack if doesn't exist ac_dir_path = os.path.join(dest_pack_base_path, "actions") if not os.path.isdir(ac_dir_path): os.mkdir(path=ac_dir_path) _clone_content_to_destination_file( source_file=source_metadata_file_path, destination_file=dest_metadata_file_path) dest_entry_point = dest_action_db["entry_point"] dest_runner_type = dest_action_db["runner_type"]["name"] if dest_entry_point: if dest_runner_type in ["orquesta", "action-chain"]: # creating workflows directory if doesn't exist wf_dir_path = os.path.join(dest_pack_base_path, "actions", "workflows") if not os.path.isdir(wf_dir_path): os.mkdir(path=wf_dir_path) source_entry_point_file_path = os.path.join(source_pack_base_path, "actions", source_entry_point) dest_entrypoint_file_path = os.path.join(dest_pack_base_path, "actions", dest_entry_point) _clone_content_to_destination_file( source_file=source_entry_point_file_path, destination_file=dest_entrypoint_file_path, ) with open(dest_metadata_file_path) as df: doc = yaml.load(df, Loader=yaml.FullLoader) doc["name"] = dest_action_db["name"] if "pack" in doc: doc["pack"] = dest_action_db["pack"] doc["entry_point"] = dest_entry_point with open(dest_metadata_file_path, "w") as df: yaml.dump(doc, df, default_flow_style=False, sort_keys=False)
def get_dependency_list(pack=None): pack_path = get_pack_base_path(pack) try: pack_metadata = get_pack_metadata(pack_dir=pack_path) result = pack_metadata.get("dependencies", None) except Exception: print("Could not open pack.yaml at location %s" % pack_path) result = None finally: return result
def post(self, pack_register_request): if pack_register_request and hasattr(pack_register_request, 'types'): types = pack_register_request.types if 'all' in types: types = PackRegisterController.CONTENT_TYPES else: types = PackRegisterController.CONTENT_TYPES if pack_register_request and hasattr(pack_register_request, 'packs'): packs = list(set(pack_register_request.packs)) else: packs = None result = defaultdict(int) # Register depended resources (actions depend on runners, rules depend on rule types, etc) if ('runner' in types or 'runners' in types) or ('action' in types or 'actions' in types): result['runners'] = runners_registrar.register_runners(experimental=True) if ('rule_type' in types or 'rule_types' in types) or \ ('rule' in types or 'rules' in types): result['rule_types'] = rule_types_registrar.register_rule_types() if ('policy_type' in types or 'policy_types' in types) or \ ('policy' in types or 'policies' in types): result['policy_types'] = policies_registrar.register_policy_types(st2common) use_pack_cache = False fail_on_failure = getattr(pack_register_request, 'fail_on_failure', True) for type, (Registrar, name) in six.iteritems(ENTITIES): if type in types or name in types: registrar = Registrar(use_pack_cache=use_pack_cache, use_runners_cache=True, fail_on_failure=fail_on_failure) if packs: for pack in packs: pack_path = content_utils.get_pack_base_path(pack) try: registered_count = registrar.register_from_pack(pack_dir=pack_path) result[name] += registered_count except ValueError as e: # Throw more user-friendly exception if requsted pack doesn't exist if re.match('Directory ".*?" doesn\'t exist', six.text_type(e)): msg = 'Pack "%s" not found on disk: %s' % (pack, six.text_type(e)) raise ValueError(msg) raise e else: packs_base_paths = content_utils.get_packs_base_paths() registered_count = registrar.register_from_packs(base_dirs=packs_base_paths) result[name] += registered_count return result
def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, inherit_parent_virtualenv=True): """ Return sandbox PYTHONPATH for a particular Python runner action. Same as get_sandbox_python_path() function, but it's intended to be used for Python runner actions. """ sandbox_python_path = get_sandbox_python_path( inherit_from_parent=inherit_from_parent, inherit_parent_virtualenv=inherit_parent_virtualenv) pack_base_path = get_pack_base_path(pack_name=pack) virtualenv_path = get_sandbox_virtualenv_path(pack=pack) if virtualenv_path and os.path.isdir(virtualenv_path): pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') virtualenv_directories = os.listdir(pack_virtualenv_lib_path) virtualenv_directories = [ dir_name for dir_name in virtualenv_directories if fnmatch.fnmatch(dir_name, 'python*') ] # Add the pack's lib directory (lib/python3.x) in front of the PYTHONPATH. pack_actions_lib_paths = os.path.join(pack_base_path, 'actions', 'lib') pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') pack_venv_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0]) # Add the pack's site-packages directory (lib/python3.x/site-packages) # in front of the Python system site-packages This is important because # we want Python 3 compatible libraries to be used from the pack virtual # environment and not system ones. pack_venv_site_packages_directory = os.path.join( pack_virtualenv_lib_path, virtualenv_directories[0], 'site-packages') full_sandbox_python_path = [ # NOTE: Order here is very important for imports to function correctly pack_venv_lib_directory, pack_venv_site_packages_directory, pack_actions_lib_paths, sandbox_python_path, ] sandbox_python_path = ':'.join(full_sandbox_python_path) return sandbox_python_path
def _write_data_file(self, pack_name, file_path, content): """ Write data file on disk. """ # Throw if pack directory doesn't exist pack_base_path = get_pack_base_path(pack_name=pack_name) if not os.path.isdir(pack_base_path): raise ValueError('Directory for pack "%s" doesn\'t exist' % (pack_name)) # Create pack sub-directory tree if it doesn't exist directory = os.path.dirname(file_path) if not os.path.isdir(directory): os.makedirs(directory) with open(file_path, 'w') as fp: fp.write(content)
def test_get_pack_base_paths(self): cfg.CONF.content.system_packs_base_path = '' cfg.CONF.content.packs_base_paths = '/opt/path1' result = get_packs_base_paths() self.assertEqual(result, ['/opt/path1']) # Multiple paths, no trailing colon cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2' result = get_packs_base_paths() self.assertEqual(result, ['/opt/path1', '/opt/path2']) # Multiple paths, trailing colon cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:' result = get_packs_base_paths() self.assertEqual(result, ['/opt/path1', '/opt/path2']) # Multiple same paths cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:/opt/path1:/opt/path2' result = get_packs_base_paths() self.assertEqual(result, ['/opt/path1', '/opt/path2']) # Assert system path is always first cfg.CONF.content.system_packs_base_path = '/opt/system' cfg.CONF.content.packs_base_paths = '/opt/path2:/opt/path1' result = get_packs_base_paths() self.assertEqual(result, ['/opt/system', '/opt/path2', '/opt/path1']) # More scenarios orig_path = cfg.CONF.content.system_packs_base_path cfg.CONF.content.system_packs_base_path = '/tests/packs' names = [ 'test_pack_1', 'test_pack_2', 'ma_pack' ] for name in names: actual = get_pack_base_path(pack_name=name) expected = os.path.join(cfg.CONF.content.system_packs_base_path, name) self.assertEqual(actual, expected) cfg.CONF.content.system_packs_base_path = orig_path
def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, inherit_parent_virtualenv=True): """ Same as get_sandbox_python_path function, but it's intended to be used for Python runner actions and also takes into account if a pack virtual environment uses Python 3. """ sandbox_python_path = get_sandbox_python_path( inherit_from_parent=inherit_from_parent, inherit_parent_virtualenv=inherit_parent_virtualenv) # If python3.? directory exists in pack virtualenv lib/ path it means Python 3 is used by # that virtual environment and we take that in to account when constructing PYTHONPATH pack_base_path = get_pack_base_path(pack_name=pack) virtualenv_path = get_sandbox_virtualenv_path(pack=pack) if virtualenv_path and os.path.isdir(virtualenv_path): pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/') pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') virtualenv_directories = os.listdir(pack_virtualenv_lib_path) virtualenv_directories = [dir_name for dir_name in virtualenv_directories if fnmatch.fnmatch(dir_name, 'python3*')] uses_python3 = bool(virtualenv_directories) else: uses_python3 = False if uses_python3: # Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid # issues with scripts trying to use packages / modules from Python 2.7 site-packages # directory instead of the versions from Python 3 stdlib. python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0]) # Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python # 2.7 system site-packages This is important because we want Python 3 compatible libraries # to be used from the pack virtual environment and not system ones. python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0], 'site-packages') sandbox_python_path = (python3_lib_directory + ':' + python3_site_packages_directory + ':' + pack_actions_lib_paths + ':' + sandbox_python_path) return sandbox_python_path
def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, inherit_parent_virtualenv=True): """ Return sandbox PYTHONPATH for a particular Python runner action. Same as get_sandbox_python_path() function, but it's intended to be used for Python runner actions and also takes into account if a pack virtual environment uses Python 3. """ sandbox_python_path = get_sandbox_python_path( inherit_from_parent=inherit_from_parent, inherit_parent_virtualenv=inherit_parent_virtualenv) pack_base_path = get_pack_base_path(pack_name=pack) virtualenv_path = get_sandbox_virtualenv_path(pack=pack) if not virtualenv_path: return sandbox_python_path uses_python3, virtualenv_directories = is_pack_virtualenv_using_python3( pack=pack) if uses_python3: # Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid # issues with scripts trying to use packages / modules from Python 2.7 site-packages # directory instead of the versions from Python 3 stdlib. pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/') pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0]) # Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python # 2.7 system site-packages This is important because we want Python 3 compatible libraries # to be used from the pack virtual environment and not system ones. python3_site_packages_directory = os.path.join( pack_virtualenv_lib_path, virtualenv_directories[0], 'site-packages') sandbox_python_path = (python3_lib_directory + ':' + python3_site_packages_directory + ':' + pack_actions_lib_paths + ':' + sandbox_python_path) return sandbox_python_path
def post(self, pack_register_request): if pack_register_request and hasattr(pack_register_request, 'types'): types = pack_register_request.types else: types = [ 'runner', 'action', 'trigger', 'sensor', 'rule', 'rule_type', 'alias', 'policy_type', 'policy', 'config' ] if pack_register_request and hasattr(pack_register_request, 'packs'): packs = pack_register_request.packs else: packs = None result = {} if 'runner' in types or 'action' in types: result['runners'] = runners_registrar.register_runners( experimental=True) if 'rule_type' in types or 'rule' in types: result['rule_types'] = rule_types_registrar.register_rule_types() if 'policy_type' in types or 'policy' in types: result['policy_types'] = policies_registrar.register_policy_types( st2common) use_pack_cache = False for type, (Registrar, name) in six.iteritems(ENTITIES): if type in types: registrar = Registrar(use_pack_cache=use_pack_cache, fail_on_failure=False) if packs: for pack in packs: pack_path = content_utils.get_pack_base_path(pack) result[name] = registrar.register_from_pack( pack_dir=pack_path) else: packs_base_paths = content_utils.get_packs_base_paths() result[name] = registrar.register_from_packs( base_dirs=packs_base_paths) return result
def _write_data_file(self, pack_ref, file_path, content): """ Write data file on disk. """ # Throw if pack directory doesn't exist pack_base_path = get_pack_base_path(pack_name=pack_ref) if not os.path.isdir(pack_base_path): raise ValueError('Directory for pack "%s" doesn\'t exist' % (pack_ref)) # Create pack sub-directory tree if it doesn't exist directory = os.path.dirname(file_path) if not os.path.isdir(directory): # NOTE: We apply same permission bits as we do on pack install. If we don't do that, # st2api won't be able to write to pack sub-directory mode = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH os.makedirs(directory, mode) with open(file_path, 'w') as fp: fp.write(content)
def run(self, template_pack, template_path, context, include_execution, include_six): if include_six: context['six'] = six if include_execution: execution = self._get_execution(include_execution) context['__execution'] = execution pack_path = content_utils.get_pack_base_path(template_pack) abs_template_path = os.path.abspath( os.path.join(pack_path, template_path)) if not abs_template_path.startswith(pack_path): raise ValueError( 'Template_path points to a file outside pack directory.') with open(abs_template_path, 'r') as f: template = f.read() return self.jinja.from_string(template).render(context)
def test_get_pack_base_paths(self): cfg.CONF.content.system_packs_base_path = '' cfg.CONF.content.packs_base_paths = '/opt/path1' result = get_packs_base_paths() self.assertEqual(result, ['/opt/path1']) # Multiple paths, no trailing colon cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2' result = get_packs_base_paths() self.assertEqual(result, ['/opt/path1', '/opt/path2']) # Multiple paths, trailing colon cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:' result = get_packs_base_paths() self.assertEqual(result, ['/opt/path1', '/opt/path2']) # Multiple same paths cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:/opt/path1:/opt/path2' result = get_packs_base_paths() self.assertEqual(result, ['/opt/path1', '/opt/path2']) # Assert system path is always first cfg.CONF.content.system_packs_base_path = '/opt/system' cfg.CONF.content.packs_base_paths = '/opt/path2:/opt/path1' result = get_packs_base_paths() self.assertEqual(result, ['/opt/system', '/opt/path2', '/opt/path1']) # More scenarios orig_path = cfg.CONF.content.system_packs_base_path cfg.CONF.content.system_packs_base_path = '/tests/packs' names = ['test_pack_1', 'test_pack_2', 'ma_pack'] for name in names: actual = get_pack_base_path(pack_name=name) expected = os.path.join(cfg.CONF.content.system_packs_base_path, name) self.assertEqual(actual, expected) cfg.CONF.content.system_packs_base_path = orig_path
def _register_triggers_from_pack(self, pack, triggers): registered_count = 0 pack_base_path = content_utils.get_pack_base_path(pack_name=pack, include_trailing_slash=True) for trigger in triggers: try: self._register_trigger_from_pack(pack_base_path=pack_base_path, pack=pack, trigger=trigger) except Exception as e: if self._fail_on_failure: msg = ('Failed to register trigger "%s" from pack "%s": %s' % (trigger, pack, six.text_type(e))) raise ValueError(msg) LOG.debug('Failed to register trigger "%s": %s', trigger, six.text_type(e)) else: LOG.debug('Trigger "%s" successfully registered', trigger) registered_count += 1 return registered_count
def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, inherit_parent_virtualenv=True): """ Same as get_sandbox_python_path function, but it's intended to be used for Python runner actions and also takes into account if a pack virtual environment uses Python 3. """ sandbox_python_path = get_sandbox_python_path( inherit_from_parent=inherit_from_parent, inherit_parent_virtualenv=inherit_parent_virtualenv) # If python3.? directory exists in pack virtualenv lib/ path it means Python 3 is used by # that virtual environment and we take that in to account when constructing PYTHONPATH pack_base_path = get_pack_base_path(pack_name=pack) virtualenv_path = get_sandbox_virtualenv_path(pack=pack) if virtualenv_path and os.path.isdir(virtualenv_path): pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/') pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') virtualenv_directories = os.listdir(pack_virtualenv_lib_path) virtualenv_directories = [dir_name for dir_name in virtualenv_directories if fnmatch.fnmatch(dir_name, 'python3*')] uses_python3 = bool(virtualenv_directories) else: uses_python3 = False if uses_python3: # Add Python 3 lib/site-packages directory infront of the system site packages # This is important because we want Python 3 compatible libraries to be used from # the pack virtual environment and not system ones python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0], 'site-packages') sandbox_python_path = (python3_site_packages_directory + ':' + pack_actions_lib_paths + ':' + sandbox_python_path) return sandbox_python_path
def _get_entry_point_for_worktree_path(self, pack_name, entry_point, worktree_path): """ Method which returns path to an action entry point which is located inside the git worktree directory. :rtype: ``str`` """ pack_base_path = get_pack_base_path(pack_name=pack_name) new_entry_point = entry_point.replace(pack_base_path, '') # Remove leading slash (if any) if new_entry_point.startswith('/'): new_entry_point = new_entry_point[1:] new_entry_point = os.path.join(worktree_path, new_entry_point) # Check to prevent directory traversal common_prefix = os.path.commonprefix([worktree_path, new_entry_point]) if common_prefix != worktree_path: raise ValueError('entry_point is not located inside the pack directory') return new_entry_point
def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, inherit_parent_virtualenv=True): """ Return sandbox PYTHONPATH for a particular Python runner action. Same as get_sandbox_python_path() function, but it's intended to be used for Python runner actions and also takes into account if a pack virtual environment uses Python 3. """ sandbox_python_path = get_sandbox_python_path( inherit_from_parent=inherit_from_parent, inherit_parent_virtualenv=inherit_parent_virtualenv) pack_base_path = get_pack_base_path(pack_name=pack) virtualenv_path = get_sandbox_virtualenv_path(pack=pack) if not virtualenv_path: return sandbox_python_path uses_python3, virtualenv_directories = is_pack_virtualenv_using_python3(pack=pack) if uses_python3: # Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid # issues with scripts trying to use packages / modules from Python 2.7 site-packages # directory instead of the versions from Python 3 stdlib. pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/') pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0]) # Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python # 2.7 system site-packages This is important because we want Python 3 compatible libraries # to be used from the pack virtual environment and not system ones. python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0], 'site-packages') sandbox_python_path = (python3_lib_directory + ':' + python3_site_packages_directory + ':' + pack_actions_lib_paths + ':' + sandbox_python_path) return sandbox_python_path
def post(self, pack_register_request): if pack_register_request and hasattr(pack_register_request, 'types'): types = pack_register_request.types else: types = ['runner', 'action', 'trigger', 'sensor', 'rule', 'rule_type', 'alias', 'policy_type', 'policy', 'config'] if pack_register_request and hasattr(pack_register_request, 'packs'): packs = pack_register_request.packs else: packs = None result = {} if 'runner' in types or 'action' in types: result['runners'] = runners_registrar.register_runners(experimental=True) if 'rule_type' in types or 'rule' in types: result['rule_types'] = rule_types_registrar.register_rule_types() if 'policy_type' in types or 'policy' in types: result['policy_types'] = policies_registrar.register_policy_types(st2common) use_pack_cache = False for type, (Registrar, name) in six.iteritems(ENTITIES): if type in types: registrar = Registrar(use_pack_cache=use_pack_cache, fail_on_failure=False) if packs: for pack in packs: pack_path = content_utils.get_pack_base_path(pack) result[name] = registrar.register_from_pack(pack_dir=pack_path) else: packs_base_paths = content_utils.get_packs_base_paths() result[name] = registrar.register_from_packs(base_dirs=packs_base_paths) return result
def __init__(self, pack_name): self.pack_name = pack_name self.pack_path = utils.get_pack_base_path(pack_name=pack_name)
def delete_action_files_from_pack(pack_name, entry_point, metadata_file): """ Prepares the path for entry_point file and metadata file of action and deletes them from disk. """ pack_base_path = get_pack_base_path(pack_name=pack_name) action_entrypoint_file_path = os.path.join(pack_base_path, "actions", entry_point) action_metadata_file_path = os.path.join(pack_base_path, metadata_file) if os.path.isfile(action_entrypoint_file_path): try: os.remove(action_entrypoint_file_path) except PermissionError: LOG.error( 'No permission to delete the "%s" file', action_entrypoint_file_path, ) msg = 'No permission to delete "%s" file from disk' % ( action_entrypoint_file_path ) raise PermissionError(msg) except Exception as e: LOG.error( 'Unable to delete "%s" file. Exception was "%s"', action_entrypoint_file_path, e, ) msg = ( 'The action file "%s" could not be removed from disk, please ' "check the logs or ask your StackStorm administrator to check " "and delete the actions files manually" % (action_entrypoint_file_path) ) raise ResourceDiskFilesRemovalError(msg) else: LOG.warning( 'The action entry point file "%s" does not exists on disk.', action_entrypoint_file_path, ) if os.path.isfile(action_metadata_file_path): try: os.remove(action_metadata_file_path) except PermissionError: LOG.error( 'No permission to delete the "%s" file', action_metadata_file_path, ) msg = 'No permission to delete "%s" file from disk' % ( action_metadata_file_path ) raise PermissionError(msg) except Exception as e: LOG.error( 'Could not delete "%s" file. Exception was "%s"', action_metadata_file_path, e, ) msg = ( 'The action file "%s" could not be removed from disk, please ' "check the logs or ask your StackStorm administrator to check " "and delete the actions files manually" % (action_metadata_file_path) ) raise ResourceDiskFilesRemovalError(msg) else: LOG.warning( 'The action metadata file "%s" does not exists on disk.', action_metadata_file_path, )
def __init__(self, pack_name, user=None): self.pack_name = pack_name self.user = user or cfg.CONF.system_user.user self.pack_path = content_utils.get_pack_base_path(pack_name=pack_name) self._config_parser = ContentPackConfigParser(pack_name=pack_name)
def post(self, pack_register_request): if pack_register_request and hasattr(pack_register_request, "types"): types = pack_register_request.types if "all" in types: types = PackRegisterController.CONTENT_TYPES else: types = PackRegisterController.CONTENT_TYPES if pack_register_request and hasattr(pack_register_request, "packs"): packs = list(set(pack_register_request.packs)) else: packs = None result = defaultdict(int) # Register depended resources (actions depend on runners, rules depend on rule types, etc) if ("runner" in types or "runners" in types) or ("action" in types or "actions" in types): result["runners"] = runners_registrar.register_runners( experimental=True) if ("rule_type" in types or "rule_types" in types) or ("rule" in types or "rules" in types): result["rule_types"] = rule_types_registrar.register_rule_types() if ("policy_type" in types or "policy_types" in types) or ("policy" in types or "policies" in types): result["policy_types"] = policies_registrar.register_policy_types( st2common) use_pack_cache = False # TODO: To speed up this operation since it's mostli IO bound we could use green thread # pool here and register different resources concurrently fail_on_failure = getattr(pack_register_request, "fail_on_failure", True) for type, (Registrar, name) in six.iteritems(ENTITIES): if type in types or name in types: registrar = Registrar( use_pack_cache=use_pack_cache, use_runners_cache=True, fail_on_failure=fail_on_failure, ) if packs: for pack in packs: pack_path = content_utils.get_pack_base_path(pack) try: registered_count = registrar.register_from_pack( pack_dir=pack_path) result[name] += registered_count except ValueError as e: # Throw more user-friendly exception if requsted pack doesn't exist if re.match('Directory ".*?" doesn\'t exist', six.text_type(e)): msg = 'Pack "%s" not found on disk: %s' % ( pack, six.text_type(e), ) raise ValueError(msg) raise e else: packs_base_paths = content_utils.get_packs_base_paths() registered_count = registrar.register_from_packs( base_dirs=packs_base_paths) result[name] += registered_count return result
def clone(self, dest_data, ref_or_id, requester_user): """ Clone an action from source pack to destination pack. Handles requests: POST /actions/{ref_or_id}/clone """ source_action_db = self._get_by_ref_or_id(ref_or_id=ref_or_id) if not source_action_db: msg = "The requested source for cloning operation doesn't exists" abort(http_client.BAD_REQUEST, six.text_type(msg)) extra = {"action_db": source_action_db} LOG.audit("Source action found. Action.id=%s" % (source_action_db.id), extra=extra) try: permission_type = PermissionType.ACTION_VIEW rbac_utils = get_rbac_backend().get_utils_class() rbac_utils.assert_user_has_resource_db_permission( user_db=requester_user, resource_db=source_action_db, permission_type=permission_type, ) except ResourceAccessDeniedError as e: abort(http_client.UNAUTHORIZED, six.text_type(e)) cloned_dest_action_db = clone_action_db( source_action_db=source_action_db, dest_pack=dest_data.dest_pack, dest_action=dest_data.dest_action, ) cloned_action_api = ActionAPI.from_model(cloned_dest_action_db) try: permission_type = PermissionType.ACTION_CREATE rbac_utils.assert_user_has_resource_api_permission( user_db=requester_user, resource_api=cloned_action_api, permission_type=permission_type, ) except ResourceAccessDeniedError as e: abort(http_client.UNAUTHORIZED, six.text_type(e)) dest_pack_base_path = get_pack_base_path(pack_name=dest_data.dest_pack) if not os.path.isdir(dest_pack_base_path): msg = "Destination pack '%s' doesn't exist" % (dest_data.dest_pack) abort(http_client.BAD_REQUEST, six.text_type(msg)) dest_pack_base_path = get_pack_base_path(pack_name=dest_data.dest_pack) dest_ref = ".".join([dest_data.dest_pack, dest_data.dest_action]) dest_action_db = self._get_by_ref(resource_ref=dest_ref) try: validate_not_part_of_system_pack_by_name(dest_data.dest_pack) except ValueValidationException as e: abort(http_client.BAD_REQUEST, six.text_type(e)) if dest_action_db: if not dest_data.overwrite: msg = "The requested destination action already exists" abort(http_client.BAD_REQUEST, six.text_type(msg)) try: permission_type = PermissionType.ACTION_DELETE rbac_utils.assert_user_has_resource_db_permission( user_db=requester_user, resource_db=dest_action_db, permission_type=permission_type, ) options = GenericRequestParam(remove_files=True) dest_metadata_file = dest_action_db["metadata_file"] dest_entry_point = dest_action_db["entry_point"] temp_sub_dir = str(uuid.uuid4()) temp_backup_action_files( dest_pack_base_path, dest_metadata_file, dest_entry_point, temp_sub_dir, ) self.delete(options, dest_ref, requester_user) except ResourceAccessDeniedError as e: abort(http_client.UNAUTHORIZED, six.text_type(e)) except Exception as e: LOG.debug( "Exception encountered during deleting existing destination action. " "Exception was: %s", e, ) abort(http_client.INTERNAL_SERVER_ERROR, six.text_type(e)) try: post_response = self.post(cloned_action_api, requester_user) if post_response.status_code != http_client.CREATED: raise Exception("Could not add cloned action to database.") cloned_dest_action_db["id"] = post_response.json["id"] clone_action_files( source_action_db=source_action_db, dest_action_db=cloned_dest_action_db, dest_pack_base_path=dest_pack_base_path, ) extra = {"cloned_acion_db": cloned_dest_action_db} LOG.audit("Action cloned. Action.id=%s" % (cloned_dest_action_db.id), extra=extra) if dest_action_db: remove_temp_action_files(temp_sub_dir) return post_response except PermissionError as e: LOG.error("No permission to clone the action. Exception was %s", e) delete_action_files_from_pack( pack_name=cloned_dest_action_db["pack"], entry_point=cloned_dest_action_db["entry_point"], metadata_file=cloned_dest_action_db["metadata_file"], ) if post_response.status_code == http_client.CREATED: Action.delete(cloned_dest_action_db) if dest_action_db: self._restore_action(dest_action_db, dest_pack_base_path, temp_sub_dir) abort(http_client.FORBIDDEN, six.text_type(e)) except Exception as e: LOG.error( "Exception encountered during cloning action. Exception was %s", e, ) delete_action_files_from_pack( pack_name=cloned_dest_action_db["pack"], entry_point=cloned_dest_action_db["entry_point"], metadata_file=cloned_dest_action_db["metadata_file"], ) if post_response.status_code == http_client.CREATED: Action.delete(cloned_dest_action_db) if dest_action_db: self._restore_action(dest_action_db, dest_pack_base_path, temp_sub_dir) abort(http_client.INTERNAL_SERVER_ERROR, six.text_type(e))
def get_pack_base_path(pack_name): return utils.get_pack_base_path(pack_name)
def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, inherit_parent_virtualenv=True): """ Return sandbox PYTHONPATH for a particular Python runner action. Same as get_sandbox_python_path() function, but it's intended to be used for Python runner actions and also takes into account if a pack virtual environment uses Python 3. """ sandbox_python_path = get_sandbox_python_path( inherit_from_parent=inherit_from_parent, inherit_parent_virtualenv=inherit_parent_virtualenv) pack_base_path = get_pack_base_path(pack_name=pack) virtualenv_path = get_sandbox_virtualenv_path(pack=pack) if not virtualenv_path: return sandbox_python_path uses_python3, virtualenv_directories = is_pack_virtualenv_using_python3( pack=pack) if uses_python3: # Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid # issues with scripts trying to use packages / modules from Python 2.7 site-packages # directory instead of the versions from Python 3 stdlib. pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/') pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0]) # Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python # 2.7 system site-packages This is important because we want Python 3 compatible libraries # to be used from the pack virtual environment and not system ones. python3_site_packages_directory = os.path.join( pack_virtualenv_lib_path, virtualenv_directories[0], 'site-packages') # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv # one (e.g. /usr/lib/python3.6) # NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2 system_prefix_dirs = [] # Take custom prefix into account (if specified) if cfg.CONF.actionrunner.python3_prefix: system_prefix_dirs.append(cfg.CONF.actionrunner.python3_prefix) # By default, Python libs are installed either in /usr/lib/python3.x or # /usr/local/lib/python3.x system_prefix_dirs.extend(['/usr/lib', '/usr/local/lib']) for system_prefix_dir in system_prefix_dirs: python3_system_lib_directory = os.path.join( system_prefix_dir, virtualenv_directories[0]) if os.path.exists(python3_system_lib_directory): break if not python3_system_lib_directory or not os.path.exists( python3_system_lib_directory): python3_system_lib_directory = None full_sandbox_python_path = [] # NOTE: Order here is very important for imports to function correctly if python3_system_lib_directory: full_sandbox_python_path.append(python3_system_lib_directory) full_sandbox_python_path.append(python3_lib_directory) full_sandbox_python_path.append(python3_site_packages_directory) full_sandbox_python_path.append(pack_actions_lib_paths) full_sandbox_python_path.append(sandbox_python_path) sandbox_python_path = ':'.join(full_sandbox_python_path) return sandbox_python_path
def get_sandbox_python_path_for_python_action(pack, inherit_from_parent=True, inherit_parent_virtualenv=True): """ Return sandbox PYTHONPATH for a particular Python runner action. Same as get_sandbox_python_path() function, but it's intended to be used for Python runner actions and also takes into account if a pack virtual environment uses Python 3. """ sandbox_python_path = get_sandbox_python_path( inherit_from_parent=inherit_from_parent, inherit_parent_virtualenv=inherit_parent_virtualenv) pack_base_path = get_pack_base_path(pack_name=pack) virtualenv_path = get_sandbox_virtualenv_path(pack=pack) if not virtualenv_path: return sandbox_python_path uses_python3, virtualenv_directories = is_pack_virtualenv_using_python3(pack=pack) if uses_python3: # Add Python 3 lib directory (lib/python3.x) in front of the PYTHONPATH. This way we avoid # issues with scripts trying to use packages / modules from Python 2.7 site-packages # directory instead of the versions from Python 3 stdlib. pack_actions_lib_paths = os.path.join(pack_base_path, 'actions/lib/') pack_virtualenv_lib_path = os.path.join(virtualenv_path, 'lib') python3_lib_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0]) # Add Python 3 site-packages directory (lib/python3.x/site-packages) in front of the Python # 2.7 system site-packages This is important because we want Python 3 compatible libraries # to be used from the pack virtual environment and not system ones. python3_site_packages_directory = os.path.join(pack_virtualenv_lib_path, virtualenv_directories[0], 'site-packages') # Work around to make sure we also add system lib dir to PYTHONPATH and not just virtualenv # one (e.g. /usr/lib/python3.6) # NOTE: We can't simply use sys.prefix dir since it will be set to /opt/stackstorm/st2 system_prefix_dirs = [] # Take custom prefix into account (if specified) if cfg.CONF.actionrunner.python3_prefix: system_prefix_dirs.append(cfg.CONF.actionrunner.python3_prefix) # By default, Python libs are installed either in /usr/lib/python3.x or # /usr/local/lib/python3.x system_prefix_dirs.extend(['/usr/lib', '/usr/local/lib']) for system_prefix_dir in system_prefix_dirs: python3_system_lib_directory = os.path.join(system_prefix_dir, virtualenv_directories[0]) if os.path.exists(python3_system_lib_directory): break if not python3_system_lib_directory or not os.path.exists(python3_system_lib_directory): python3_system_lib_directory = None full_sandbox_python_path = [] # NOTE: Order here is very important for imports to function correctly if python3_system_lib_directory: full_sandbox_python_path.append(python3_system_lib_directory) full_sandbox_python_path.append(python3_lib_directory) full_sandbox_python_path.append(python3_site_packages_directory) full_sandbox_python_path.append(pack_actions_lib_paths) full_sandbox_python_path.append(sandbox_python_path) sandbox_python_path = ':'.join(full_sandbox_python_path) return sandbox_python_path