def _register_pack_db(self, pack_name, pack_dir): manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) content['ref'] = pack_name # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(pack_name).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def _register_pack(self, pack_name, pack_dir): """ Register a pack (create a DB object in the system). Note: Pack registration now happens when registering the content and not when installing a pack using packs.install. Eventually this will be moved to the pack management API. """ manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) content['ref'] = pack_name pack_api = PackAPI(**content) pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(pack_name).id except ValueError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def _register_pack_db(self, pack_name, pack_dir): content = get_pack_metadata(pack_dir=pack_dir) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata(metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def _delete_pack_db_object(self, pack): pack_db = None # 1. Try by ref try: pack_db = Pack.get_by_ref(value=pack) except StackStormDBObjectNotFoundError: pack_db = None # 2. Try by name (here for backward compatibility) # TODO: This shouldn't be needed in the future, remove it in v2.1 or similar if not pack_db: try: pack_db = Pack.get_by_name(value=pack) except StackStormDBObjectNotFoundError: pack_db = None if not pack_db: self.logger.exception('Pack DB object not found') return try: Pack.delete(pack_db) except: self.logger.exception('Failed to remove DB object %s.', pack_db)
def _register_pack(self, pack_name, pack_dir): """ Register a pack (create a DB object in the system). Note: Pack registration now happens when registering the content and not when installing a pack using packs.install. Eventually this will be moved to the pack management API. """ manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) content['ref'] = pack_name # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(pack_name).id except ValueError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def test_get_pack_files_binary_files_are_excluded(self): binary_files = [ "icon.png", "etc/permissions.png", "etc/travisci.png", "etc/generate_new_token.png", ] pack_db = Pack.get_by_ref("dummy_pack_1") all_files_count = len(pack_db.files) non_binary_files_count = all_files_count - len(binary_files) resp = self.app.get("/v1/packs/views/files/dummy_pack_1") self.assertEqual(resp.status_int, http_client.OK) self.assertEqual(len(resp.json), non_binary_files_count) for file_path in binary_files: self.assertIn(file_path, pack_db.files) # But not in files controller response for file_path in binary_files: item = [ item for item in resp.json if item["file_path"] == file_path ] self.assertFalse(item)
def _register_pack_db(self, pack_name, pack_dir): content = get_pack_metadata(pack_dir=pack_dir) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata( metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def test_post_include_files(self): # Verify initial state pack_db = Pack.get_by_ref(ACTION_12["pack"]) self.assertNotIn("actions/filea.txt", pack_db.files) action = copy.deepcopy(ACTION_12) action["data_files"] = [{"file_path": "filea.txt", "content": "test content"}] post_resp = self.__do_post(action) # Verify file has been written on disk for file_path in self.to_delete_files: self.assertTrue(os.path.exists(file_path)) # Verify PackDB.files has been updated pack_db = Pack.get_by_ref(ACTION_12["pack"]) self.assertIn("actions/filea.txt", pack_db.files) self.__do_delete(self.__get_action_id(post_resp))
def test_post_include_files(self): # Verify initial state pack_db = Pack.get_by_ref(ACTION_12["pack"]) self.assertTrue("actions/filea.txt" not in pack_db.files) action = copy.deepcopy(ACTION_12) action["data_files"] = [{"file_path": "filea.txt", "content": "test content"}] post_resp = self.__do_post(action) # Verify file has been written on disk for file_path in self.to_delete_files: self.assertTrue(os.path.exists(file_path)) # Verify PackDB.files has been updated pack_db = Pack.get_by_ref(ACTION_12["pack"]) self.assertTrue("actions/filea.txt" in pack_db.files) self.__do_delete(self.__get_action_id(post_resp))
def _register_pack_db(self, pack_name, pack_dir): pack_name = pack_name or '' manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that if content.get('ref', None): content['ref'] = content['ref'] elif re.match(PACK_REF_WHITELIST_REGEX, pack_name): content['ref'] = pack_name else: if re.match(PACK_REF_WHITELIST_REGEX, content['name']): content['ref'] = content['name'] else: raise ValueError( 'Pack name "%s" contains invalid characters and "ref" ' 'attribute is not available' % (content['name'])) # Note: We use a ref if available, if not we fall back to pack name # (pack directory name) content['ref'] = content.get('ref', pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list # Note: If some version values are not explicitly surrounded by quotes they are recognized # as numbers so we cast them to string if 'version' in content: content['version'] = str(content['version']) pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def test_post_include_files(self): # Verify initial state pack_db = Pack.get_by_ref(ACTION_12['pack']) self.assertTrue('actions/filea.txt' not in pack_db.files) action = copy.deepcopy(ACTION_12) action['data_files'] = [{ 'file_path': 'filea.txt', 'content': 'test content' }] post_resp = self.__do_post(action) # Verify file has been written on disk for file_path in self.to_delete_files: self.assertTrue(os.path.exists(file_path)) # Verify PackDB.files has been updated pack_db = Pack.get_by_ref(ACTION_12['pack']) self.assertTrue('actions/filea.txt' in pack_db.files) self.__do_delete(self.__get_action_id(post_resp))
def test_post_include_files(self): # Verify initial state pack_db = Pack.get_by_ref(ACTION_12['pack']) self.assertTrue('actions/filea.txt' not in pack_db.files) action = copy.deepcopy(ACTION_12) action['data_files'] = [ { 'file_path': 'filea.txt', 'content': 'test content' } ] post_resp = self.__do_post(action) # Verify file has been written on disk for file_path in self.to_delete_files: self.assertTrue(os.path.exists(file_path)) # Verify PackDB.files has been updated pack_db = Pack.get_by_ref(ACTION_12['pack']) self.assertTrue('actions/filea.txt' in pack_db.files) self.__do_delete(self.__get_action_id(post_resp))
def _update_pack_model(self, pack_name, data_files, written_file_paths): """ Update PackDB models (update files list). """ file_paths = [] # A list of paths relative to the pack directory for new files for file_path in written_file_paths: file_path = get_relative_path_to_pack(pack_name=pack_name, file_path=file_path) file_paths.append(file_path) pack_db = Pack.get_by_ref(pack_name) pack_db.files = set(pack_db.files) pack_db.files.update(set(file_paths)) pack_db.files = list(pack_db.files) pack_db = Pack.add_or_update(pack_db) return pack_db
def _update_pack_model(self, pack_ref, data_files, written_file_paths): """ Update PackDB models (update files list). """ file_paths = [] # A list of paths relative to the pack directory for new files for file_path in written_file_paths: file_path = get_relative_path_to_pack_file(pack_ref=pack_ref, file_path=file_path) file_paths.append(file_path) pack_db = Pack.get_by_ref(pack_ref) pack_db.files = set(pack_db.files) pack_db.files.update(set(file_paths)) pack_db.files = list(pack_db.files) pack_db = Pack.add_or_update(pack_db) return pack_db
def _register_pack_db(self, pack_name, pack_dir): pack_name = pack_name or '' manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata( metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def test_get_pack_files_binary_files_are_excluded(self): binary_files = [ 'icon.png', 'etc/permissions.png', 'etc/travisci.png', 'etc/generate_new_token.png' ] pack_db = Pack.get_by_ref('dummy_pack_1') all_files_count = len(pack_db.files) non_binary_files_count = all_files_count - len(binary_files) resp = self.app.get('/v1/packs/views/files/dummy_pack_1') self.assertEqual(resp.status_int, httplib.OK) self.assertEqual(len(resp.json), non_binary_files_count) for file_path in binary_files: self.assertTrue(file_path in pack_db.files) # But not in files controller response for file_path in binary_files: item = [item for item in resp.json if item['file_path'] == file_path] self.assertFalse(item)
def test_get_pack_files_binary_files_are_excluded(self): binary_files = [ 'icon.png', 'etc/permissions.png', 'etc/travisci.png', 'etc/generate_new_token.png' ] pack_db = Pack.get_by_ref('dummy_pack_1') all_files_count = len(pack_db.files) non_binary_files_count = all_files_count - len(binary_files) resp = self.app.get('/v1/packs/views/files/dummy_pack_1') self.assertEqual(resp.status_int, httplib.OK) self.assertEqual(len(resp.json), non_binary_files_count) for file_path in binary_files: self.assertTrue(file_path in pack_db.files) # But not in files controller response for file_path in binary_files: item = [ item for item in resp.json if item['file_path'] == file_path ] self.assertFalse(item)
def _register_pack_db(self, pack_name, pack_dir): pack_name = pack_name or '' manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME) if not os.path.isfile(manifest_path): raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME)) content = self._meta_loader.load(manifest_path) if not content: raise ValueError('Pack "%s" metadata file is empty' % (pack_name)) # The rules for the pack ref are as follows: # 1. If ref attribute is available, we used that # 2. If pack_name is available we use that (this only applies to packs # 2hich are in sub-directories) # 2. If attribute is not available, but pack name is and pack name meets the valid name # criteria, we use that content['ref'] = get_pack_ref_from_metadata(metadata=content, pack_directory_name=pack_name) # Include a list of pack files pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS) content['files'] = pack_file_list pack_api = PackAPI(**content) pack_api.validate() pack_db = PackAPI.to_model(pack_api) try: pack_db.id = Pack.get_by_ref(content['ref']).id except StackStormDBObjectNotFoundError: LOG.debug('Pack %s not found. Creating new one.', pack_name) pack_db = Pack.add_or_update(pack_db) LOG.debug('Pack %s registered.' % (pack_name)) return pack_db
def run(self, action_parameters): LOG.debug('Running pythonrunner.') LOG.debug('Getting pack name.') pack = self.get_pack_ref() pack_db = Pack.get_by_ref(pack) LOG.debug('Getting user.') user = self.get_user() LOG.debug('Serializing parameters.') serialized_parameters = json.dumps( action_parameters) if action_parameters else '' LOG.debug('Getting virtualenv_path.') virtualenv_path = get_sandbox_virtualenv_path(pack=pack) LOG.debug('Getting python path.') if self._sandbox: python_path = get_sandbox_python_binary_path(pack=pack) else: python_path = sys.executable LOG.debug('Checking virtualenv path.') if virtualenv_path and not os.path.isdir(virtualenv_path): format_values = {'pack': pack, 'virtualenv_path': virtualenv_path} msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values LOG.error('virtualenv_path set but not a directory: %s', msg) raise Exception(msg) LOG.debug('Checking entry_point.') if not self.entry_point: LOG.error('Action "%s" is missing entry_point attribute' % (self.action.name)) raise Exception('Action "%s" is missing entry_point attribute' % (self.action.name)) # Note: We pass config as command line args so the actual wrapper process is standalone # and doesn't need access to db LOG.debug('Setting args.') args = [ python_path, '-u', # unbuffered mode so streaming mode works as expected WRAPPER_SCRIPT_PATH, '--pack=%s' % (pack), '--file-path=%s' % (self.entry_point), '--parameters=%s' % (serialized_parameters), '--user=%s' % (user), '--parent-args=%s' % (json.dumps(sys.argv[1:])), ] if self._config: args.append('--config=%s' % (json.dumps(self._config))) if self._log_level != 'debug': # We only pass --log-level parameter if non default log level value is specified args.append('--log-level=%s' % (self._log_level)) # We need to ensure all the st2 dependencies are also available to the # subprocess LOG.debug('Setting env.') env = os.environ.copy() env['PATH'] = get_sandbox_path(virtualenv_path=virtualenv_path) sandbox_python_path = get_sandbox_python_path( inherit_from_parent=True, inherit_parent_virtualenv=True) pack_common_libs_path = get_pack_common_libs_path(pack_db=pack_db) if self._enable_common_pack_libs and pack_common_libs_path: env['PYTHONPATH'] = pack_common_libs_path + ':' + sandbox_python_path else: env['PYTHONPATH'] = sandbox_python_path # Include user provided environment variables (if any) user_env_vars = self._get_env_vars() env.update(user_env_vars) # Include common st2 environment variables st2_env_vars = self._get_common_action_env_variables() env.update(st2_env_vars) datastore_env_vars = self._get_datastore_access_env_vars() env.update(datastore_env_vars) stdout = StringIO() stderr = StringIO() store_execution_stdout_line = functools.partial( store_execution_output_data, output_type='stdout') store_execution_stderr_line = functools.partial( store_execution_output_data, output_type='stderr') read_and_store_stdout = make_read_and_store_stream_func( execution_db=self.execution, action_db=self.action, store_data_func=store_execution_stdout_line) read_and_store_stderr = make_read_and_store_stream_func( execution_db=self.execution, action_db=self.action, store_data_func=store_execution_stderr_line) command_string = list2cmdline(args) LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'], command_string)) exit_code, stdout, stderr, timed_out = run_command( cmd=args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, env=env, timeout=self._timeout, read_stdout_func=read_and_store_stdout, read_stderr_func=read_and_store_stderr, read_stdout_buffer=stdout, read_stderr_buffer=stderr) LOG.debug('Returning values: %s, %s, %s, %s' % (exit_code, stdout, stderr, timed_out)) LOG.debug('Returning.') return self._get_output_values(exit_code, stdout, stderr, timed_out)
def get_pack_common_libs_path_for_pack_ref(pack_ref): pack_db = Pack.get_by_ref(pack_ref) pack_common_libs_path = get_pack_common_libs_path_for_pack_db(pack_db=pack_db) return pack_common_libs_path
def _spawn_sensor_process(self, sensor): """ Spawn a new process for the provided sensor. New process uses isolated Python binary from a virtual environment belonging to the sensor pack. """ sensor_id = self._get_sensor_id(sensor=sensor) pack_ref = sensor['pack'] pack_db = Pack.get_by_ref(pack_ref) virtualenv_path = get_sandbox_virtualenv_path(pack=pack_ref) python_path = get_sandbox_python_binary_path(pack=pack_ref) if virtualenv_path and not os.path.isdir(virtualenv_path): format_values = { 'pack': sensor['pack'], 'virtualenv_path': virtualenv_path } msg = PACK_VIRTUALENV_DOESNT_EXIST % format_values raise Exception(msg) trigger_type_refs = sensor['trigger_types'] or [] trigger_type_refs = ','.join(trigger_type_refs) parent_args = json.dumps(sys.argv[1:]) args = [ python_path, WRAPPER_SCRIPT_PATH, '--pack=%s' % (sensor['pack']), '--file-path=%s' % (sensor['file_path']), '--class-name=%s' % (sensor['class_name']), '--trigger-type-refs=%s' % (trigger_type_refs), '--parent-args=%s' % (parent_args) ] if sensor['poll_interval']: args.append('--poll-interval=%s' % (sensor['poll_interval'])) sandbox_python_path = get_sandbox_python_path( inherit_from_parent=True, inherit_parent_virtualenv=True) pack_common_libs_path = get_pack_common_libs_path(pack_db=pack_db) env = os.environ.copy() if self._enable_common_pack_libs and pack_common_libs_path: env['PYTHONPATH'] = pack_common_libs_path + ':' + sandbox_python_path else: env['PYTHONPATH'] = sandbox_python_path # Include full api URL and API token specific to that sensor ttl = cfg.CONF.auth.service_token_ttl metadata = { 'service': 'sensors_container', 'sensor_path': sensor['file_path'], 'sensor_class': sensor['class_name'] } temporary_token = create_token(username='******', ttl=ttl, metadata=metadata, service=True) env[API_URL_ENV_VARIABLE_NAME] = get_full_public_api_url() env[AUTH_TOKEN_ENV_VARIABLE_NAME] = temporary_token.token # TODO 1: Purge temporary token when service stops or sensor process dies # TODO 2: Store metadata (wrapper process id) with the token and delete # tokens for old, dead processes on startup cmd = ' '.join(args) LOG.debug('Running sensor subprocess (cmd="%s")', cmd) # TODO: Intercept stdout and stderr for aggregated logging purposes try: process = subprocess.Popen(args=args, stdin=None, stdout=None, stderr=None, shell=False, env=env, preexec_fn=on_parent_exit('SIGTERM')) except Exception as e: cmd = ' '.join(args) message = ('Failed to spawn process for sensor %s ("%s"): %s' % (sensor_id, cmd, str(e))) raise Exception(message) self._processes[sensor_id] = process self._sensors[sensor_id] = sensor self._sensor_start_times[sensor_id] = int(time.time()) self._dispatch_trigger_for_sensor_spawn(sensor=sensor, process=process, cmd=cmd) return process
def get_pack_by_ref(pack_ref): """ Retrieve PackDB by the provided reference. """ pack_db = Pack.get_by_ref(pack_ref) return pack_db
def get_pack_common_libs_path_for_pack_ref(pack_ref): pack_db = Pack.get_by_ref(pack_ref) pack_common_libs_path = get_pack_common_libs_path_for_pack_db( pack_db=pack_db) return pack_common_libs_path