def delete_app_meta_data(self, key=None, update_last_modify=True): """ Delete values from default namespace & app :key: delete this key, or all the keys when it's None :return: None, or throws Exceptions """ if key is None: self.project_meta_state.delete_state(self.namespace) _logger.debug('delete app meta data. app:%s, namespace:%s', self.app_name, self.namespace) else: app_meta = self.project_meta_state.get_state(self.namespace) if app_meta and key in app_meta: app_meta = copy.deepcopy(app_meta) del app_meta[key] self.project_meta_state.update_state(self.namespace, app_meta) if update_last_modify: self._update_last_modify_time() _logger.debug( 'Delete meta success. Key:%s, meta:%s, app:%s, namespace:%s', key, logger.hide_sensitive_field(app_meta), self.app_name, self.namespace) else: _logger.error( 'Delete meta fails. Key %s not found in meta %s. app:%s, namespace:%s', key, logger.hide_sensitive_field(app_meta), self.app_name, self.namespace)
def generate_input_module_content(self, datainput): input_var_names = [ opt['name'] for opt in datainput.get('data_inputs_options', []) ] if not input_var_names: input_var_names = ['modinput_var_name'] global_vars = [] if 'global_settings' in datainput: global_var_metas = datainput.get('global_settings', {}).get( 'customized_settings', []) global_vars = [m['name'] for m in global_var_metas] if not global_vars: self._update_global_var_list() global_vars = self.__global_vars temp_file = self._get_module_py_template(datainput) temp = Template(filename=temp_file) self.__logger.debug( "Generate the input module: modinput_vars:%s, global_vars:%s, input_meta:%s", input_var_names, self.__global_vars, logger.hide_sensitive_field(datainput)) content = temp.render( appname=self.__appname, app_namespace=self.__app_namespace, datainput=datainput, global_vars=global_vars, modinput_vars=input_var_names) return content
def validate_global_setting_meta(self, meta): try: if self.CUSTOMIZED_SETTING_KEY in meta: cus_setting = meta[self.CUSTOMIZED_SETTING_KEY] for setting in cus_setting: for k in self.CUSTOMIZED_VAR_REQUIRED_KEYS: if k not in setting: emsg = 'Required field {} not found in customized variable {}.'.format( k, logger.hide_sensitive_field(setting)) self._logger.error(emsg) raise CommonException(err_code=11001, options={ 'field': k, 'v_name': setting.get( 'name', 'unknown') }, e_message=emsg) if setting['name'] in self.RESERVED_CUSTOMIZED_VAR_NAMES: emsg = 'Global variable name can not be {}.'.format( setting['name']) self._logger.error(emsg) raise CommonException( err_code=11006, options={'var_name': setting['name']}, e_message=emsg) except CommonException as ce: raise ce except Exception as e: self._logger.error('validate global setting meta fails. %s', traceback.format_exc()) raise CommonException(e_message=e.message, err_code=11000, options={'message': e.message})
def meta(self, new_meta): if new_meta: self._logger.debug('save global setting meta:%s', logger.hide_sensitive_field(new_meta)) self._meta_mgr.set_app_meta_data({'global_settings': new_meta}) else: self.delete_global_settings_meta()
def _validate_basic_input_meta(self, meta, is_update=False): if common_util.contain_reserved_chars(meta.get('name', '')): e = CommonException() e.set_err_code(3015) raise e sourcetype = meta.get('sourcetype', None) if sourcetype is None: e = CommonException() e.set_err_code(3116) e.set_option('name', meta.get('name')) raise e if not is_update: st_builder = TASourcetypeBuilder(self.__appname, self.__uri, self.__session_key) if sourcetype in st_builder.get_all_sourcetype_names(): e = CommonException() e.set_err_code(3010) e.set_option('sourcetype', sourcetype) raise e # splunk may not restart yet. TA is not load. so, use the conf mgr # with tab context sourcetype_existed = self.__conf_mgr_with_tab_context.get_conf( "props").get_all() if sourcetype in sourcetype_existed: self.__logger.error( "Error when validating meta: %s, Error: sourcetype exists.", logger.hide_sensitive_field(meta)) e = CommonException() e.set_err_code(3012) e.set_option('sourcetype', sourcetype) raise e
def set_customized_options(self, uuid, customized_options): datainputs_existed = self._get_inputs() for datainput in datainputs_existed: if datainput.get('uuid', None) == uuid: datainput['customized_options'] = customized_options self.__logger.info("set customized_options for data input. %s", logger.hide_sensitive_field(datainput)) self.__meta_mgr.set_app_meta_data({"datainputs": datainputs_existed})
def meta(self): meta = self._meta_mgr.get_app_meta_data('global_settings') self._logger.debug('get global setting meta from meta store: %s', logger.hide_sensitive_field(meta)) if meta: return meta else: return {}
def upgrade_from_2_0_0_to_2_1_0(self, input_upgraded): global_settings = self.get_global_settings() if global_settings: self.ta_setup_clean(self._appname, self._current_app_dir) if input_upgraded and not global_settings: # if no global settings, but found inputs are upgraded. # at least, need a log level part to generate UCC global_settings = { 'log_settings': { 'log_level': 'INFO', } } if not global_settings: return False CREDENTIAL_SETTING_KEY = ta_configuration_meta.GlobalSettingMeta.CREDENTIAL_SETTING_KEY CUSTOMIZED_SETTING_KEY = ta_configuration_meta.GlobalSettingMeta.CUSTOMIZED_SETTING_KEY if CREDENTIAL_SETTING_KEY in global_settings: credentials = global_settings.get(CREDENTIAL_SETTING_KEY, {}) if isinstance(credentials, dict): global_settings[CREDENTIAL_SETTING_KEY] = [{ 'username': key, 'password': value.get('password') } for key, value in list(credentials.items())] if CUSTOMIZED_SETTING_KEY in global_settings: customized_settings = global_settings.get(CUSTOMIZED_SETTING_KEY, []) for variable in customized_settings: variable.update({'type': variable.get('format_type')}) if 'value' not in variable: if variable['type'] == 'checkbox': variable['value'] = variable.get('default_value', 0) else: variable['value'] = variable.get('default_value', '') self._logger.info( 'regenerate the UCC resource during upgrading. meta:%s', logger.hide_sensitive_field(global_settings)) # should update the meta before calling upgrade global settings # in update global settings, the meta will be validated self._global_setting_meta.meta = global_settings latest_tab_version, latest_tab_build = upgrade_util.get_latest_tabuilder_version( self._service_with_tab_context) if latest_tab_version == '2.1.0': # only regenerate ucc resources when current installation is 2.1.0 self.update_global_settings(global_settings) return True
def set_app_meta_data(self, key_values, update_last_modify=True): """ Set the meta data based on default namespace & app replace the old meta data with the key_values :key_values: a dict to set. Raise Exception when it's not dict :return None, or throws exception when fails. """ if not isinstance(key_values, dict): raise ValueError("The input parameter should be a dict.") # force reload cache self.project_meta_state.get_state() self.project_meta_state.update_state(self.namespace, key_values) if update_last_modify: self._update_last_modify_time() _logger.debug("Set state. app:%s, key:%s, value:%s", self.app_name, self.namespace, logger.hide_sensitive_field(key_values))
def rename_app(service, old_name, new_name): """ Static method to rename app in all namespaces :service: an splunk service object :old_name: old app name to rename :new_name: new app name after rename :return: True - rename success. False - rename failure. """ builder_meta = meta_client.MetaClient(service, TA_META_COLLECTION_NAME) old_project_meta = meta_client.MetaClient(service, old_name) new_project_meta = meta_client.MetaClient(service, new_name) if builder_meta.get_state(new_name): _logger.error( 'Can not rename to app name {0}, the target app already exists.' .format(new_name)) return False _logger.info("begin to rename add-on from %s to %s", old_name, new_name) old_meta = old_project_meta.get_state() if old_meta is None: _logger.error( "Rename add-on fails. Can not find any meta for add-on %s", old_name) return False try: for k, v in list(old_meta.items()): new_project_meta.update_state(k, v) old_project_meta.delete_state() old_builder_meta = builder_meta.get_state(old_name) builder_meta.update_state(new_name, old_builder_meta) builder_meta.delete_state(old_name) _logger.debug("update meta %s from app<%s> to app<%s>", logger.hide_sensitive_field(old_meta), old_name, new_name) except Exception as e: _logger.error("fail to rename app meta. %s", traceback.format_exc()) return False return True
def create_input_meta(self, datainput): self._validate_new_meta(datainput) datainputs = self._get_inputs() self._validate_input_name(datainput['name'], datainputs) self.__logger.debug("get data inputs meta from meta store:%s", logger.hide_sensitive_field(datainputs)) if self._input_exist(datainput, datainputs): e = CommonException() e.set_err_code(3011) e.set_option('name', datainput.get('name', '')) raise e data_input_meta = self.add_default_values( builder_util.add_unique_identification(datainput)) datainputs.append(data_input_meta) self.__meta_mgr.set_app_meta_data({"datainputs": datainputs}) return datainputs, data_input_meta
def read_global_configuration(self, global_setting_meta): ucc_meta = self.get_ucc_meta(global_setting_meta) schema = GlobalConfigSchema(ucc_meta) global_config = GlobalConfig( common_util.get_splunkd_uri(self._service_with_tab_context), self._service_with_tab_context.token, schema) accounts = global_config.configs.load().get('account', []) settings = global_config.settings.load() global_configuration = dict() global_configuration['credential_settings'] = [{ "username": account['username'], "password": account['password'], "name": account['name'] } for account in accounts] for setting in settings.get('settings', []): self._logger.info("setting: {}".format( logger.hide_sensitive_field(setting))) if setting['name'] == 'logging': global_configuration["log_settings"] = { "log_level": setting.get('loglevel') } elif setting['name'] == "proxy": if 'disabled' in setting: del setting['disabled'] global_configuration["proxy_settings"] = setting else: # should be customized settings global_configuration[ "customized_settings"] = global_setting_meta.get( "customized_settings", []) for customized_setting in global_configuration[ "customized_settings"]: var_name = customized_setting['name'] if var_name in setting: customized_setting['value'] = setting[var_name] return global_configuration
def fetch_input_code(self, datainput): ''' this function tries to get the input python code based on input meta. The data input may not be created yet. will generate python file content ''' data_input_meta = self.__input_meta_mgr.add_default_values( builder_util.add_unique_identification(datainput)) try: # when testing the code, the input may not be saved yet. self.__asset_generator.generate_python_libs_if_not_exist() self.__asset_generator.generate_import_declare_if_not_exist() data_input_meta['code'] = self._get_data_input_code( data_input_meta) return data_input_meta except Exception: self.__logger.error("Fail to get the code. datainput:%s", logger.hide_sensitive_field(data_input_meta)) ce = CommonException() ce.set_err_code(3129) ce.set_option('input_name', data_input_meta['name']) raise ce
def write_global_configuration(self, global_setting_meta, old_meta=None): if old_meta is None: old_meta = self.meta ucc_meta = self.get_ucc_meta(global_setting_meta) schema = GlobalConfigSchema(ucc_meta) global_config = GlobalConfig( common_util.get_splunkd_uri(self._service_with_tab_context), self._service_with_tab_context.token, schema) all_payload = dict() configs_payload = self.get_ucc_configs_content_meta( global_setting_meta, old_meta) if configs_payload: all_payload.update(configs_payload) settings_payload = self.get_ucc_settings_content_meta( global_setting_meta, old_meta) if settings_payload: all_payload.update(settings_payload) if all_payload: self._logger.debug('Save global settings to UCC. payload: %s', logger.hide_sensitive_field(all_payload)) global_config.save(all_payload)
def update_app_meta_data(self, key_values, update_last_modify=True): """ Update meta data based on default namespace & app merge the key_values into existing meta :key_values: a dict to set. Raise exception when it's not a dict :return None, or throws exception when fails. """ if not isinstance(key_values, dict): raise ValueError("The input parameter should be a dict.") app_meta = self.project_meta_state.get_state(self.namespace) if app_meta is None: app_meta = key_values else: app_meta = copy.deepcopy(app_meta) app_meta.update(key_values) self.project_meta_state.update_state(self.namespace, app_meta) if update_last_modify: self._update_last_modify_time() _logger.debug("Update state. app:%s, key: %s, value: %s", self.app_name, self.namespace, logger.hide_sensitive_field(app_meta))
def _run(self): ''' return: 3 element tuple (return_code, raw_stdout_out, raw_stderr_out) ''' ckpt = self._ckpter.get(CKPT_NAME) if ckpt is None: ckpt = {} if CKPT_KEY not in ckpt: ckpt[CKPT_KEY] = {} input_scheme = Template(OPTION_FILE_CONTENT).render( server_uri=self._server_uri, session_key=self._session_key, checkpoint_dir=self._checkpoint_dir, options=self._options, interval=self._interval, input_name=self._input_name, sourcetype=self._sourcetype) # runner_logger.debug('input stream:' + input_scheme) # use python3 for test by default if os.path.isfile(make_splunk_path(('bin', "python3"))) \ or os.path.isfile(make_splunk_path(('bin', "python3.exe"))): cmd2 = [self._get_splunk_bin(), 'cmd', 'python3', self._file_path] else: cmd2 = [self._get_splunk_bin(), 'cmd', 'python', self._file_path] # make it the same as core cwd = "C:\Windows\system32" if platform.system() == "Windows" else '/' # prepare the env child_env = os.environ.copy() child_env[AOB_TEST_FLAG] = 'true' if self._globalsettings: child_env[GLOBALSETTINGS] = json.dumps(self._globalsettings) child_env[DATA_INPUTS_OPTIONS] = json.dumps(self._data_inputs_options) runner_logger.debug("Start the test subprocess with env:%s", logger.hide_sensitive_field({ GLOBALSETTINGS: self._globalsettings, DATA_INPUTS_OPTIONS: self._data_inputs_options })) try: child2 = subprocess.Popen( cmd2, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE, cwd=cwd, env=child_env) ckpt[CKPT_KEY][self._test_id] = { 'pid': child2.pid, 'app': self._app, 'input': self._input_name } self._ckpter.update(CKPT_NAME, ckpt) stdout_str, stderr_str = child2.communicate(input=input_scheme.encode()) stdout_str = stdout_str.decode() stderr_str = stderr_str.decode() retcode = child2.returncode del ckpt[CKPT_KEY][self._test_id] if not has_kill_flag(CKPT_DIR, self._test_id): # normal exist, not killed self._ckpter.update(CKPT_NAME, ckpt) return retcode, stdout_str, stderr_str except subprocess.CalledProcessError as e: runner_logger.error('Fail to execute the test process:%s. %s', e.cmd, traceback.format_exc()) return e.returncode, '', e.output
def upgrade_from_2_0_0_to_2_1_0(self, global_setting_meta): meta_updated = False modular_alert_meta = self._get_alert_meta() for meta in modular_alert_meta: # should update the default value of checkbox parameters # the default value and value of checkbox should be number 0 or 1 for param in meta.get('parameters', []): if param.get('format_type') == 'checkbox': if param.get('default_value') is not None: default_value = 1 if common_util.is_true(param['default_value']) else 0 param['default_value'] = default_value meta_updated = True if param.get('value') is not None: value = 1 if common_util.is_true(param['value']) else 0 param['value'] = value meta_updated = True self.__logger.info('[upgrade] modular metas:%s', modular_alert_meta) if meta_updated: self.__meta_mgr.set_app_meta_data({"modular_alerts": modular_alert_meta}) self.__logger.info('moduler alert meta is upgraded from 2_0_0 to 2_1_0. updated meta:%s', logger.hide_sensitive_field(modular_alert_meta)) # regen all the modular alerts self.do_build(modular_alert_meta, global_setting_meta, output_dir=self.__splunk_app_dir)