def update_TA_alert(self, params): modular_alert_new = params.get("modular_alert", {}) global_settings = params.get("global_settings", {}) if not os.path.isdir(self.__current_ta_dir): raise Exception("{} does not exist".format(self.__appname)) # todo: validate new modular alert data modular_alerts = self._get_alert_meta() uuid = modular_alert_new.get("uuid", None) if not uuid: raise Exception("uuid not found in meta {0}".format(modular_alert_new)) if not self.__alert_exist(modular_alert_new, modular_alerts, use_uuid=True): raise Exception("Modular alert dose not exist") modular_alert_old = [alert for alert in modular_alerts if alert['uuid'] == uuid][0] modular_alerts = [one_alert for one_alert in modular_alerts if one_alert['uuid'] != uuid] if self.__alert_exist(modular_alert_new, modular_alerts): e = CommonException() e.set_err_code(3011) e.set_option('name', modular_alert_new.get('short_name', '')) raise e modular_alerts.append(modular_alert_new) self.__meta_mgr.set_app_meta_data({"modular_alerts": modular_alerts}) self.do_build([modular_alert_new], global_settings, output_dir=self.__splunk_app_dir) common_util.reload_splunk_apps(self.__service_with_tab_context) return modular_alerts
def check_event_count(self, sourcetype): event_count_dict = builder_util.get_event_count(self.service) if event_count_dict.get(sourcetype, 0) == 0: ce = CommonException() ce.set_err_code(4011) ce.set_option('sourcetype', sourcetype) raise ce
def __validate_alert_name(self, name, all_inputs_meta): if TAAlertBuilder.INPUT_NAME_PATTERN.match(name) is None: ce = CommonException() ce.set_err_code(3133) raise ce loaded_modinputs = [ i.name for i in self.__service_with_tab_context.modular_input_kinds ] modinputs_in_meta = [i['name'] for i in all_inputs_meta] name_exist = name in loaded_modinputs or name in modinputs_in_meta if name_exist: ce = CommonException() ce.set_err_code(3119) ce.set_option('name', name) raise ce if common_util.contain_reserved_chars(name): ce = CommonException() ce.set_err_code(3015) raise ce try: checked_module = importlib.import_module(name) if checked_module: ce = CommonException(e_message='Input name {} conflicts with existing python module name.'.format( name), err_code=3136, options={'input_name': name}) raise ce except ImportError: # this is expected errors. self.__logger.debug( 'input name is valid. No package named %s', name)
def update_input_meta(self, datainput_new): self._validate_exist_meta(datainput_new) datainputs = self._get_inputs() uuid = datainput_new.get("uuid", None) if not uuid: raise Exception("uuid not found in meta {0}".format(datainput_new)) if not self._input_exist(datainput_new, datainputs, use_uuid=True): raise Exception("Input dose not exist") self._validate_input_name(datainput_new['name'], datainputs, uuid) datainput_old = [ oneinput for oneinput in datainputs if oneinput['uuid'] == uuid ][0] datainputs = list( [oneinput for oneinput in datainputs if oneinput['uuid'] != uuid]) if self._input_exist(datainput_new, datainputs): e = CommonException() e.set_err_code(3011) e.set_option('name', datainput_new.get('name', '')) raise e data_input_meta_new = self.add_default_values(datainput_new) datainputs.append(data_input_meta_new) self.__meta_mgr.set_app_meta_data({"datainputs": datainputs}) return datainputs, datainput_old, data_input_meta_new
def _create_sourcetype_stanzas(self, datainput): sourcetype = datainput.get('sourcetype', None) if sourcetype is None: e = CommonException() e.set_err_code(3116) e.set_option('name', datainput.get('name')) raise e st_builder = TASourcetypeBuilder(self.__appname, self.__uri, self.__session_key) if sourcetype in st_builder.get_all_sourcetype_names(): e = CommonException() e.set_err_code(3010) e.set_option('sourcetype', sourcetype) raise e try: # should write the props.conf directly, ta may not be loaded yet. st_builder.create_sourcetype(sourcetype, { "SHOULD_LINEMERGE": "0", "pulldown_type": "1", "category": "Splunk App Add-on Builder" }) except Exception as e: self.__logger.error( 'get error when creating sourcetype stanza. %s', traceback.format_exc()) ce = CommonException() ce.set_err_code(3117) ce.set_option('name', datainput.get('name', '')) ce.set_option('msg', '{}'.format(e)) ce.set_option('sourcetype', sourcetype) raise ce
def create_TA_input(self, datainput, reload_input): if not os.path.isdir(self.__current_ta_dir): e = CommonException() e.set_err_code(3118) e.set_option('name', datainput.get('name', '')) raise e global_settings = datainput.get(TAInputMetaMgr.GLOBAL_SETTING_KEY) if TAInputMetaMgr.GLOBAL_SETTING_KEY in datainput: del datainput[TAInputMetaMgr.GLOBAL_SETTING_KEY] datainputs, data_input_meta = self.__input_meta_mgr.create_input_meta( datainput) filtered_datainputs, input_kinds = self.__input_meta_mgr.get_datainputs_and_kinds_for_conf( datainputs) # always refresh the declare py self.__asset_generator.generate_import_declare() # only generate the libs if there is no such a dir self.__asset_generator.generate_python_libs_if_not_exist() self._create_input_py(data_input_meta, overwrite=True, global_settings_meta=global_settings) self._generate_inputs_conf(filtered_datainputs, input_kinds) self._generate_inputs_conf_spec(filtered_datainputs) # detect the single instance mode after py file is updated # have to save the meta again mode_changed = False for _input in datainputs: if _input['uuid'] == data_input_meta['uuid']: old_mode = _input.get('is_single_instance', False) self.detect_single_instance_mode(_input) mode_changed = old_mode != _input.get('is_single_instance', False) break if mode_changed: self.__logger.info('Set single instance mode when create input %s', data_input_meta['name']) self.__input_meta_mgr.set_meta(datainputs) # write global setting meta and ucc configuration content together # at frontend, when creating or updating an input, no need to save global settings in a seperate step ucc_resource_generated = self._ta_configuration_builder.update_global_settings(global_settings, reload_app=reload_input) # common_util.reload_local_app(self.__service_with_tab_context, self.__appname) # regenerate_ucc_resources will reload the app resources, no need to reload here self._create_sourcetype_stanzas(data_input_meta) if datainputs and len(datainputs) == 1: # create the modinput log stanzas # only create the stanza once, calling the conf rest costs time log_source = data_input_util.get_input_log_source_stanza(self.__appname) cce_log_source = data_input_util.get_cce_log_source_stanza(self.__appname) stanza = {"SHOULD_LINEMERGE": "true", "sourcetype": self.get_input_log_sourcetype()} conf_mgr = TabConfMgr(self.__uri, self.__session_key, self.__appname, self.__service_with_tab_context) conf_mgr.update_conf_stanza('props', log_source, {}, stanza) conf_mgr.update_conf_stanza('props', cce_log_source, {}, stanza) self.__logger.debug("create the input log props stanza in app " + self.__appname) return datainputs, data_input_meta, ucc_resource_generated
def delete_temp_dir(self, dirname, folder="."): abs_dir = self._get_file_path(dirname, folder) try: if os.path.isdir(abs_dir): shutil.rmtree(abs_dir) except: ex = CommonException() ex.set_err_code(9001) ex.set_option('filepath', abs_dir) raise ex
def create_temp_dir(self, dirname, folder=".", delete_exist=True): if delete_exist: self.delete_temp_dir(dirname, folder) try: abs_dir = self._get_file_path(dirname, folder) os.makedirs(abs_dir) except: ex = CommonException() ex.set_err_code(9000) ex.set_option('filepath', abs_dir) raise ex
def get_temp_file_cont(self, filename, folder="."): fpath = self._get_file_path(filename, folder) if not os.path.isfile(fpath): return None try: with open(fpath, "r") as f: cont = f.read() return cont except: ex = CommonException() ex.set_err_code(9004) ex.set_option('filepath', fpath) raise ex
def delete_temp_file(self, filename, check_exist=False): fpath = self._get_file_path(filename) if check_exist and not os.path.isfile(fpath): return False try: if os.path.isfile(fpath): os.remove(fpath) return True except: ex = CommonException() ex.set_err_code(9001) ex.set_option('filepath', fpath) raise ex
def copy_to_temp(self, src, temp_filename, folder=".", force=True): fpath = self._get_file_path(temp_filename, folder) if not force and os.path.isfile(fpath): return False try: directory = os.path.dirname(fpath) if not os.path.isdir(directory): os.mkdir(directory) if force and os.path.isfile(fpath): os.remove(fpath) shutil.copy(src, fpath) return True except Exception as e: ex = CommonException(e_message=str(e)) ex.set_err_code(9002) ex.set_option('src_file', src) ex.set_option("dst_file", fpath) raise ex
def create_input_meta(self, datainput): self._validate_new_meta(datainput) datainputs = self._get_inputs() self._validate_input_name(datainput['name'], datainputs) self.__logger.debug("get data inputs meta from meta store:%s", logger.hide_sensitive_field(datainputs)) if self._input_exist(datainput, datainputs): e = CommonException() e.set_err_code(3011) e.set_option('name', datainput.get('name', '')) raise e data_input_meta = self.add_default_values( builder_util.add_unique_identification(datainput)) datainputs.append(data_input_meta) self.__meta_mgr.set_app_meta_data({"datainputs": datainputs}) return datainputs, data_input_meta
def get_table_format_results(self, sourcetype, delim=None): if not delim: meta = self.get_meta_results(sourcetype) or {} delim = meta.get("table_results", {}).get("delim", r" ") try: events = self.get_events(sourcetype, batch_size=1000) if not events: ce = CommonException() ce.set_err_code(4011) ce.set_option("sourcetype", sourcetype) raise ce handler = TableHandler(events, delim) return handler.get_table_results() except InvalidRegex as e: ex = CommonException() ex.set_err_code(4012) ex.set_option('regex', e.regex) raise ex
def rotate_temp_files(self): curr_time = time.time() for fpath in os.listdir(self.temp_dir): # skip the dir like __events__ if fpath.startswith("__"): continue fpath = os.path.join(self.temp_dir, fpath) last_modified = os.path.getmtime(fpath) if curr_time - last_modified >= self.time_delta: try: if os.path.isfile(fpath): os.remove(fpath) elif os.path.isdir(fpath): shutil.rmtree(fpath) except: ex = CommonException() ex.set_err_code(9001) ex.set_option('filepath', fpath) raise ex
def run(self, sourcetype, progress_func=None): events = self.get_events(sourcetype) if not events: ce = CommonException() ce.set_err_code(4011) ce.set_option('sourcetype', sourcetype) raise ce loader = RegexLoader() regex_results = loader.run(events, progress_func) if not regex_results: return None self.event_mgr.load_all_values() self.event_mgr.remove_events(sourcetype) res = self.get_frontend_results(regex_results) # move the events to files rather than KV store for group in regex_results.get("groups"): group_id = group.get("group_id") self.event_mgr.save_events(sourcetype, group_id, group["raw_events"]) del group["events"] del group["raw_events"] del group["regex"]["possible_values"] del group["seed"]["shared_str_indexes"] del group["seed"]["insert_str_points"] del group["seed"]["raw_events"] meta = self.meta_mgr.get_app_meta_data(sourcetype) or {} meta.update({"regex_results_temp": regex_results.copy()}) try: self.meta_mgr.update_app_meta_data({sourcetype: meta}) except UnicodeDecodeError: ce = CommonException() ce.set_err_code(4017) raise ce return res
def copy_temp_to_dest(self, temp_filename, dest, force=True): if not force and os.path.isfile(dest): return False fpath = self._get_file_path(temp_filename) if not os.path.isfile(fpath): return False if force and os.path.isfile(dest): os.remove(dest) try: shutil.copy(fpath, dest) return True except Exception as e: ex = CommonException(e_message=str(e)) ex.set_err_code(9002) ex.set_option('src_file', fpath) ex.set_option("dst_file", dest) raise ex
def _validate_customized_vars(self, datainput): var_list = datainput.get('data_inputs_options', []) for v in var_list: if v.get('rest_header', False) or v.get( 'type', '') in self.REST_CKPT_OPTIONS: continue if 'name' not in v: self.__logger.error( "name field is not found in customized option part:%s", v) ce = CommonException() ce.set_err_code(3130) raise ce else: name = v['name'] t = v.get('type', '') if t == data_input_util.CUSTOMIZED_VAR_TYPE and TAInputMetaMgr.INPUT_PROPERTY_NAME_PATTERN.match( name ) is None and name != TAInputMetaMgr.GLOBAL_ACCOUNT_NAME: self.__logger.error( "customized variable name:%s is not valid.", name) ce = CommonException(err_code=3131) ce.set_option('prop_name', name) raise ce if name in TAInputMetaMgr.INPUT_RESERVED_PROPERTIES: self.__logger.error( "customized variable name:%s is in the reseved list.", name) ce = CommonException() ce.set_err_code(3132) ce.set_option('prop_name', name) raise ce if t == data_input_util.CUSTOMIZED_VAR_TYPE: for k in TAInputMetaMgr.CUSTOMIZED_VAR_REQUIRED_KEYS: if k not in v: emsg = 'Required field {} not found in customized variable.'.format( k) self.__logger.error(emsg) ce = CommonException(err_code=3144, e_message=emsg, options={'attribute': k}) raise ce
def detect_sample_format(self, sourcetype, except_if_no_events=True): meta = self.get_meta_results(sourcetype) or {} if meta.get("is_parsed", False): return { "data_format": meta.get("data_format", None), "table_results": { "delim": meta.get("table_results", {}).get("delim", None) }, "is_parsed": True, } events = self.get_events(sourcetype, 50) if len(events) < 10: _LOGGER.warn( "Skip sample format detection since the event count < 10 for sourcetype {}" .format(sourcetype)) return {} from field_extraction_builder.data_format.format_handler_base import DataFormatHandler handler = DataFormatHandler(events) formats = handler.get_format() if not formats: if except_if_no_events: ex = CommonException() ex.set_err_code(4011) ex.set_option('sourcetype', sourcetype) raise ex else: return {} table_delim = formats.get("table_delim") meta["data_format"] = formats.get("data_format") if table_delim: meta["table_results"] = {"delim": table_delim} self.update_meta_results(sourcetype, meta) return meta
def _validate_basic_input_meta(self, meta, is_update=False): if common_util.contain_reserved_chars(meta.get('name', '')): e = CommonException() e.set_err_code(3015) raise e sourcetype = meta.get('sourcetype', None) if sourcetype is None: e = CommonException() e.set_err_code(3116) e.set_option('name', meta.get('name')) raise e if not is_update: st_builder = TASourcetypeBuilder(self.__appname, self.__uri, self.__session_key) if sourcetype in st_builder.get_all_sourcetype_names(): e = CommonException() e.set_err_code(3010) e.set_option('sourcetype', sourcetype) raise e # splunk may not restart yet. TA is not load. so, use the conf mgr # with tab context sourcetype_existed = self.__conf_mgr_with_tab_context.get_conf( "props").get_all() if sourcetype in sourcetype_existed: self.__logger.error( "Error when validating meta: %s, Error: sourcetype exists.", logger.hide_sensitive_field(meta)) e = CommonException() e.set_err_code(3012) e.set_option('sourcetype', sourcetype) raise e
def fetch_input_code(self, datainput): ''' this function tries to get the input python code based on input meta. The data input may not be created yet. will generate python file content ''' data_input_meta = self.__input_meta_mgr.add_default_values( builder_util.add_unique_identification(datainput)) try: # when testing the code, the input may not be saved yet. self.__asset_generator.generate_python_libs_if_not_exist() self.__asset_generator.generate_import_declare_if_not_exist() data_input_meta['code'] = self._get_data_input_code( data_input_meta) return data_input_meta except Exception: self.__logger.error("Fail to get the code. datainput:%s", logger.hide_sensitive_field(data_input_meta)) ce = CommonException() ce.set_err_code(3129) ce.set_option('input_name', data_input_meta['name']) raise ce
def create_TA_alert(self, params): modular_alert = params.get("modular_alert", {}) global_settings = params.get("global_settings", {}) modular_alerts = self._get_alert_meta() if self.__alert_exist(modular_alert, modular_alerts): e = CommonException() e.set_err_code(10000) e.set_option('name', modular_alert.get('short_name', '')) raise e modular_alert_meta = builder_util.add_unique_identification( modular_alert) modular_alerts.append(modular_alert_meta) self.__meta_mgr.set_app_meta_data({"modular_alerts": modular_alerts}) self.do_build([modular_alert], global_settings, output_dir=self.__splunk_app_dir) self.create_alert_log_stanza(modular_alert['short_name']) common_util.reload_splunk_apps(self.__service_with_tab_context) return modular_alert_meta.get("uuid")
def _write_temp_file(self, filename, filecont, folder=".", check_exist=False, mode="w"): fpath = self._get_file_path(filename, folder) if check_exist and not os.path.isfile(fpath): return False try: directory = os.path.dirname(fpath) if not os.path.isdir(directory): os.makedirs(directory) with open(fpath, mode) as f: f.write(filecont) return True except: ex = CommonException() ex.set_err_code(9003) ex.set_option('filepath', fpath) raise ex
def get_kv_format_results(self, sourcetype, delim_pair, delim_kv, regex): try: events = self.get_events(sourcetype, batch_size=1000) if not events: ce = CommonException() ce.set_err_code(4011) ce.set_option("sourcetype", sourcetype) raise ce handler = KVHandler(events, delim_pair, delim_kv, regex) return handler.get_kv_results() except InvalidRegex as e: ex = CommonException() ex.set_err_code(4012) ex.set_option('regex', e.regex) raise ex except CaptureGroupCountError as e: ex = CommonException() ex.set_err_code(4013) ex.set_option('regex', e.regex) raise ex
def _validate_new_meta(self, meta, is_update=False): for k in self.required_meta_keys: if k not in meta: ce = CommonException( e_message='{0} not found in meta {1}'.format(k, meta), err_code=3137, options={'property_name': k}) raise ce self._validate_basic_input_meta(meta, is_update) self._validate_customized_vars(meta) input_type = meta['type'] if input_type == 'customized': options = meta.get('data_inputs_options', []) options_existed = {} for option in options: if 'name' not in option: raise Exception( 'name is not found for input option {0}'.format( option)) else: if option['name'] in options_existed: e = CommonException() e.set_err_code(3013) e.set_option('name', option['name']) raise e else: options_existed[option['name']] = True elif input_type == 'rest': options = meta.get('data_inputs_options', []) url_option_found = False method_option_found = False for option in options: if option.get( 'type', '' ) == data_input_util.CUSTOMIZED_VAR_TYPE or option.get( 'type', '') in self.REST_CKPT_OPTIONS: continue if 'name' not in option: raise Exception( 'name is not found for input option {0}'.format( option)) if 'value' not in option: raise Exception( 'value is not found for input option {0}'.format( option)) if 'rest_header' not in option: raise Exception( 'rest_handler is not found for input option {0}'. format(option)) if option['name'] == '_rest_api_url': url_option_found = True if option['name'] == '_rest_api_method': method_option_found = True if url_option_found is False: raise Exception('_rest_api_url option not found.') if method_option_found is False: raise Exception('_rest_api_method option not found.') elif input_type == 'command': options = meta.get('data_inputs_options', []) command_found = False for option in options: if option.get('type', '') == data_input_util.CUSTOMIZED_VAR_TYPE: continue if 'name' not in option: raise Exception( 'name is not found for input option {0}'.format( option)) if 'value' not in option: raise Exception( 'value is not found for input option {0}'.format( option)) if option['name'] == '_command': command_found = True break if command_found is False: raise Exception('command option is not found.') else: raise Exception('Invalid data input type: {0}'.format(input_type))
def _validate_input_name(self, name, all_inputs_meta, uuid=None): should_check_module_name = True if uuid: for _input in all_inputs_meta: if _input['uuid'] == uuid: # if it is update request, and no name changes, do not check # the module name, because there is a input python file # there should_check_module_name = (_input['name'] != name) if not name: ce = CommonException( e_message='data input name should not be empty.', err_code=3137, options={'property_name': 'name'}) raise ce if name in TAInputMetaMgr.INPUT_RESERVED_NAMES: ce = CommonException(e_message='data input name is reserved.', err_code=3145, options={'input_name': name}) raise ce if TAInputMetaMgr.INPUT_NAME_PATTERN.match(name) is None: ce = CommonException() ce.set_err_code(3133) raise ce if common_util.contain_reserved_chars(name): ce = CommonException() ce.set_err_code(3015) raise ce # for name conflicts, should tell if this is a update or an create need_check_input_name_conflict = True if uuid: input_meta = [ meta for meta in all_inputs_meta if meta['uuid'] == uuid ] if not input_meta: ce = CommonException( e_message='No input with uuid {}'.format(uuid), err_code=3138) self.__logger.error( 'No input with uuid %s, validate input name fails.', uuid) raise ce input_meta = input_meta[0] if input_meta['name'] == name: # It is fine. No rename for input update need_check_input_name_conflict = False if need_check_input_name_conflict: # need to check name conflict when creating an input and renaming # the input loaded_modinputs = [ i.name for i in self.__service_with_tab_context.modular_input_kinds ] modinputs_in_meta = [i['name'] for i in all_inputs_meta] name_exist = name in loaded_modinputs or name in modinputs_in_meta if name_exist: ce = CommonException() ce.set_err_code(3119) ce.set_option('name', name) raise ce try: if should_check_module_name: checked_module = importlib.import_module(name) if checked_module: ce = CommonException( e_message= 'Input name {} conflicts with existing python module name.' .format(name), err_code=3136, options={'input_name': name}) raise ce except ImportError: # this is expected errors. self.__logger.debug('input name is valid. No package named %s', name) # check if the input name conflicts with alert name if self.__alert_builder: self.__logger.debug('validate input name:%s', name) if self.__alert_builder.is_alert_exist(name): ce = CommonException( e_message= 'Input name {} conflicts with an existing alert action name.' .format(name), err_code=3143, options={'input_name': name}) raise ce