def validate_global_setting_meta(self, meta): try: if self.CUSTOMIZED_SETTING_KEY in meta: cus_setting = meta[self.CUSTOMIZED_SETTING_KEY] for setting in cus_setting: for k in self.CUSTOMIZED_VAR_REQUIRED_KEYS: if k not in setting: emsg = 'Required field {} not found in customized variable {}.'.format( k, logger.hide_sensitive_field(setting)) self._logger.error(emsg) raise CommonException(err_code=11001, options={ 'field': k, 'v_name': setting.get( 'name', 'unknown') }, e_message=emsg) if setting['name'] in self.RESERVED_CUSTOMIZED_VAR_NAMES: emsg = 'Global variable name can not be {}.'.format( setting['name']) self._logger.error(emsg) raise CommonException( err_code=11006, options={'var_name': setting['name']}, e_message=emsg) except CommonException as ce: raise ce except Exception as e: self._logger.error('validate global setting meta fails. %s', traceback.format_exc()) raise CommonException(e_message=e.message, err_code=11000, options={'message': e.message})
def reload_local_app(service, app_name): ''' reload the specific app with the endpoint: /servicesNS/nobody/system/apps/local/<app_name>/_reload This endpoint will reload all the data inputs :param service: splunklib service object :type service: ``object`` :param app_name: the app needs to be reloaded :type app_name: ``string`` ''' endpoint_path = '/servicesNS/nobody/system/apps/local/{}/_reload'.format( app_name) try: response = service.get(endpoint_path) if response['status'] != 200: _logger.error('fail to reload splunk app %s. status is not 200.', app_name) raise CommonException( e_message='fail to reload splunk app {}'.format(app_name), err_code=45, options={ 'app': app_name, 'http_code': response['status'] }) except HTTPError as he: _logger.error('fail to reload splunk app %s. %s', app_name, traceback.format_exc()) raise CommonException( e_message='fail to reload splunk app {}'.format(app_name), err_code=45, options={ 'app': app_name, 'http_code': response['status'] })
def submit_validation_job(self, action, **params): cl = cherrypy.request.headers["Content-Length"] raw_body = cherrypy.request.body.read(int(cl)) params = json.loads(raw_body) session_key = cherrypy.session.get("sessionKey") splunkd_uri = scc.getMgmtUri() appname = params["app_name"] or ta_project_meta.get_appname({}) try: vbuilder = TAValidationBuilder(splunkd_uri, session_key, appname) validators = [v.strip() for v in params["validators"].split(",")] # validate the username & password when app certification is selected if "app_cert_validation" in validators: conf_mgr = common_util.create_conf_mgr(session_key, splunkd_uri) settings = conf_mgr.get_conf( builder_constant.GLOBAL_SETTING_CONF_NAME) conf = settings.get(builder_constant.APP_CERT_STANZA) if not conf.get("username") or not conf.get("password"): ce = CommonException() ce.set_err_code(6008) raise ce # try to get the token. If failed, will throw exceptions app_cert = AppCert(splunkd_uri, session_key, builder_constant.ADDON_BUILDER_APP_NAME) app_cert.app_conf = app_cert._get_app_cert_conf( need_validation=False) app_cert.get_token() # vbuilder.remove_all_validation_data_inputs() vid = vbuilder.start_validation_job(validators) result = { "validation_id": vid, "submission_result": "success", "app_name": appname } return self.render_json(result) except CommonException as e: vbuilder.cancel_validation_job() logger.error( 'Get CommonException when starting validation. meta:%s, error:%s', params, traceback.format_exc()) return self.render_json({ 'err_code': e.get_err_code(), 'err_args': e.get_options() }) except Exception as e: vbuilder.cancel_validation_job() logger.error("Cannot start validation. error: %s", traceback.format_exc()) raise e
def update_eval(self, sourcetypes, output_field, expression, search, old_output_field=None, old_expression=None, check_exist=False): output_field = output_field.strip(" ") # validate if the field name contains blanks err = ko_common_util.get_field_name_err(output_field, quote="'") if err: raise err if old_output_field == output_field and old_expression == expression: return None # check if eval exists for all sourcetypes if check_exist or output_field != old_output_field: stanzas = self.tab_conf_mgr.get_conf_stanza("props", curr_app_only=True) for stanza in stanzas: sourcetype = stanza.get("name") eval_name = "EVAL-" + output_field if sourcetype in sourcetypes and stanza.get(eval_name): raise CommonException(err_code=5020, options={"field": output_field}) for sourcetype in sourcetypes: eval_util.update_eval(self.tab_conf_mgr, sourcetype, output_field, expression, old_output_field, old_expression, check_exist=check_exist) res = self._get_new_values(search, output_field, old_output_field) return res
def create_eventtype(self, name, search, sourcetype_dict): self.validate_eventtype_name(name) self.get_sourcetypes_from_eventtype_search(search, sourcetype_dict) success = eventtype_util.create_eventtype(self.tab_conf_mgr, name, search) if not success: raise CommonException(err_code=5007, options={"eventtype": name})
def delete_app(service, app_id): ''' delete the app with splunkd rest api :param service: a splunklib service object. :type service: ``object`` :param app_id: the app id to be deleted :type app_id: ``string`` :returns: None. If fails, exception is thrown :rtype: ``None`` ''' endpoint_path = '/services/apps/local/{}'.format(app_id) try: response = service.delete(endpoint_path) if response['status'] not in (200, 404): _logger.error('fail to delete splunk app %s. status is %s.', app_id, response['status']) raise CommonException( e_message='fail to delete splunk app {}'.format(app_id), err_code=48, options={ 'app': app_id, 'http_code': response['status'] }) except HTTPError as he: if he.status != 404: _logger.warning('fail to delete splunk app %s via API. %s', app_id, traceback.format_exc())
def _validate_exist_meta(self, meta): self._validate_new_meta(meta, True) if "uuid" not in meta: ce = CommonException(e_message='{0} not found in meta {1}'.format( 'uuid', meta), err_code=3137, options={'property_name': 'uuid'}) raise ce
def validate_eval(self, sourcetypes, expression, field_values, eval_functions): # TODO: add this to validator eval_function_names = list(eval_functions.keys()) common_util.validate_brackets_and_quotes(expression) input_fields = eval_util.get_eval_input_fields(expression, eval_function_names) for values in field_values: name = values.get("name") # validate if the input fields are from another EVAL if name in input_fields: input_fields.remove(name) for sourcetype in sourcetypes: source = values.get("source", {}).get(sourcetype, []) raise CommonException(err_code=5018, options={"sourcetype": sourcetype, "field": name}) # validate if all the input fields exist if input_fields: raise CommonException(err_code=5019, options={"fields": "/".join(input_fields)})
def run(self, sourcetype, progress_func=None): events = self.get_events(sourcetype) if not events: ce = CommonException() ce.set_err_code(4011) ce.set_option('sourcetype', sourcetype) raise ce loader = RegexLoader() regex_results = loader.run(events, progress_func) if not regex_results: return None self.event_mgr.load_all_values() self.event_mgr.remove_events(sourcetype) res = self.get_frontend_results(regex_results) # move the events to files rather than KV store for group in regex_results.get("groups"): group_id = group.get("group_id") self.event_mgr.save_events(sourcetype, group_id, group["raw_events"]) del group["events"] del group["raw_events"] del group["regex"]["possible_values"] del group["seed"]["shared_str_indexes"] del group["seed"]["insert_str_points"] del group["seed"]["raw_events"] meta = self.meta_mgr.get_app_meta_data(sourcetype) or {} meta.update({"regex_results_temp": regex_results.copy()}) try: self.meta_mgr.update_app_meta_data({sourcetype: meta}) except UnicodeDecodeError: ce = CommonException() ce.set_err_code(4017) raise ce return res
def validate_brackets_and_quotes(data, brackets=[('(', ')')]): blen = len(brackets) bcount = [0 for i in range(blen)] in_quote = False quote = None msg = "Brackets mismatched in the string '{}'.".format(data) bracket_except = CommonException(err_code=5015, e_message=msg, options={"data": data}) for c in data: if c == '"' and not in_quote: in_quote = True quote = c elif c == "'" and not in_quote: in_quote = True quote = c elif in_quote and c == quote: in_quote = False quote = None if in_quote: continue for i in range(blen): start, end = brackets[i] if c == start: bcount[i] += 1 elif c == end: bcount[i] -= 1 if bcount[i] < 0: raise bracket_except if in_quote: msg = "Quotes mismatched in the string '{}'.".format(data) raise CommonException(err_code=5016, e_message=msg, options={"data": data}) for count in bcount: if count != 0: raise bracket_except
def update_alias(self, sourcetypes, output_field, input_field, search, old_output_field=None, old_input_field=None, check_exist=False): output_field = output_field.strip(" ") input_field = input_field.strip(" ") # validate if the field name contains blanks err = ko_common_util.get_field_name_err(output_field) if err: raise err # validate if the field name contains blanks err = ko_common_util.get_field_name_err(input_field) if err: raise err if output_field == input_field: raise CommonException(err_code=5023, options={"field": output_field}) if old_output_field == output_field and old_input_field == input_field: return None # Use closure is only for performance monitor @metric_util.function_run_time(tags=['cim_builder']) def check_alias_exist(): # check if alias exists for all sourcetypes if check_exist or output_field != old_output_field or input_field != old_input_field: stanzas = self.tab_conf_mgr.get_conf_stanza("props", curr_app_only=True) for stanza in stanzas: sourcetype = stanza.get("name") if sourcetype not in sourcetypes: continue alias_value_regex = r"{} +[Aa][Ss] +{}".format(input_field, output_field) for k, v in list(stanza.items()): if not k.startswith("FIELDALIAS-"): continue if re.match(alias_value_regex, v): raise CommonException(err_code=5024, options={"field": output_field}) check_alias_exist() for sourcetype in sourcetypes: alias_util.update_alias(self.tab_conf_mgr, sourcetype, input_field, output_field, old_input_field, old_output_field, check_exist=check_exist) res = self._get_new_values(search, output_field, old_output_field) return res
def check_event_count(self, sourcetype): event_count_dict = builder_util.get_event_count(self.service) if event_count_dict.get(sourcetype, 0) == 0: ce = CommonException() ce.set_err_code(4011) ce.set_option('sourcetype', sourcetype) raise ce
def update_TA_alert(self, params): modular_alert_new = params.get("modular_alert", {}) global_settings = params.get("global_settings", {}) if not os.path.isdir(self.__current_ta_dir): raise Exception("{} does not exist".format(self.__appname)) # todo: validate new modular alert data modular_alerts = self._get_alert_meta() uuid = modular_alert_new.get("uuid", None) if not uuid: raise Exception("uuid not found in meta {0}".format(modular_alert_new)) if not self.__alert_exist(modular_alert_new, modular_alerts, use_uuid=True): raise Exception("Modular alert dose not exist") modular_alert_old = [alert for alert in modular_alerts if alert['uuid'] == uuid][0] modular_alerts = [one_alert for one_alert in modular_alerts if one_alert['uuid'] != uuid] if self.__alert_exist(modular_alert_new, modular_alerts): e = CommonException() e.set_err_code(3011) e.set_option('name', modular_alert_new.get('short_name', '')) raise e modular_alerts.append(modular_alert_new) self.__meta_mgr.set_app_meta_data({"modular_alerts": modular_alerts}) self.do_build([modular_alert_new], global_settings, output_dir=self.__splunk_app_dir) common_util.reload_splunk_apps(self.__service_with_tab_context) return modular_alerts
def update_input_meta(self, datainput_new): self._validate_exist_meta(datainput_new) datainputs = self._get_inputs() uuid = datainput_new.get("uuid", None) if not uuid: raise Exception("uuid not found in meta {0}".format(datainput_new)) if not self._input_exist(datainput_new, datainputs, use_uuid=True): raise Exception("Input dose not exist") self._validate_input_name(datainput_new['name'], datainputs, uuid) datainput_old = [ oneinput for oneinput in datainputs if oneinput['uuid'] == uuid ][0] datainputs = list( [oneinput for oneinput in datainputs if oneinput['uuid'] != uuid]) if self._input_exist(datainput_new, datainputs): e = CommonException() e.set_err_code(3011) e.set_option('name', datainput_new.get('name', '')) raise e data_input_meta_new = self.add_default_values(datainput_new) datainputs.append(data_input_meta_new) self.__meta_mgr.set_app_meta_data({"datainputs": datainputs}) return datainputs, datainput_old, data_input_meta_new
def reload_splunk_apps(service): ''' reload the splunk apps with the services/apps/local/_reload endpoint :param service: a splunklib service object. You can use create_splunk_service to create one :type service: ``object`` :returns: None. If fails, exception is thrown :rtype: ``None`` ''' try: response = service.apps.get('_reload') if response['status'] != 200: _logger.error('Fail to reload the splunk apps. http response:%s', response) raise CommonException(e_message='fail to reload the splunk apps.', err_code=39, options={'http_code': response['status']}) except HTTPError as he: _logger.error('Fail to reload the splunk apps. %s', traceback.format_exc()) raise CommonException(e_message='fail to reload the splunk apps.', err_code=39, options={'http_code': he.status})
def create_TA_input(self, datainput, reload_input): if not os.path.isdir(self.__current_ta_dir): e = CommonException() e.set_err_code(3118) e.set_option('name', datainput.get('name', '')) raise e global_settings = datainput.get(TAInputMetaMgr.GLOBAL_SETTING_KEY) if TAInputMetaMgr.GLOBAL_SETTING_KEY in datainput: del datainput[TAInputMetaMgr.GLOBAL_SETTING_KEY] datainputs, data_input_meta = self.__input_meta_mgr.create_input_meta( datainput) filtered_datainputs, input_kinds = self.__input_meta_mgr.get_datainputs_and_kinds_for_conf( datainputs) # always refresh the declare py self.__asset_generator.generate_import_declare() # only generate the libs if there is no such a dir self.__asset_generator.generate_python_libs_if_not_exist() self._create_input_py(data_input_meta, overwrite=True, global_settings_meta=global_settings) self._generate_inputs_conf(filtered_datainputs, input_kinds) self._generate_inputs_conf_spec(filtered_datainputs) # detect the single instance mode after py file is updated # have to save the meta again mode_changed = False for _input in datainputs: if _input['uuid'] == data_input_meta['uuid']: old_mode = _input.get('is_single_instance', False) self.detect_single_instance_mode(_input) mode_changed = old_mode != _input.get('is_single_instance', False) break if mode_changed: self.__logger.info('Set single instance mode when create input %s', data_input_meta['name']) self.__input_meta_mgr.set_meta(datainputs) # write global setting meta and ucc configuration content together # at frontend, when creating or updating an input, no need to save global settings in a seperate step ucc_resource_generated = self._ta_configuration_builder.update_global_settings(global_settings, reload_app=reload_input) # common_util.reload_local_app(self.__service_with_tab_context, self.__appname) # regenerate_ucc_resources will reload the app resources, no need to reload here self._create_sourcetype_stanzas(data_input_meta) if datainputs and len(datainputs) == 1: # create the modinput log stanzas # only create the stanza once, calling the conf rest costs time log_source = data_input_util.get_input_log_source_stanza(self.__appname) cce_log_source = data_input_util.get_cce_log_source_stanza(self.__appname) stanza = {"SHOULD_LINEMERGE": "true", "sourcetype": self.get_input_log_sourcetype()} conf_mgr = TabConfMgr(self.__uri, self.__session_key, self.__appname, self.__service_with_tab_context) conf_mgr.update_conf_stanza('props', log_source, {}, stanza) conf_mgr.update_conf_stanza('props', cce_log_source, {}, stanza) self.__logger.debug("create the input log props stanza in app " + self.__appname) return datainputs, data_input_meta, ucc_resource_generated
def download_exported_ta_project_file(self, action, **params): uri = scc.getMgmtUri() session = cherrypy.session.get("sessionKey") app = params.get('app', None) try: if app: migrator = AppMigrator(app, uri, session) app_root_dir = make_splunk_path(['etc', 'apps', app]) tar_file = migrator.get_exported_file_full_path(app_root_dir) if not os.path.isfile(tar_file): raise CommonException( e_message='tgz file {} not found.'.format(tar_file), err_code=41, options={'app': app}) return serve_file(tar_file, "application/x-download", "attachment") else: raise CommonException(e_message='app is not set.', err_code=40) except CommonException as ce: logger.error('%s', traceback.format_exc()) return self.render_json({ 'err_code': ce.get_err_code(), 'err_args': ce.get_options() })
def check_alias_exist(): # check if alias exists for all sourcetypes if check_exist or output_field != old_output_field or input_field != old_input_field: stanzas = self.tab_conf_mgr.get_conf_stanza("props", curr_app_only=True) for stanza in stanzas: sourcetype = stanza.get("name") if sourcetype not in sourcetypes: continue alias_value_regex = r"{} +[Aa][Ss] +{}".format(input_field, output_field) for k, v in list(stanza.items()): if not k.startswith("FIELDALIAS-"): continue if re.match(alias_value_regex, v): raise CommonException(err_code=5024, options={"field": output_field})
def copy_to_temp(self, src, temp_filename, folder=".", force=True): fpath = self._get_file_path(temp_filename, folder) if not force and os.path.isfile(fpath): return False try: directory = os.path.dirname(fpath) if not os.path.isdir(directory): os.mkdir(directory) if force and os.path.isfile(fpath): os.remove(fpath) shutil.copy(src, fpath) return True except Exception as e: ex = CommonException(e_message=str(e)) ex.set_err_code(9002) ex.set_option('src_file', src) ex.set_option("dst_file", fpath) raise ex
def delete_temp_dir(self, dirname, folder="."): abs_dir = self._get_file_path(dirname, folder) try: if os.path.isdir(abs_dir): shutil.rmtree(abs_dir) except: ex = CommonException() ex.set_err_code(9001) ex.set_option('filepath', abs_dir) raise ex
def _dump_add_on_project_meta(self, workspace): ''' dump all the meta data from KVstore ''' meta = meta_manager.MetaManager.get_app_all_meta( self.service, self.app) if not meta: raise CommonException( e_message='fail to get the meta for project ' + self.app, err_code=34, options={'app': self.app}) meta = meta_util.remove_user_credential_in_meta(meta) output_file = os.path.join( workspace, package_util.get_aob_meta_file_name(self.app)) with open(output_file, 'w') as f: json.dump(meta, f) _logger.info('dump the app %s meta', self.app) return meta
def copy_temp_to_dest(self, temp_filename, dest, force=True): if not force and os.path.isfile(dest): return False fpath = self._get_file_path(temp_filename) if not os.path.isfile(fpath): return False if force and os.path.isfile(dest): os.remove(dest) try: shutil.copy(fpath, dest) return True except Exception as e: ex = CommonException(e_message=str(e)) ex.set_err_code(9002) ex.set_option('src_file', fpath) ex.set_option("dst_file", dest) raise ex
def create_temp_dir(self, dirname, folder=".", delete_exist=True): if delete_exist: self.delete_temp_dir(dirname, folder) try: abs_dir = self._get_file_path(dirname, folder) os.makedirs(abs_dir) except: ex = CommonException() ex.set_err_code(9000) ex.set_option('filepath', abs_dir) raise ex
def get_temp_file_cont(self, filename, folder="."): fpath = self._get_file_path(filename, folder) if not os.path.isfile(fpath): return None try: with open(fpath, "r") as f: cont = f.read() return cont except: ex = CommonException() ex.set_err_code(9004) ex.set_option('filepath', fpath) raise ex
def delete_temp_file(self, filename, check_exist=False): fpath = self._get_file_path(filename) if check_exist and not os.path.isfile(fpath): return False try: if os.path.isfile(fpath): os.remove(fpath) return True except: ex = CommonException() ex.set_err_code(9001) ex.set_option('filepath', fpath) raise ex
def load_app_all_meta(service, app, meta, overwrite=False): """ load the meta for the app. If loading fails, exception is thrown This method checks if the app already exists in the meta store """ builder_meta = meta_client.MetaClient(service, TA_META_COLLECTION_NAME) project_meta = meta_client.MetaClient(service, app) app_meta = builder_meta.get_state(app) if app_meta: if overwrite: project_meta.delete_state() builder_meta.delete_state(app) else: raise CommonException(e_message='app {} exists'.format(app), err_code=36, options={'app': app}) for k, v in list(meta.items()): project_meta.update_state(k, v) app_meta = {LAST_MODIFY_TIME_KEY: MetaManager.get_current_time()} builder_meta.update_state(app, app_meta)
def get_sourcetypes_from_eventtype_search(self, search_str, sourcetype_dict, validate_selected=True): # validate if the search string contains sourcetypes valid_search = search_util.is_splunk_search_valid(self.service, search_str) if not valid_search: raise CommonException(err_code=5027, options={"search": search_str}) search_without_quotes = common_util.replace_quotes(search_str).get("data") if "|" in search_without_quotes: raise CommonException(err_code=5028) sourcetypes = eventtype_util.get_sourcetypes_from_search_str(search_str) if not sourcetypes: e = CommonException(err_code=5013, options={"search": search_str}) raise e app_sourcetypes = list(sourcetype_dict.keys()) if validate_selected: selected_sourcetypes = [s for s, selected in list(sourcetype_dict.items()) if selected] diffset = set(sourcetypes) - set(selected_sourcetypes) if diffset: raise CommonException(err_code=5025, options={'sourcetypes': "/".join(diffset)}) diffset = set(selected_sourcetypes) - set(sourcetypes) if diffset: raise CommonException(err_code=5026, options={'sourcetypes': "/".join(diffset)}) for sourcetype in sourcetypes: # throw exception when wildcard in sourcetype if "*" in sourcetype: e = CommonException(err_code=5012, options={'sourcetype': sourcetype, 'search': search_str}) raise e # throw exception when search sourcetypes not in app sourcetypes if sourcetype not in app_sourcetypes: e = CommonException(err_code=5014, options={'sourcetype': sourcetype, 'search': search_str}) raise e return sourcetypes
def export_ta_project_as_file(self, action, **params): uri = scc.getMgmtUri() session = cherrypy.session.get("sessionKey") app = params.get('app', None) try: if app: migrator = AppMigrator(app, uri, session) migrator.export_project() return self.render_json({'status': 'success'}) else: raise CommonException(e_message='app is not set.', err_code=40) except CommonException as ce: logger.error('export project %s fails. %s', app, traceback.format_exc()) return self.render_json({ 'err_code': ce.get_err_code(), 'err_args': ce.get_options() }) except Exception as e: logger.error('export project %s fails. %s', app, traceback.format_exc()) return self.render_json({'err_code': 43, 'err_args': {'app': app}})
def rotate_temp_files(self): curr_time = time.time() for fpath in os.listdir(self.temp_dir): # skip the dir like __events__ if fpath.startswith("__"): continue fpath = os.path.join(self.temp_dir, fpath) last_modified = os.path.getmtime(fpath) if curr_time - last_modified >= self.time_delta: try: if os.path.isfile(fpath): os.remove(fpath) elif os.path.isdir(fpath): shutil.rmtree(fpath) except: ex = CommonException() ex.set_err_code(9001) ex.set_option('filepath', fpath) raise ex
def create_input_meta(self, datainput): self._validate_new_meta(datainput) datainputs = self._get_inputs() self._validate_input_name(datainput['name'], datainputs) self.__logger.debug("get data inputs meta from meta store:%s", logger.hide_sensitive_field(datainputs)) if self._input_exist(datainput, datainputs): e = CommonException() e.set_err_code(3011) e.set_option('name', datainput.get('name', '')) raise e data_input_meta = self.add_default_values( builder_util.add_unique_identification(datainput)) datainputs.append(data_input_meta) self.__meta_mgr.set_app_meta_data({"datainputs": datainputs}) return datainputs, data_input_meta