def update_TA_alert(self, params):
modular_alert_new = params.get("modular_alert", {})
global_settings = params.get("global_settings", {})
if not os.path.isdir(self.__current_ta_dir):
raise Exception("{} does not exist".format(self.__appname))
# todo: validate new modular alert data
modular_alerts = self._get_alert_meta()
uuid = modular_alert_new.get("uuid", None)
if not uuid:
raise Exception("uuid not found in meta {0}".format(modular_alert_new))
if not self.__alert_exist(modular_alert_new, modular_alerts, use_uuid=True):
raise Exception("Modular alert dose not exist")
modular_alert_old = [alert for alert in modular_alerts
if alert['uuid'] == uuid][0]
modular_alerts = [one_alert for one_alert in modular_alerts if one_alert['uuid'] != uuid]
if self.__alert_exist(modular_alert_new, modular_alerts):
e = CommonException()
e.set_err_code(3011)
e.set_option('name', modular_alert_new.get('short_name', ''))
raise e
modular_alerts.append(modular_alert_new)
self.__meta_mgr.set_app_meta_data({"modular_alerts": modular_alerts})
self.do_build([modular_alert_new],
global_settings,
output_dir=self.__splunk_app_dir)
common_util.reload_splunk_apps(self.__service_with_tab_context)
return modular_alerts
def check_event_count(self, sourcetype):
event_count_dict = builder_util.get_event_count(self.service)
if event_count_dict.get(sourcetype, 0) == 0:
ce = CommonException()
ce.set_err_code(4011)
ce.set_option('sourcetype', sourcetype)
raise ce
def __validate_alert_name(self, name, all_inputs_meta):
if TAAlertBuilder.INPUT_NAME_PATTERN.match(name) is None:
ce = CommonException()
ce.set_err_code(3133)
raise ce
loaded_modinputs = [
i.name for i in self.__service_with_tab_context.modular_input_kinds
]
modinputs_in_meta = [i['name'] for i in all_inputs_meta]
name_exist = name in loaded_modinputs or name in modinputs_in_meta
if name_exist:
ce = CommonException()
ce.set_err_code(3119)
ce.set_option('name', name)
raise ce
if common_util.contain_reserved_chars(name):
ce = CommonException()
ce.set_err_code(3015)
raise ce
try:
checked_module = importlib.import_module(name)
if checked_module:
ce = CommonException(e_message='Input name {} conflicts with existing python module name.'.format(
name), err_code=3136, options={'input_name': name})
raise ce
except ImportError:
# this is expected errors.
self.__logger.debug(
'input name is valid. No package named %s', name)
def update_input_meta(self, datainput_new):
self._validate_exist_meta(datainput_new)
datainputs = self._get_inputs()
uuid = datainput_new.get("uuid", None)
if not uuid:
raise Exception("uuid not found in meta {0}".format(datainput_new))
if not self._input_exist(datainput_new, datainputs, use_uuid=True):
raise Exception("Input dose not exist")
self._validate_input_name(datainput_new['name'], datainputs, uuid)
datainput_old = [
oneinput for oneinput in datainputs if oneinput['uuid'] == uuid
][0]
datainputs = list(
[oneinput for oneinput in datainputs if oneinput['uuid'] != uuid])
if self._input_exist(datainput_new, datainputs):
e = CommonException()
e.set_err_code(3011)
e.set_option('name', datainput_new.get('name', ''))
raise e
data_input_meta_new = self.add_default_values(datainput_new)
datainputs.append(data_input_meta_new)
self.__meta_mgr.set_app_meta_data({"datainputs": datainputs})
return datainputs, datainput_old, data_input_meta_new
def _create_sourcetype_stanzas(self, datainput):
sourcetype = datainput.get('sourcetype', None)
if sourcetype is None:
e = CommonException()
e.set_err_code(3116)
e.set_option('name', datainput.get('name'))
raise e
st_builder = TASourcetypeBuilder(self.__appname, self.__uri,
self.__session_key)
if sourcetype in st_builder.get_all_sourcetype_names():
e = CommonException()
e.set_err_code(3010)
e.set_option('sourcetype', sourcetype)
raise e
try:
# should write the props.conf directly, ta may not be loaded yet.
st_builder.create_sourcetype(sourcetype, {
"SHOULD_LINEMERGE": "0",
"pulldown_type": "1",
"category": "Splunk App Add-on Builder"
})
except Exception as e:
self.__logger.error(
'get error when creating sourcetype stanza. %s',
traceback.format_exc())
ce = CommonException()
ce.set_err_code(3117)
ce.set_option('name', datainput.get('name', ''))
ce.set_option('msg', '{}'.format(e))
ce.set_option('sourcetype', sourcetype)
raise ce
def create_TA_input(self, datainput, reload_input):
if not os.path.isdir(self.__current_ta_dir):
e = CommonException()
e.set_err_code(3118)
e.set_option('name', datainput.get('name', ''))
raise e
global_settings = datainput.get(TAInputMetaMgr.GLOBAL_SETTING_KEY)
if TAInputMetaMgr.GLOBAL_SETTING_KEY in datainput:
del datainput[TAInputMetaMgr.GLOBAL_SETTING_KEY]
datainputs, data_input_meta = self.__input_meta_mgr.create_input_meta(
datainput)
filtered_datainputs, input_kinds = self.__input_meta_mgr.get_datainputs_and_kinds_for_conf(
datainputs)
# always refresh the declare py
self.__asset_generator.generate_import_declare()
# only generate the libs if there is no such a dir
self.__asset_generator.generate_python_libs_if_not_exist()
self._create_input_py(data_input_meta, overwrite=True, global_settings_meta=global_settings)
self._generate_inputs_conf(filtered_datainputs, input_kinds)
self._generate_inputs_conf_spec(filtered_datainputs)
# detect the single instance mode after py file is updated
# have to save the meta again
mode_changed = False
for _input in datainputs:
if _input['uuid'] == data_input_meta['uuid']:
old_mode = _input.get('is_single_instance', False)
self.detect_single_instance_mode(_input)
mode_changed = old_mode != _input.get('is_single_instance', False)
break
if mode_changed:
self.__logger.info('Set single instance mode when create input %s', data_input_meta['name'])
self.__input_meta_mgr.set_meta(datainputs)
# write global setting meta and ucc configuration content together
# at frontend, when creating or updating an input, no need to save global settings in a seperate step
ucc_resource_generated = self._ta_configuration_builder.update_global_settings(global_settings, reload_app=reload_input)
# common_util.reload_local_app(self.__service_with_tab_context, self.__appname)
# regenerate_ucc_resources will reload the app resources, no need to reload here
self._create_sourcetype_stanzas(data_input_meta)
if datainputs and len(datainputs) == 1:
# create the modinput log stanzas
# only create the stanza once, calling the conf rest costs time
log_source = data_input_util.get_input_log_source_stanza(self.__appname)
cce_log_source = data_input_util.get_cce_log_source_stanza(self.__appname)
stanza = {"SHOULD_LINEMERGE": "true", "sourcetype": self.get_input_log_sourcetype()}
conf_mgr = TabConfMgr(self.__uri, self.__session_key, self.__appname, self.__service_with_tab_context)
conf_mgr.update_conf_stanza('props', log_source, {}, stanza)
conf_mgr.update_conf_stanza('props', cce_log_source, {}, stanza)
self.__logger.debug("create the input log props stanza in app " + self.__appname)
return datainputs, data_input_meta, ucc_resource_generated
def delete_temp_dir(self, dirname, folder="."):
abs_dir = self._get_file_path(dirname, folder)
try:
if os.path.isdir(abs_dir):
shutil.rmtree(abs_dir)
except:
ex = CommonException()
ex.set_err_code(9001)
ex.set_option('filepath', abs_dir)
raise ex
def create_temp_dir(self, dirname, folder=".", delete_exist=True):
if delete_exist:
self.delete_temp_dir(dirname, folder)
try:
abs_dir = self._get_file_path(dirname, folder)
os.makedirs(abs_dir)
except:
ex = CommonException()
ex.set_err_code(9000)
ex.set_option('filepath', abs_dir)
raise ex
def get_temp_file_cont(self, filename, folder="."):
fpath = self._get_file_path(filename, folder)
if not os.path.isfile(fpath):
return None
try:
with open(fpath, "r") as f:
cont = f.read()
return cont
except:
ex = CommonException()
ex.set_err_code(9004)
ex.set_option('filepath', fpath)
raise ex
def delete_temp_file(self, filename, check_exist=False):
fpath = self._get_file_path(filename)
if check_exist and not os.path.isfile(fpath):
return False
try:
if os.path.isfile(fpath):
os.remove(fpath)
return True
except:
ex = CommonException()
ex.set_err_code(9001)
ex.set_option('filepath', fpath)
raise ex
def copy_to_temp(self, src, temp_filename, folder=".", force=True):
fpath = self._get_file_path(temp_filename, folder)
if not force and os.path.isfile(fpath):
return False
try:
directory = os.path.dirname(fpath)
if not os.path.isdir(directory):
os.mkdir(directory)
if force and os.path.isfile(fpath):
os.remove(fpath)
shutil.copy(src, fpath)
return True
except Exception as e:
ex = CommonException(e_message=str(e))
ex.set_err_code(9002)
ex.set_option('src_file', src)
ex.set_option("dst_file", fpath)
raise ex
def create_input_meta(self, datainput):
self._validate_new_meta(datainput)
datainputs = self._get_inputs()
self._validate_input_name(datainput['name'], datainputs)
self.__logger.debug("get data inputs meta from meta store:%s",
logger.hide_sensitive_field(datainputs))
if self._input_exist(datainput, datainputs):
e = CommonException()
e.set_err_code(3011)
e.set_option('name', datainput.get('name', ''))
raise e
data_input_meta = self.add_default_values(
builder_util.add_unique_identification(datainput))
datainputs.append(data_input_meta)
self.__meta_mgr.set_app_meta_data({"datainputs": datainputs})
return datainputs, data_input_meta
def get_table_format_results(self, sourcetype, delim=None):
if not delim:
meta = self.get_meta_results(sourcetype) or {}
delim = meta.get("table_results", {}).get("delim", r" ")
try:
events = self.get_events(sourcetype, batch_size=1000)
if not events:
ce = CommonException()
ce.set_err_code(4011)
ce.set_option("sourcetype", sourcetype)
raise ce
handler = TableHandler(events, delim)
return handler.get_table_results()
except InvalidRegex as e:
ex = CommonException()
ex.set_err_code(4012)
ex.set_option('regex', e.regex)
raise ex
def rotate_temp_files(self):
curr_time = time.time()
for fpath in os.listdir(self.temp_dir):
# skip the dir like __events__
if fpath.startswith("__"):
continue
fpath = os.path.join(self.temp_dir, fpath)
last_modified = os.path.getmtime(fpath)
if curr_time - last_modified >= self.time_delta:
try:
if os.path.isfile(fpath):
os.remove(fpath)
elif os.path.isdir(fpath):
shutil.rmtree(fpath)
except:
ex = CommonException()
ex.set_err_code(9001)
ex.set_option('filepath', fpath)
raise ex
def run(self, sourcetype, progress_func=None):
events = self.get_events(sourcetype)
if not events:
ce = CommonException()
ce.set_err_code(4011)
ce.set_option('sourcetype', sourcetype)
raise ce
loader = RegexLoader()
regex_results = loader.run(events, progress_func)
if not regex_results:
return None
self.event_mgr.load_all_values()
self.event_mgr.remove_events(sourcetype)
res = self.get_frontend_results(regex_results)
# move the events to files rather than KV store
for group in regex_results.get("groups"):
group_id = group.get("group_id")
self.event_mgr.save_events(sourcetype, group_id,
group["raw_events"])
del group["events"]
del group["raw_events"]
del group["regex"]["possible_values"]
del group["seed"]["shared_str_indexes"]
del group["seed"]["insert_str_points"]
del group["seed"]["raw_events"]
meta = self.meta_mgr.get_app_meta_data(sourcetype) or {}
meta.update({"regex_results_temp": regex_results.copy()})
try:
self.meta_mgr.update_app_meta_data({sourcetype: meta})
except UnicodeDecodeError:
ce = CommonException()
ce.set_err_code(4017)
raise ce
return res
def copy_temp_to_dest(self, temp_filename, dest, force=True):
if not force and os.path.isfile(dest):
return False
fpath = self._get_file_path(temp_filename)
if not os.path.isfile(fpath):
return False
if force and os.path.isfile(dest):
os.remove(dest)
try:
shutil.copy(fpath, dest)
return True
except Exception as e:
ex = CommonException(e_message=str(e))
ex.set_err_code(9002)
ex.set_option('src_file', fpath)
ex.set_option("dst_file", dest)
raise ex
def _validate_customized_vars(self, datainput):
var_list = datainput.get('data_inputs_options', [])
for v in var_list:
if v.get('rest_header', False) or v.get(
'type', '') in self.REST_CKPT_OPTIONS:
continue
if 'name' not in v:
self.__logger.error(
"name field is not found in customized option part:%s", v)
ce = CommonException()
ce.set_err_code(3130)
raise ce
else:
name = v['name']
t = v.get('type', '')
if t == data_input_util.CUSTOMIZED_VAR_TYPE and TAInputMetaMgr.INPUT_PROPERTY_NAME_PATTERN.match(
name
) is None and name != TAInputMetaMgr.GLOBAL_ACCOUNT_NAME:
self.__logger.error(
"customized variable name:%s is not valid.", name)
ce = CommonException(err_code=3131)
ce.set_option('prop_name', name)
raise ce
if name in TAInputMetaMgr.INPUT_RESERVED_PROPERTIES:
self.__logger.error(
"customized variable name:%s is in the reseved list.",
name)
ce = CommonException()
ce.set_err_code(3132)
ce.set_option('prop_name', name)
raise ce
if t == data_input_util.CUSTOMIZED_VAR_TYPE:
for k in TAInputMetaMgr.CUSTOMIZED_VAR_REQUIRED_KEYS:
if k not in v:
emsg = 'Required field {} not found in customized variable.'.format(
k)
self.__logger.error(emsg)
ce = CommonException(err_code=3144,
e_message=emsg,
options={'attribute': k})
raise ce
def detect_sample_format(self, sourcetype, except_if_no_events=True):
meta = self.get_meta_results(sourcetype) or {}
if meta.get("is_parsed", False):
return {
"data_format": meta.get("data_format", None),
"table_results": {
"delim": meta.get("table_results", {}).get("delim", None)
},
"is_parsed": True,
}
events = self.get_events(sourcetype, 50)
if len(events) < 10:
_LOGGER.warn(
"Skip sample format detection since the event count < 10 for sourcetype {}"
.format(sourcetype))
return {}
from field_extraction_builder.data_format.format_handler_base import DataFormatHandler
handler = DataFormatHandler(events)
formats = handler.get_format()
if not formats:
if except_if_no_events:
ex = CommonException()
ex.set_err_code(4011)
ex.set_option('sourcetype', sourcetype)
raise ex
else:
return {}
table_delim = formats.get("table_delim")
meta["data_format"] = formats.get("data_format")
if table_delim:
meta["table_results"] = {"delim": table_delim}
self.update_meta_results(sourcetype, meta)
return meta
def _validate_basic_input_meta(self, meta, is_update=False):
if common_util.contain_reserved_chars(meta.get('name', '')):
e = CommonException()
e.set_err_code(3015)
raise e
sourcetype = meta.get('sourcetype', None)
if sourcetype is None:
e = CommonException()
e.set_err_code(3116)
e.set_option('name', meta.get('name'))
raise e
if not is_update:
st_builder = TASourcetypeBuilder(self.__appname, self.__uri,
self.__session_key)
if sourcetype in st_builder.get_all_sourcetype_names():
e = CommonException()
e.set_err_code(3010)
e.set_option('sourcetype', sourcetype)
raise e
# splunk may not restart yet. TA is not load. so, use the conf mgr
# with tab context
sourcetype_existed = self.__conf_mgr_with_tab_context.get_conf(
"props").get_all()
if sourcetype in sourcetype_existed:
self.__logger.error(
"Error when validating meta: %s, Error: sourcetype exists.",
logger.hide_sensitive_field(meta))
e = CommonException()
e.set_err_code(3012)
e.set_option('sourcetype', sourcetype)
raise e
def fetch_input_code(self, datainput):
'''
this function tries to get the input python code based on input meta.
The data input may not be created yet. will generate python file content
'''
data_input_meta = self.__input_meta_mgr.add_default_values(
builder_util.add_unique_identification(datainput))
try:
# when testing the code, the input may not be saved yet.
self.__asset_generator.generate_python_libs_if_not_exist()
self.__asset_generator.generate_import_declare_if_not_exist()
data_input_meta['code'] = self._get_data_input_code(
data_input_meta)
return data_input_meta
except Exception:
self.__logger.error("Fail to get the code. datainput:%s",
logger.hide_sensitive_field(data_input_meta))
ce = CommonException()
ce.set_err_code(3129)
ce.set_option('input_name', data_input_meta['name'])
raise ce
def create_TA_alert(self, params):
modular_alert = params.get("modular_alert", {})
global_settings = params.get("global_settings", {})
modular_alerts = self._get_alert_meta()
if self.__alert_exist(modular_alert, modular_alerts):
e = CommonException()
e.set_err_code(10000)
e.set_option('name', modular_alert.get('short_name', ''))
raise e
modular_alert_meta = builder_util.add_unique_identification(
modular_alert)
modular_alerts.append(modular_alert_meta)
self.__meta_mgr.set_app_meta_data({"modular_alerts": modular_alerts})
self.do_build([modular_alert],
global_settings,
output_dir=self.__splunk_app_dir)
self.create_alert_log_stanza(modular_alert['short_name'])
common_util.reload_splunk_apps(self.__service_with_tab_context)
return modular_alert_meta.get("uuid")
def _write_temp_file(self,
filename,
filecont,
folder=".",
check_exist=False,
mode="w"):
fpath = self._get_file_path(filename, folder)
if check_exist and not os.path.isfile(fpath):
return False
try:
directory = os.path.dirname(fpath)
if not os.path.isdir(directory):
os.makedirs(directory)
with open(fpath, mode) as f:
f.write(filecont)
return True
except:
ex = CommonException()
ex.set_err_code(9003)
ex.set_option('filepath', fpath)
raise ex
def get_kv_format_results(self, sourcetype, delim_pair, delim_kv, regex):
try:
events = self.get_events(sourcetype, batch_size=1000)
if not events:
ce = CommonException()
ce.set_err_code(4011)
ce.set_option("sourcetype", sourcetype)
raise ce
handler = KVHandler(events, delim_pair, delim_kv, regex)
return handler.get_kv_results()
except InvalidRegex as e:
ex = CommonException()
ex.set_err_code(4012)
ex.set_option('regex', e.regex)
raise ex
except CaptureGroupCountError as e:
ex = CommonException()
ex.set_err_code(4013)
ex.set_option('regex', e.regex)
raise ex
def _validate_new_meta(self, meta, is_update=False):
for k in self.required_meta_keys:
if k not in meta:
ce = CommonException(
e_message='{0} not found in meta {1}'.format(k, meta),
err_code=3137,
options={'property_name': k})
raise ce
self._validate_basic_input_meta(meta, is_update)
self._validate_customized_vars(meta)
input_type = meta['type']
if input_type == 'customized':
options = meta.get('data_inputs_options', [])
options_existed = {}
for option in options:
if 'name' not in option:
raise Exception(
'name is not found for input option {0}'.format(
option))
else:
if option['name'] in options_existed:
e = CommonException()
e.set_err_code(3013)
e.set_option('name', option['name'])
raise e
else:
options_existed[option['name']] = True
elif input_type == 'rest':
options = meta.get('data_inputs_options', [])
url_option_found = False
method_option_found = False
for option in options:
if option.get(
'type', ''
) == data_input_util.CUSTOMIZED_VAR_TYPE or option.get(
'type', '') in self.REST_CKPT_OPTIONS:
continue
if 'name' not in option:
raise Exception(
'name is not found for input option {0}'.format(
option))
if 'value' not in option:
raise Exception(
'value is not found for input option {0}'.format(
option))
if 'rest_header' not in option:
raise Exception(
'rest_handler is not found for input option {0}'.
format(option))
if option['name'] == '_rest_api_url':
url_option_found = True
if option['name'] == '_rest_api_method':
method_option_found = True
if url_option_found is False:
raise Exception('_rest_api_url option not found.')
if method_option_found is False:
raise Exception('_rest_api_method option not found.')
elif input_type == 'command':
options = meta.get('data_inputs_options', [])
command_found = False
for option in options:
if option.get('type',
'') == data_input_util.CUSTOMIZED_VAR_TYPE:
continue
if 'name' not in option:
raise Exception(
'name is not found for input option {0}'.format(
option))
if 'value' not in option:
raise Exception(
'value is not found for input option {0}'.format(
option))
if option['name'] == '_command':
command_found = True
break
if command_found is False:
raise Exception('command option is not found.')
else:
raise Exception('Invalid data input type: {0}'.format(input_type))
def _validate_input_name(self, name, all_inputs_meta, uuid=None):
should_check_module_name = True
if uuid:
for _input in all_inputs_meta:
if _input['uuid'] == uuid:
# if it is update request, and no name changes, do not check
# the module name, because there is a input python file
# there
should_check_module_name = (_input['name'] != name)
if not name:
ce = CommonException(
e_message='data input name should not be empty.',
err_code=3137,
options={'property_name': 'name'})
raise ce
if name in TAInputMetaMgr.INPUT_RESERVED_NAMES:
ce = CommonException(e_message='data input name is reserved.',
err_code=3145,
options={'input_name': name})
raise ce
if TAInputMetaMgr.INPUT_NAME_PATTERN.match(name) is None:
ce = CommonException()
ce.set_err_code(3133)
raise ce
if common_util.contain_reserved_chars(name):
ce = CommonException()
ce.set_err_code(3015)
raise ce
# for name conflicts, should tell if this is a update or an create
need_check_input_name_conflict = True
if uuid:
input_meta = [
meta for meta in all_inputs_meta if meta['uuid'] == uuid
]
if not input_meta:
ce = CommonException(
e_message='No input with uuid {}'.format(uuid),
err_code=3138)
self.__logger.error(
'No input with uuid %s, validate input name fails.', uuid)
raise ce
input_meta = input_meta[0]
if input_meta['name'] == name:
# It is fine. No rename for input update
need_check_input_name_conflict = False
if need_check_input_name_conflict:
# need to check name conflict when creating an input and renaming
# the input
loaded_modinputs = [
i.name
for i in self.__service_with_tab_context.modular_input_kinds
]
modinputs_in_meta = [i['name'] for i in all_inputs_meta]
name_exist = name in loaded_modinputs or name in modinputs_in_meta
if name_exist:
ce = CommonException()
ce.set_err_code(3119)
ce.set_option('name', name)
raise ce
try:
if should_check_module_name:
checked_module = importlib.import_module(name)
if checked_module:
ce = CommonException(
e_message=
'Input name {} conflicts with existing python module name.'
.format(name),
err_code=3136,
options={'input_name': name})
raise ce
except ImportError:
# this is expected errors.
self.__logger.debug('input name is valid. No package named %s',
name)
# check if the input name conflicts with alert name
if self.__alert_builder:
self.__logger.debug('validate input name:%s', name)
if self.__alert_builder.is_alert_exist(name):
ce = CommonException(
e_message=
'Input name {} conflicts with an existing alert action name.'
.format(name),
err_code=3143,
options={'input_name': name})
raise ce