def get_unstructured_data_status(self, action, **params):
session = cherrypy.session.get("sessionKey")
splunk_uri = scc.getMgmtUri()
app_name = params['app_name']
sourcetype = params['sourcetype']
try:
tabuilder = TABuilder(app_name, splunk_uri, session)
results = tabuilder.get_unstructured_data_status(sourcetype)
if results.get("error"):
tabuilder.cancel_parse_unstructured_data(sourcetype)
return self.render_json(results.get("error"))
return self.render_json({"data": results})
except CommonException as e:
tabuilder.cancel_parse_unstructured_data(sourcetype)
logger.error('Get CommonException when getting auto extraction results. meta:%s, error:%s',
params, traceback.format_exc())
return self.render_json({'err_code': e.get_err_code(),
'err_args': e.get_options()})
except Exception as e:
tabuilder.cancel_parse_unstructured_data(sourcetype)
logger.error("Cannot get extractions for sourcetype %s. error: %s",
sourcetype, traceback.format_exc())
raise e
def create_sourcetype(self, action, **params):
session = cherrypy.session.get("sessionKey")
splunk_uri = scc.getMgmtUri()
app_name = params['app_name']
sourcetype = params.get('sourcetype', '').strip()
if not sourcetype:
return self.render_json({"err_code": 8005})
key_values = json.loads(params['key_values'])
try:
tabuilder = TABuilder(app_name, splunk_uri, session)
if not SOURCETYPE_NAMING_REGEX.match(sourcetype):
return self.render_json({
'err_code': 8016,
'err_args': {
"sourcetype": sourcetype
},
})
if sourcetype in builder_constant.RESERVED_SOURCETYPES:
return self.render_json({
'err_code': 8012,
'err_args': {
"sourcetype": sourcetype
},
})
# check if the sourcetype exists in splunk
import_sourcetypes = tabuilder.get_sourcetypes_from_index()
existing_sourcetypes = [
list(st.values())[0] for st in import_sourcetypes
]
if sourcetype in existing_sourcetypes:
return self.render_json({
'err_code': 8002,
'err_args': {
"sourcetype": sourcetype
},
})
success = tabuilder.create_sourcetype(sourcetype, key_values)
if success:
return self.render_json({'status': 'success'})
else:
ret = {
'err_code': 8002,
'err_args': {
"sourcetype": sourcetype
},
}
return self.render_json(ret)
except Exception as e:
logger.error("Cannot create sourcetype %s. error: %s", sourcetype,
traceback.format_exc())
raise e
def get_app_sourcetype_names(self, action, **params):
session = cherrypy.session.get("sessionKey")
splunk_uri = scc.getMgmtUri()
app_name = controller_util.get_current_ta_project()
try:
tabuilder = TABuilder(app_name, splunk_uri, session)
sourcetypes = tabuilder.get_app_sourcetypes()
return self.render_json([{'name': s} for s in sourcetypes])
except Exception as e:
logger.error("Cannot get basic info. error: %s",
traceback.format_exc())
raise e
def get_kv_templates(self, action, **params):
session = cherrypy.session.get("sessionKey")
splunk_uri = scc.getMgmtUri()
app_name = params['app_name']
try:
tabuilder = TABuilder(app_name, splunk_uri, session)
res = tabuilder.get_kv_templates()
return self.render_json({"data": res})
except Exception as e:
logger.error("Cannot get KV templates. error: %s",
traceback.format_exc())
raise e
def get_indexed_sourcetypes(self, action, **params):
session = cherrypy.session.get("sessionKey")
splunk_uri = scc.getMgmtUri()
app_name = controller_util.get_current_ta_project()
try:
tabuilder = TABuilder(app_name, splunk_uri, session)
res = tabuilder.get_import_sourcetype()
return self.render_json({'indexed_sourcetypes': res})
except Exception as e:
logger.error("Cannot get sourcetype names from index. error: %s",
traceback.format_exc())
raise e
def delete_extraction(self, action, **params):
session = cherrypy.session.get("sessionKey")
splunk_uri = scc.getMgmtUri()
app_name = params['app_name']
sourcetype = params['sourcetype']
try:
tabuilder = TABuilder(app_name, splunk_uri, session)
res = tabuilder.delete_extraction(sourcetype)
return self.render_json({"data": res})
except Exception as e:
logger.error("Cannot delete extractions for sourcetype %s. error: %s",
sourcetype, traceback.format_exc())
raise e
def cancel_regex_generation(self, action, **params):
session = cherrypy.session.get("sessionKey")
splunk_uri = scc.getMgmtUri()
app_name = params['app_name']
sourcetype = params['sourcetype']
try:
tabuilder = TABuilder(app_name, splunk_uri, session)
tabuilder.cancel_extraction_process()
return self.render_json({"data": {"successful": True}})
except Exception as e:
logger.error("Cannot cancel extraction process for sourcetype %s. error: %s",
sourcetype, traceback.format_exc())
raise e
def ta_summary(self, action, **params):
app_name = params['app_name']
uri = scc.getMgmtUri()
session = cherrypy.session.get("sessionKey")
try:
result = {'app_name': app_name}
tabuilder = TABuilder(app_name, uri, session)
# sourcetype_basic and input_basic should be consistent
sourcetype_basic = tabuilder.get_sourcetype_basic_info() or {}
input_basic = tabuilder.get_inputs_basic_info() or {}
cim_basic = tabuilder.get_TA_cim_basic_info() or {}
logger.debug("sourcetype basic:%s", sourcetype_basic)
logger.debug("input basic:%s", input_basic)
logger.debug("cim basic:%s", input_basic)
sourcetype_summary = []
for k, v in list(sourcetype_basic.items()):
item = {"sourcetype": k}
st_meta = v.get('metadata', {})
name = st_meta.get('data_input_name', None)
if name:
item['input_name'] = name
item['input_type'] = input_basic[k]['data_input_type']
item['has_field_extraction'] = st_meta.get('is_parsed', False)
item['data_format'] = st_meta.get('data_format', None)
if k in cim_basic and (cim_basic[k]['eval_count'] +
cim_basic[k]['alias_count']) > 0:
item['has_cimmapping'] = True
else:
item['has_cimmapping'] = False
item['event_count'] = st_meta.get('event_count', 0)
sourcetype_summary.append(item)
result['sourcetype_count'] = len(sourcetype_basic)
result['input_count'] = len(input_basic)
result['sourcetype_summary'] = sourcetype_summary
return self.render_json(result)
except builder_exception.CommonException as ce:
logger.error("Can not get TA summary info. error:%s",
traceback.format_exc())
return self.render_json({
'err_code': ce.get_err_code(),
'err_args': ce.get_options()
})
except Exception as e:
logger.error("Cannot get TA summary info. error: %s",
traceback.format_exc())
raise e
def save_xml_format_results(self, action, **params):
session = cherrypy.session.get("sessionKey")
splunk_uri = scc.getMgmtUri()
cl = cherrypy.request.headers["Content-Length"]
raw_body = cherrypy.request.body.read(int(cl))
params = json.loads(raw_body) # get param from json body
app_name = params['app_name']
sourcetype = params['sourcetype']
try:
tabuilder = TABuilder(app_name, splunk_uri, session)
res = tabuilder.save_xml_format_results(sourcetype)
return self.render_json({"data": res})
except Exception as e:
logger.error("Cannot save XML results for sourcetype %s. error: %s",
sourcetype, traceback.format_exc())
raise e
def load_kv_format_results(self, action, **params):
session = cherrypy.session.get("sessionKey")
splunk_uri = scc.getMgmtUri()
app_name = params['app_name']
sourcetype = params['sourcetype']
try:
tabuilder = TABuilder(app_name, splunk_uri, session)
res = tabuilder.load_kv_format_results(sourcetype)
return self.render_json({"data": res})
except CommonException as e:
logger.error('Get CommonException when getting kv format results. meta:%s, error:%s',
params, traceback.format_exc())
return self.render_json({'err_code': e.get_err_code(),
'err_args': e.get_options()})
except Exception as e:
logger.error("Cannot get KV results for sourcetype %s. error: %s",
sourcetype, traceback.format_exc())
raise e
def get_events(self, action, **params):
session = cherrypy.session.get("sessionKey")
splunk_uri = scc.getMgmtUri()
app_name = params['app_name']
sourcetype = params['sourcetype']
batch_size = params.get("batch_size", 1000)
try:
tabuilder = TABuilder(app_name, splunk_uri, session)
events = tabuilder.get_events(sourcetype, batch_size)
if not events:
return self.render_json({
"err_code": 4011,
"err_args": {"sourcetype": sourcetype}
})
return self.render_json({"data": [e.get("_raw") for e in events]})
except Exception as e:
logger.error("Cannot get events for sourcetype %s. error: %s",
sourcetype, traceback.format_exc())
raise e
def load_unstructured_data_result(self, action, **params):
session = cherrypy.session.get("sessionKey")
splunk_uri = scc.getMgmtUri()
app_name = params['app_name']
sourcetype = params['sourcetype']
try:
tabuilder = TABuilder(app_name, splunk_uri, session)
results = tabuilder.load_unstructured_data_result(sourcetype)
if not results:
ret = {
'err_code': 4006,
'err_args': {"sourcetype": sourcetype},
}
return self.render_json(ret)
return self.render_json({"data": results})
except Exception as e:
logger.error("Cannot load extractions for sourcetype %s. error: %s",
sourcetype, traceback.format_exc())
raise e
def generate_link(self, action, **params):
uri = scc.getMgmtUri()
session = cherrypy.session.get("sessionKey")
appname = params[
"app_name"] or ta_project_meta.get_current_creating_app({})
tabuilder = TABuilder(appname, uri, session)
package_file_name = workspace_util.package_app(tabuilder)
return self.render_json({
"name":
os.path.basename(package_file_name),
"link":
"../../custom/splunk_app_addon-builder/app_package/file_download?app_name="
+ appname
})