def _create_input_py(self, datainput, overwrite=False, global_settings_meta=None):
'''
@param: overwrite -- whether to overwrite the modinput python file
with the code from meta data.
'''
datainput_name = datainput['name']
# create the mod input py file
self._create_modular_input_py(datainput, global_settings_meta)
self._create_input_module_py(datainput)
if overwrite and 'code' in datainput:
targetfile = os.path.join(
builder_util.get_target_folder(self.__current_ta_dir, "bin"),
"{}.py".format(
self.get_input_module_file_name(datainput_name)))
with open(targetfile, 'w') as f:
f.write(datainput['code'])
self.__logger.info(
'Overwrite modular input file %s with code in meta.',
targetfile)
if 'code' in datainput:
# do not put the code in meta any more!
# the code logic is in the file
# then, use can use IDE to edit the pyfile directly
del datainput['code']
def _get_data_input_code(self, datainput):
targetfile = os.path.join(
builder_util.get_target_folder(self.__current_ta_dir, "bin"),
"{}.py".format(self.get_input_module_file_name(datainput['name'])))
if os.path.isfile(targetfile):
with open(targetfile, 'r') as tf:
return tf.read()
else:
return self.generate_input_module_content(datainput)
def _generate_inputs_conf_spec(self, datainputs):
filename = os.path.join(self.__resource_dir, "README",
"inputs.conf.spec.template")
temp = Template(filename=filename)
tran = temp.render(datainputs=datainputs)
targetfile = os.path.join(
builder_util.get_target_folder(self.__current_ta_dir, "README"),
"inputs.conf.spec")
with open(targetfile, "w+") as write_file:
write_file.write(tran.strip())
def _generate_inputs_conf(self, datainputs, input_kinds):
targetfile = os.path.join(
builder_util.get_target_folder(self.__current_ta_dir, "local"),
"inputs.conf")
backup_conf = os.path.join(builder_util.get_target_folder(self.__current_ta_dir, "local"),
"inputs.conf.bak")
new_inputs_conf_content = None
try:
if os.path.isfile(backup_conf):
os.remove(backup_conf)
if os.path.isfile(targetfile):
os.rename(targetfile, backup_conf)
filename = os.path.join(self.__resource_dir, "local",
"inputs.conf.template")
temp = Template(filename=filename)
new_inputs_conf_content = temp.render(datainputs=datainputs)
with open(targetfile, "w+") as write_file:
write_file.write(new_inputs_conf_content.strip())
inputs_conf_parser = conf_parser.TABConfigParser()
inputs_conf_parser.read(targetfile)
bak_conf_parser = conf_parser.TABConfigParser()
bak_conf_parser.read(backup_conf)
for stanza, props in list(bak_conf_parser.item_dict().items()):
stanza_parts = stanza.split('://')
if len(stanza_parts) == 2 and stanza_parts[0] in input_kinds:
inputs_conf_parser.add_section(stanza)
for k, v in list(props.items()):
inputs_conf_parser.set(stanza, k, v)
with open(targetfile, 'w') as conf_fp:
inputs_conf_parser.write(conf_fp)
# clean up inputs.conf.bak
if os.path.isfile(backup_conf):
os.remove(backup_conf)
except Exception as e:
self.__logger.error('Fail to generate inputs.conf. The new inputs.conf content:%s, %s', new_inputs_conf_content, traceback.format_exc())
# revert the conf
if os.path.isfile(backup_conf):
if os.path.isfile(targetfile):
os.remove(targetfile)
os.rename(backup_conf, targetfile)
raise e
def upgrade_from_1_0_1_to_1_1_0(self):
datainputs = self.get_all_TA_inputs()
for datainput in datainputs:
input_name = datainput['name']
target_file = os.path.join(
builder_util.get_target_folder(self.__current_ta_dir, 'bin'),
"{}.py".format(input_name))
self.upgrade_modular_input_from_1_0_1_to_1_1_0(target_file,
datainput)
self.__logger.info("Upgrade modinput %s from 1.0.1 to 1.1.0.",
input_name)
if datainputs:
self.__input_meta_mgr.set_meta(datainputs)
def _create_modular_input_py(self, datainput, global_settings_meta=None):
'''
create input_name.py
'''
bin_folder = builder_util.get_target_folder(self.__current_ta_dir, "bin")
if datainput.get('type') == data_input_util.INPUT_METHOD_REST:
cc_input_builder = CloudConnectDataInputBuilder(datainput, global_settings_meta or {})
cc_input_builder.save_cc_input(self.__resource_dir, bin_folder)
else:
datainput_name = datainput['name']
tran = self.generate_python_modinput_content(datainput)
targetfile = os.path.join(
bin_folder,
"{}.py".format(datainput_name))
with open(targetfile, "w+") as write_file:
write_file.write(tran)
def _delete_input_py(self, datainput):
datainput_name = datainput['name']
bin_folder = builder_util.get_target_folder(self.__current_ta_dir,
"bin")
file_names = [
escape_character(datainput_name),
self.get_input_module_file_name(datainput_name)
]
file_paths = [
os.path.join(bin_folder, '{}.py'.format(f)) for f in file_names
]
for targetfile in file_paths:
if os.path.exists(targetfile):
os.remove(targetfile)
else:
self.__logger.error("mod input module file not found: %s",
targetfile)
def _update_input_py(self, datainput_old, datainput_new, global_settings_meta=None):
'''
update the datainput. Currently, the old input and new input
should be of the same type. But the input name will be changed.
'''
input_type = datainput_new['type']
datainput_name_new = datainput_new['name']
datainput_name_old = datainput_old['name']
bin_folder = builder_util.get_target_folder(self.__current_ta_dir,
"bin")
if input_type in [
data_input_util.INPUT_METHOD_CUSTOMIZED, data_input_util.INPUT_METHOD_REST,
data_input_util.INPUT_METHOD_CMD
]:
# just remove the old one, generate new input py file
self._create_input_py(datainput_new, overwrite=True, global_settings_meta=global_settings_meta)
else:
raise Exception('unknown data input type.')
# remove the old modinput in the end. After we finish all the job
# both name.py and module.py should exist for all input types
# TODO: maybe we should think about how to make the update action as atomic
if datainput_name_new != datainput_name_old:
if input_type == data_input_util.INPUT_METHOD_REST:
# cc data input is special, should remove the json and py
cc_input_builder = CloudConnectDataInputBuilder(datainput_old, global_settings_meta)
cc_input_builder.delete_cc_input(bin_folder)
else:
targetfile_old = os.path.join(bin_folder,
"{}.py".format(datainput_name_old))
if os.path.isfile(targetfile_old):
os.remove(targetfile_old)
else:
self.__logger.error(
"Can not find the old modular input python file:%s",
targetfile_old)
targetfile_old = os.path.join(bin_folder, "{}.py".format(
self.get_input_module_file_name(datainput_name_old)))
if os.path.isfile(targetfile_old):
os.remove(targetfile_old)
else:
self.__logger.error(
"Can not find the old modular input python file: %s",
targetfile_old)
def _on_rename_input(self, old_input_meta, new_input_meta):
old_name = old_input_meta['name']
new_name = new_input_meta['name']
parser = conf_parser.TABConfigParser()
input_conf = os.path.join(
builder_util.get_target_folder(self.__current_ta_dir, "local"),
"inputs.conf")
parser.read(input_conf)
for section in parser.sections():
stanza_names = section.split('://')
if len(stanza_names) == 2 and stanza_names[0].strip() == old_name:
# copy all the old stanzas into the new name stanzas
new_section = new_name + '://' + stanza_names[1]
parser.add_section(new_section)
for item in parser.items(section):
parser.set(new_section, item[0], item[1])
with open(input_conf, 'w') as fp:
parser.write(fp)
def _create_input_module_py(self, datainput):
'''
create customized_input_module.py or cmd_input_module.py
For rest, cc data input is special
'''
input_type = datainput['type']
if input_type == data_input_util.INPUT_METHOD_REST:
return
datainput_name = datainput['name']
targetfile = os.path.join(
builder_util.get_target_folder(self.__current_ta_dir, "bin"),
"{}.py".format(self.get_input_module_file_name(datainput_name)))
if input_type == data_input_util.INPUT_METHOD_CUSTOMIZED and os.path.isfile(
targetfile):
self.__logger.info(
"customized modular input module %s exists. Do not regen the python file anymore.",
targetfile)
else:
tran = self.generate_input_module_content(datainput)
with open(targetfile, "w+") as write_file:
write_file.write(tran)
def dryrun_modinput_code(self, datainput):
'''
The dryrun returns the following structure as the results
{
'status': 'success/fail',
// a list of results, each result is a python dict
'results': [event1, event2, event3],
error: 'error messages'
}
'''
# TODO: dryrun is sync call now. If the modinput is long running,
# should make it as async
dryrun_result = None
datainput = self.__input_meta_mgr.add_default_values(datainput)
self.__input_meta_mgr.validate_new_meta(datainput, 'uuid' in datainput)
# if it is cc data input, should process the meta
if datainput.get('type') == data_input_util.INPUT_METHOD_REST:
datainput = data_input_util.process_cc_data_input_meta(datainput)
if 'test_id' not in datainput:
ce = CommonException(
e_message='dry run job id not found.',
err_code=3142,
options={'input_name': datainput['name']})
raise ce
datainput['server_uri'] = self.__uri
datainput['session_key'] = self.__session_key
datainput['checkpoint_dir'] = common_util.make_splunk_path([
'var', 'lib', 'splunk', 'modinputs', datainput['name'],
'test_' + datainput['name']
])
test_id = datainput['test_id']
bin_dir = builder_util.get_target_folder(self.__current_ta_dir, "bin")
cc_input_builder = None
if datainput.get('type') == data_input_util.INPUT_METHOD_REST:
cc_input_builder = CloudConnectDataInputBuilder(datainput, datainput.get('global_settings', {}))
datainput['cc_json_file'] = cc_input_builder.get_cc_json_file_path(
bin_dir, True)
# generate {mod input}.py
if datainput[
'type'] == data_input_util.INPUT_METHOD_CUSTOMIZED and 'code' not in datainput:
raise CommonException(
e_message='No code in data input:{}'.format(datainput['name']),
err_code=3141,
options={'input_name': datainput['name']})
elif datainput['type'] in [
data_input_util.INPUT_METHOD_CMD, data_input_util.INPUT_METHOD_REST
]:
datainput['code'] = self.generate_input_module_content(datainput)
test_input_module = self.get_input_module_file_name(datainput['name'] +
test_id)
targetfile = os.path.join(bin_dir, '{}.py'.format(test_input_module))
with open(targetfile, 'w') as f:
f.write(datainput['code'])
datainput['input_module_file'] = targetfile
# generate input.py
modinput_content = self.generate_python_modinput_content(datainput)
# Important: should replace the base input module name, since it is
# changed!
old_import = TAInputMetaMgr.BASE_INPUT_MODULE_IMPORT.format(
self.get_input_module_file_name(datainput['name']))
new_import = TAInputMetaMgr.BASE_INPUT_MODULE_IMPORT.format(
test_input_module)
modinput_content = modinput_content.replace(old_import, new_import)
datainput['code'] = modinput_content
datainput['modinput_file'] = os.path.join(
bin_dir, '{}.py'.format(datainput['name'] + test_id))
try:
self.__asset_generator.generate_import_declare_if_not_exist()
self.__asset_generator.generate_python_libs_if_not_exist()
# generate cc json
if cc_input_builder:
cc_input_builder.generate_cc_input_json(bin_dir, True)
# Do not open this log in production env. It may log some user credential: TAB-2191
# self.__logger.debug("begine to test data input %s", logger.hide_sensitive_field(datainput))
code_runner = runner.CodeRunner(self.__appname, datainput)
return_code, stdout_buffer, stderr_buffer = code_runner.run()
if cc_input_builder:
dryrun_result = cc_input_builder.process_cc_input_dry_run_result(
return_code, stdout_buffer, stderr_buffer)
else:
if return_code == 0:
# success
raw_events = data_input_util.parse_MI_output_xml(
stdout_buffer)
dryrun_result = {
'status': 'success',
'results': raw_events
}
else:
dryrun_result = {
'status': 'fail',
'results': [],
'error': stderr_buffer
}
except Exception as e:
self.__logger.error('Error happens when dry run input:%s. \n %s',
datainput['modinput_file'],
traceback.format_exc())
raise e
finally:
# clean up the base modinput python files. The modinput file will
# be cleaned in code runner
if 'input_module_file' in datainput and os.path.isfile(datainput[
'input_module_file']):
os.remove(datainput['input_module_file'])
self.__logger.debug(
'remove input module file:%s after testing.',
datainput['input_module_file'])
self.__asset_generator.cleanup_ta_bin_folder()
for f in os.listdir(bin_dir):
if f.endswith('.pyc'):
self.__logger.debug('remove %s after testing.', f)
os.remove(os.path.join(bin_dir, f))
if cc_input_builder:
cc_json = cc_input_builder.get_cc_json_file_path(bin_dir, True)
if os.path.isfile(cc_json):
self.__logger.debug('delete dryrun cc json:%s.', cc_json)
os.remove(cc_json)
return dryrun_result
