def delete_app_meta_data(self, key=None, update_last_modify=True):
"""
Delete values from default namespace & app
:key: delete this key, or all the keys when it's None
:return: None, or throws Exceptions
"""
if key is None:
self.project_meta_state.delete_state(self.namespace)
_logger.debug('delete app meta data. app:%s, namespace:%s',
self.app_name, self.namespace)
else:
app_meta = self.project_meta_state.get_state(self.namespace)
if app_meta and key in app_meta:
app_meta = copy.deepcopy(app_meta)
del app_meta[key]
self.project_meta_state.update_state(self.namespace, app_meta)
if update_last_modify:
self._update_last_modify_time()
_logger.debug(
'Delete meta success. Key:%s, meta:%s, app:%s, namespace:%s',
key, logger.hide_sensitive_field(app_meta), self.app_name,
self.namespace)
else:
_logger.error(
'Delete meta fails. Key %s not found in meta %s. app:%s, namespace:%s',
key, logger.hide_sensitive_field(app_meta), self.app_name,
self.namespace)
def generate_input_module_content(self, datainput):
input_var_names = [
opt['name'] for opt in datainput.get('data_inputs_options', [])
]
if not input_var_names:
input_var_names = ['modinput_var_name']
global_vars = []
if 'global_settings' in datainput:
global_var_metas = datainput.get('global_settings', {}).get(
'customized_settings', [])
global_vars = [m['name'] for m in global_var_metas]
if not global_vars:
self._update_global_var_list()
global_vars = self.__global_vars
temp_file = self._get_module_py_template(datainput)
temp = Template(filename=temp_file)
self.__logger.debug(
"Generate the input module: modinput_vars:%s, global_vars:%s, input_meta:%s",
input_var_names, self.__global_vars, logger.hide_sensitive_field(datainput))
content = temp.render(
appname=self.__appname,
app_namespace=self.__app_namespace,
datainput=datainput,
global_vars=global_vars,
modinput_vars=input_var_names)
return content
def validate_global_setting_meta(self, meta):
try:
if self.CUSTOMIZED_SETTING_KEY in meta:
cus_setting = meta[self.CUSTOMIZED_SETTING_KEY]
for setting in cus_setting:
for k in self.CUSTOMIZED_VAR_REQUIRED_KEYS:
if k not in setting:
emsg = 'Required field {} not found in customized variable {}.'.format(
k, logger.hide_sensitive_field(setting))
self._logger.error(emsg)
raise CommonException(err_code=11001,
options={
'field':
k,
'v_name':
setting.get(
'name', 'unknown')
},
e_message=emsg)
if setting['name'] in self.RESERVED_CUSTOMIZED_VAR_NAMES:
emsg = 'Global variable name can not be {}.'.format(
setting['name'])
self._logger.error(emsg)
raise CommonException(
err_code=11006,
options={'var_name': setting['name']},
e_message=emsg)
except CommonException as ce:
raise ce
except Exception as e:
self._logger.error('validate global setting meta fails. %s',
traceback.format_exc())
raise CommonException(e_message=e.message,
err_code=11000,
options={'message': e.message})
def meta(self, new_meta):
if new_meta:
self._logger.debug('save global setting meta:%s',
logger.hide_sensitive_field(new_meta))
self._meta_mgr.set_app_meta_data({'global_settings': new_meta})
else:
self.delete_global_settings_meta()
def _validate_basic_input_meta(self, meta, is_update=False):
if common_util.contain_reserved_chars(meta.get('name', '')):
e = CommonException()
e.set_err_code(3015)
raise e
sourcetype = meta.get('sourcetype', None)
if sourcetype is None:
e = CommonException()
e.set_err_code(3116)
e.set_option('name', meta.get('name'))
raise e
if not is_update:
st_builder = TASourcetypeBuilder(self.__appname, self.__uri,
self.__session_key)
if sourcetype in st_builder.get_all_sourcetype_names():
e = CommonException()
e.set_err_code(3010)
e.set_option('sourcetype', sourcetype)
raise e
# splunk may not restart yet. TA is not load. so, use the conf mgr
# with tab context
sourcetype_existed = self.__conf_mgr_with_tab_context.get_conf(
"props").get_all()
if sourcetype in sourcetype_existed:
self.__logger.error(
"Error when validating meta: %s, Error: sourcetype exists.",
logger.hide_sensitive_field(meta))
e = CommonException()
e.set_err_code(3012)
e.set_option('sourcetype', sourcetype)
raise e
def set_customized_options(self, uuid, customized_options):
datainputs_existed = self._get_inputs()
for datainput in datainputs_existed:
if datainput.get('uuid', None) == uuid:
datainput['customized_options'] = customized_options
self.__logger.info("set customized_options for data input. %s",
logger.hide_sensitive_field(datainput))
self.__meta_mgr.set_app_meta_data({"datainputs": datainputs_existed})
def meta(self):
meta = self._meta_mgr.get_app_meta_data('global_settings')
self._logger.debug('get global setting meta from meta store: %s',
logger.hide_sensitive_field(meta))
if meta:
return meta
else:
return {}
def upgrade_from_2_0_0_to_2_1_0(self, input_upgraded):
global_settings = self.get_global_settings()
if global_settings:
self.ta_setup_clean(self._appname, self._current_app_dir)
if input_upgraded and not global_settings:
# if no global settings, but found inputs are upgraded.
# at least, need a log level part to generate UCC
global_settings = {
'log_settings': {
'log_level': 'INFO',
}
}
if not global_settings:
return False
CREDENTIAL_SETTING_KEY = ta_configuration_meta.GlobalSettingMeta.CREDENTIAL_SETTING_KEY
CUSTOMIZED_SETTING_KEY = ta_configuration_meta.GlobalSettingMeta.CUSTOMIZED_SETTING_KEY
if CREDENTIAL_SETTING_KEY in global_settings:
credentials = global_settings.get(CREDENTIAL_SETTING_KEY, {})
if isinstance(credentials, dict):
global_settings[CREDENTIAL_SETTING_KEY] = [{
'username':
key,
'password':
value.get('password')
} for key, value in list(credentials.items())]
if CUSTOMIZED_SETTING_KEY in global_settings:
customized_settings = global_settings.get(CUSTOMIZED_SETTING_KEY,
[])
for variable in customized_settings:
variable.update({'type': variable.get('format_type')})
if 'value' not in variable:
if variable['type'] == 'checkbox':
variable['value'] = variable.get('default_value', 0)
else:
variable['value'] = variable.get('default_value', '')
self._logger.info(
'regenerate the UCC resource during upgrading. meta:%s',
logger.hide_sensitive_field(global_settings))
# should update the meta before calling upgrade global settings
# in update global settings, the meta will be validated
self._global_setting_meta.meta = global_settings
latest_tab_version, latest_tab_build = upgrade_util.get_latest_tabuilder_version(
self._service_with_tab_context)
if latest_tab_version == '2.1.0':
# only regenerate ucc resources when current installation is 2.1.0
self.update_global_settings(global_settings)
return True
def set_app_meta_data(self, key_values, update_last_modify=True):
"""
Set the meta data based on default namespace & app
replace the old meta data with the key_values
:key_values: a dict to set. Raise Exception when it's not dict
:return None, or throws exception when fails.
"""
if not isinstance(key_values, dict):
raise ValueError("The input parameter should be a dict.")
# force reload cache
self.project_meta_state.get_state()
self.project_meta_state.update_state(self.namespace, key_values)
if update_last_modify:
self._update_last_modify_time()
_logger.debug("Set state. app:%s, key:%s, value:%s", self.app_name,
self.namespace, logger.hide_sensitive_field(key_values))
def rename_app(service, old_name, new_name):
"""
Static method to rename app in all namespaces
:service: an splunk service object
:old_name: old app name to rename
:new_name: new app name after rename
:return: True - rename success.
False - rename failure.
"""
builder_meta = meta_client.MetaClient(service, TA_META_COLLECTION_NAME)
old_project_meta = meta_client.MetaClient(service, old_name)
new_project_meta = meta_client.MetaClient(service, new_name)
if builder_meta.get_state(new_name):
_logger.error(
'Can not rename to app name {0}, the target app already exists.'
.format(new_name))
return False
_logger.info("begin to rename add-on from %s to %s", old_name,
new_name)
old_meta = old_project_meta.get_state()
if old_meta is None:
_logger.error(
"Rename add-on fails. Can not find any meta for add-on %s",
old_name)
return False
try:
for k, v in list(old_meta.items()):
new_project_meta.update_state(k, v)
old_project_meta.delete_state()
old_builder_meta = builder_meta.get_state(old_name)
builder_meta.update_state(new_name, old_builder_meta)
builder_meta.delete_state(old_name)
_logger.debug("update meta %s from app<%s> to app<%s>",
logger.hide_sensitive_field(old_meta), old_name,
new_name)
except Exception as e:
_logger.error("fail to rename app meta. %s",
traceback.format_exc())
return False
return True
def create_input_meta(self, datainput):
self._validate_new_meta(datainput)
datainputs = self._get_inputs()
self._validate_input_name(datainput['name'], datainputs)
self.__logger.debug("get data inputs meta from meta store:%s",
logger.hide_sensitive_field(datainputs))
if self._input_exist(datainput, datainputs):
e = CommonException()
e.set_err_code(3011)
e.set_option('name', datainput.get('name', ''))
raise e
data_input_meta = self.add_default_values(
builder_util.add_unique_identification(datainput))
datainputs.append(data_input_meta)
self.__meta_mgr.set_app_meta_data({"datainputs": datainputs})
return datainputs, data_input_meta
def read_global_configuration(self, global_setting_meta):
ucc_meta = self.get_ucc_meta(global_setting_meta)
schema = GlobalConfigSchema(ucc_meta)
global_config = GlobalConfig(
common_util.get_splunkd_uri(self._service_with_tab_context),
self._service_with_tab_context.token, schema)
accounts = global_config.configs.load().get('account', [])
settings = global_config.settings.load()
global_configuration = dict()
global_configuration['credential_settings'] = [{
"username":
account['username'],
"password":
account['password'],
"name":
account['name']
} for account in accounts]
for setting in settings.get('settings', []):
self._logger.info("setting: {}".format(
logger.hide_sensitive_field(setting)))
if setting['name'] == 'logging':
global_configuration["log_settings"] = {
"log_level": setting.get('loglevel')
}
elif setting['name'] == "proxy":
if 'disabled' in setting:
del setting['disabled']
global_configuration["proxy_settings"] = setting
else: # should be customized settings
global_configuration[
"customized_settings"] = global_setting_meta.get(
"customized_settings", [])
for customized_setting in global_configuration[
"customized_settings"]:
var_name = customized_setting['name']
if var_name in setting:
customized_setting['value'] = setting[var_name]
return global_configuration
def fetch_input_code(self, datainput):
'''
this function tries to get the input python code based on input meta.
The data input may not be created yet. will generate python file content
'''
data_input_meta = self.__input_meta_mgr.add_default_values(
builder_util.add_unique_identification(datainput))
try:
# when testing the code, the input may not be saved yet.
self.__asset_generator.generate_python_libs_if_not_exist()
self.__asset_generator.generate_import_declare_if_not_exist()
data_input_meta['code'] = self._get_data_input_code(
data_input_meta)
return data_input_meta
except Exception:
self.__logger.error("Fail to get the code. datainput:%s",
logger.hide_sensitive_field(data_input_meta))
ce = CommonException()
ce.set_err_code(3129)
ce.set_option('input_name', data_input_meta['name'])
raise ce
def write_global_configuration(self, global_setting_meta, old_meta=None):
if old_meta is None:
old_meta = self.meta
ucc_meta = self.get_ucc_meta(global_setting_meta)
schema = GlobalConfigSchema(ucc_meta)
global_config = GlobalConfig(
common_util.get_splunkd_uri(self._service_with_tab_context),
self._service_with_tab_context.token, schema)
all_payload = dict()
configs_payload = self.get_ucc_configs_content_meta(
global_setting_meta, old_meta)
if configs_payload:
all_payload.update(configs_payload)
settings_payload = self.get_ucc_settings_content_meta(
global_setting_meta, old_meta)
if settings_payload:
all_payload.update(settings_payload)
if all_payload:
self._logger.debug('Save global settings to UCC. payload: %s',
logger.hide_sensitive_field(all_payload))
global_config.save(all_payload)
def update_app_meta_data(self, key_values, update_last_modify=True):
"""
Update meta data based on default namespace & app
merge the key_values into existing meta
:key_values: a dict to set. Raise exception when it's not a dict
:return None, or throws exception when fails.
"""
if not isinstance(key_values, dict):
raise ValueError("The input parameter should be a dict.")
app_meta = self.project_meta_state.get_state(self.namespace)
if app_meta is None:
app_meta = key_values
else:
app_meta = copy.deepcopy(app_meta)
app_meta.update(key_values)
self.project_meta_state.update_state(self.namespace, app_meta)
if update_last_modify:
self._update_last_modify_time()
_logger.debug("Update state. app:%s, key: %s, value: %s",
self.app_name, self.namespace,
logger.hide_sensitive_field(app_meta))
def _run(self):
'''
return: 3 element tuple
(return_code, raw_stdout_out, raw_stderr_out)
'''
ckpt = self._ckpter.get(CKPT_NAME)
if ckpt is None:
ckpt = {}
if CKPT_KEY not in ckpt:
ckpt[CKPT_KEY] = {}
input_scheme = Template(OPTION_FILE_CONTENT).render(
server_uri=self._server_uri,
session_key=self._session_key,
checkpoint_dir=self._checkpoint_dir,
options=self._options,
interval=self._interval,
input_name=self._input_name,
sourcetype=self._sourcetype)
# runner_logger.debug('input stream:' + input_scheme)
# use python3 for test by default
if os.path.isfile(make_splunk_path(('bin', "python3"))) \
or os.path.isfile(make_splunk_path(('bin', "python3.exe"))):
cmd2 = [self._get_splunk_bin(), 'cmd', 'python3', self._file_path]
else:
cmd2 = [self._get_splunk_bin(), 'cmd', 'python', self._file_path]
# make it the same as core
cwd = "C:\Windows\system32" if platform.system() == "Windows" else '/'
# prepare the env
child_env = os.environ.copy()
child_env[AOB_TEST_FLAG] = 'true'
if self._globalsettings:
child_env[GLOBALSETTINGS] = json.dumps(self._globalsettings)
child_env[DATA_INPUTS_OPTIONS] = json.dumps(self._data_inputs_options)
runner_logger.debug("Start the test subprocess with env:%s",
logger.hide_sensitive_field({
GLOBALSETTINGS: self._globalsettings,
DATA_INPUTS_OPTIONS: self._data_inputs_options
}))
try:
child2 = subprocess.Popen(
cmd2,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
cwd=cwd,
env=child_env)
ckpt[CKPT_KEY][self._test_id] = {
'pid': child2.pid,
'app': self._app,
'input': self._input_name
}
self._ckpter.update(CKPT_NAME, ckpt)
stdout_str, stderr_str = child2.communicate(input=input_scheme.encode())
stdout_str = stdout_str.decode()
stderr_str = stderr_str.decode()
retcode = child2.returncode
del ckpt[CKPT_KEY][self._test_id]
if not has_kill_flag(CKPT_DIR, self._test_id):
# normal exist, not killed
self._ckpter.update(CKPT_NAME, ckpt)
return retcode, stdout_str, stderr_str
except subprocess.CalledProcessError as e:
runner_logger.error('Fail to execute the test process:%s. %s',
e.cmd, traceback.format_exc())
return e.returncode, '', e.output
def upgrade_from_2_0_0_to_2_1_0(self, global_setting_meta):
meta_updated = False
modular_alert_meta = self._get_alert_meta()
for meta in modular_alert_meta:
# should update the default value of checkbox parameters
# the default value and value of checkbox should be number 0 or 1
for param in meta.get('parameters', []):
if param.get('format_type') == 'checkbox':
if param.get('default_value') is not None:
default_value = 1 if common_util.is_true(param['default_value']) else 0
param['default_value'] = default_value
meta_updated = True
if param.get('value') is not None:
value = 1 if common_util.is_true(param['value']) else 0
param['value'] = value
meta_updated = True
self.__logger.info('[upgrade] modular metas:%s', modular_alert_meta)
if meta_updated:
self.__meta_mgr.set_app_meta_data({"modular_alerts": modular_alert_meta})
self.__logger.info('moduler alert meta is upgraded from 2_0_0 to 2_1_0. updated meta:%s', logger.hide_sensitive_field(modular_alert_meta))
# regen all the modular alerts
self.do_build(modular_alert_meta,
global_setting_meta,
output_dir=self.__splunk_app_dir)
