def rm_default_conf_properties(add_on_project_dir):
'''
This is the final chance to clean up the default stanza.
This is not the perfect way. The perfect way is not saving
default properties when saving the stanza
'''
conf_dir = os.path.join(add_on_project_dir, 'default')
for conf_file in os.listdir(conf_dir):
file_path = os.path.join(conf_dir, conf_file)
if conf_file.endswith('.conf') and os.path.isfile(file_path):
conf_name = conf_file[:-5]
default_stanza = get_default_stanza(conf_name)
if default_stanza:
to_be_delete = []
parser = TABConfigParser()
parser.read(file_path)
for section in parser.sections():
for item in parser.items(section):
if is_option_equals(default_stanza.get(item[0]), item[1]):
to_be_delete.append((section, item[0]))
for item in to_be_delete:
parser.remove_option(item[0], item[1])
_logger.debug('Remove option:%s section:%s in conf:%s', item[1], item[0], conf_name)
if len(to_be_delete) > 0:
with open(file_path, 'w') as fp:
parser.write(fp)
_logger.debug('Save conf file:%s', file_path)
def get_default_stanza(conf_name):
if conf_name in g_default_stanzas:
return g_default_stanzas[conf_name]
conf_file = make_splunkhome_path(['etc', 'system', 'default', conf_name + '.conf'])
if os.path.isfile(conf_file):
# parse the conf file to get the default values
p = TABConfigParser()
p.read(conf_file)
default_stanza = {}
if p.has_section('default'):
default_stanza.update(p.items('default'))
outside_stanza_lines = [l.strip() for l in p.fields_outside_stanza]
for l in outside_stanza_lines:
kv = l.split('=')
if len(kv) == 2:
key = kv[0].strip()
value = kv[1].strip()
if key and value:
default_stanza[key] = value
g_default_stanzas[conf_name] = default_stanza
else:
g_default_stanzas[conf_name] = None
_logger.info('Get default [%s] conf props: %s', conf_name, g_default_stanzas[conf_name])
return g_default_stanzas[conf_name]
def get_app_name(app_conf_filename):
name = None
if os.path.isfile(app_conf_filename):
parser = TABConfigParser()
parser.read(app_conf_filename)
if parser.has_option('package', 'id'):
name = parser.get('package', 'id')
else:
_logger.debug("File %s not found!", app_conf_filename)
return name
def _parse_version_in_app_conf(conf_file_path):
'''
:param conf_file_path: the app.conf file full path
'''
ver = None
if os.path.isfile(conf_file_path):
parser = TABConfigParser()
parser.read(conf_file_path)
if parser.has_option('launcher', 'version'):
ver = parser.get('launcher', 'version')
else:
_logger.debug("File %s not found!", conf_file_path)
return ver
def remove_alert_from_conf_file(alert, conf_file, logger):
if not alert or not conf_file:
logger.info('alert="%s", conf_file="%s"', alert, conf_file)
return
if not isinstance(alert, dict):
msg = 'alert="{}", event="alert is not a dict, don\'t remove anything form file {}"'.format(alert, conf_file)
raise aae.AlertCleaningFormatFailure(msg)
parser = TABConfigParser()
parser.read(conf_file)
conf_dict = parser.item_dict()
for stanza, key_values in conf_dict.items():
if stanza == alert[ac.SHORT_NAME] or \
stanza == alert[ac.SHORT_NAME] + "_modaction_result" or \
stanza == "eventtype=" + alert[ac.SHORT_NAME] + "_modaction_result":
logger.info('alert="%s", conf_file="%s", stanza="%s"',
alert[ac.SHORT_NAME],
conf_file, stanza)
parser.remove_section(stanza)
with open(conf_file, "w") as cf:
parser.write(cf)
def write_file(file_name, file_path, content, logger, merge="stanza_overwrite"):
logger.debug('operation="write", object="%s" object_type="file"',
file_path)
do_merge = False
if file_name.endswith('.conf') or file_name.endswith('conf.spec'):
do_merge = True
else:
logger.info('event="Will not merge file="%s", ' +
'reason="Only support conf file merge"', file_path)
if file_path:
new_file = None
if op.exists(file_path) and do_merge:
new_file = op.join(op.dirname(file_path), "new_" + file_name)
if new_file:
try:
with open(new_file, 'w+') as fhandler:
fhandler.write(content)
merge_conf_file(new_file, file_path, merge)
finally:
if op.exists(new_file):
remove(new_file)
else:
if not op.exists(op.dirname(file_path)):
makedirs(op.dirname(file_path))
with open(file_path, 'w+') as fhandler:
fhandler.write(content)
if do_merge:
# need to process the file with conf parser
parser = TABConfigParser()
parser.read(file_path)
with open(file_path, 'w') as df:
parser.write(df)
else:
sys.stdout.write("\n##################File {}##################\n".format(file_name))
sys.stdout.write(content)
def merge_conf_file(src_file, dst_file, merge_mode="stanza_overwrite"):
if not os.path.isfile(src_file):
return
if not os.path.isfile(dst_file):
return
if bn(src_file) in merge_black_list:
return
parser = TABConfigParser()
parser.read(src_file)
src_dict = parser.item_dict()
parser.read(dst_file)
dst_dict = parser.item_dict()
if merge_mode == "stanza_overwrite":
for stanza, key_values in src_dict.items():
if stanza in dst_dict:
parser.remove_section(stanza)
parser.add_section(stanza)
for k, v in key_values.items():
parser.set(stanza, k, v)
elif merge_mode == "item_overwrite":
for stanza, key_values in src_dict.items():
if stanza not in dst_dict:
parser.add_section(stanza)
for k, v in key_values.items():
if v:
parser.set(stanza, k, v)
else:
parser.remove_option(stanza, k)
else:
# overwrit the whole file
parser.read(src_file)
with open(dst_file, "w") as df:
parser.write(df)
def _load_tabuilder_conf(self):
if not self.conf_parser:
self.conf_parser = TABConfigParser()
if os.path.isfile(self.tabuilder_conf_file):
self.conf_parser.read(self.tabuilder_conf_file)
self.tabuilder_conf = self.conf_parser.item_dict()
class App(object):
"""
The splunk app object
"""
BUILDER_VERSION = 'builder_version'
BUILDER_BUILD = 'builder_build'
IS_EDITED = 'is_edited'
def __init__(self, app_name, service):
self._app_path = os.path.join(os.environ['SPLUNK_HOME'], 'etc', 'apps',
app_name)
if not os.path.isdir(self._app_path):
raise Exception("App directory {} not found.".format(
self._app_path))
self._app = app_name
self.logger = logger.get_app_instance_logger()
self.tabuilder_conf = {}
self.conf_parser = None
self._service = service
self.tabuilder_conf_file = os.path.join(self._app_path, 'default',
'addon_builder.conf')
self.logger.info("App instance '%s' is created.", self._app)
def is_powered_by_tab(self):
return os.path.isfile(self.tabuilder_conf_file)
def _load_tabuilder_conf(self):
if not self.conf_parser:
self.conf_parser = TABConfigParser()
if os.path.isfile(self.tabuilder_conf_file):
self.conf_parser.read(self.tabuilder_conf_file)
self.tabuilder_conf = self.conf_parser.item_dict()
def get_tabuilder_version(self):
"""
get the tabuilder version for this app.
"""
self._load_tabuilder_conf()
tab_version = '1.0.1' # no tab_conf, it is 1.0.1
if 'base' in self.tabuilder_conf:
base = self.tabuilder_conf['base']
tab_version = base.get(App.BUILDER_VERSION, '1.0.1')
return tab_version
def get_tabuilder_build(self):
self._load_tabuilder_conf()
return self.tabuilder_conf.get('base', {}).get(App.BUILDER_VERSION,
None)
def update_tabuilder_version(self, version, build=None):
self._load_tabuilder_conf()
if 'base' not in self.tabuilder_conf:
self.conf_parser.add_section('base')
self.conf_parser.set('base', App.BUILDER_VERSION, version)
if build:
self.conf_parser.set('base', App.BUILDER_BUILD, build)
with open(self.tabuilder_conf_file, "w") as f:
self.conf_parser.write(f)
def update_is_edited_flag(self, is_edited):
self._load_tabuilder_conf()
if 'base' not in self.tabuilder_conf:
self.conf_parser.add_section('base')
flag = '1' if is_edited else '0'
self.conf_parser.set('base', App.IS_EDITED, flag)
with open(self.tabuilder_conf_file, 'w') as f:
self.conf_parser.write(f)
def upgrade(self, ta_builder=None):
latest_tab_version, latest_tab_build = upgrade_util.get_latest_tabuilder_version(
self._service)
if not latest_tab_version:
raise Exception("tabuilder version is unknown.")
current_tab_version = self.get_tabuilder_version()
if ta_builder:
self.logger.debug(
"Try to upgrade app %s from verison %s to version %s",
self._app, current_tab_version, latest_tab_version)
ta_builder.upgrade(current_tab_version, latest_tab_version)
self.logger.debug("Upgrade app %s successfully.", self._app)
self.update_tabuilder_version(latest_tab_version, latest_tab_build)
def clean_package_dir(app_root_dir, app_name):
'''
clean up the workspace and set all the files in bin with execute permission
'''
# remove all the hidden files. splunkbase does not allow hidden files
# remove all the empty dirs
for root, dirs, files in os.walk(app_root_dir, topdown=False):
for f in [i for i in files if i.startswith('.')]:
p = os.path.join(root, f)
os.remove(p)
_logger.debug("Remove file %s", p)
for d in [i for i in dirs if i.startswith('.')]:
p = os.path.join(root, d)
shutil.rmtree(p)
_logger.debug("Remove directory %s", p)
if len(dirs + files) == 0 or len(os.listdir(root)) == 0:
# the children of root may be removed. So, we need to list it again
os.rmdir(root)
meta_files = [get_dumped_event_file_name(), get_aob_meta_file_name(app_name)]
for meta_file in meta_files:
file_path = os.path.join(app_root_dir, meta_file)
if os.path.isfile(file_path):
_logger.debug("Clean up the meta file:%s", file_path)
os.remove(file_path)
account_conf_file = global_setting_util.get_global_account_conf_file_name(app_name)
local_confs = set([
global_setting_util.get_global_settings_conf_file_name(app_name),
global_setting_util.GLOBAL_PASSWORD_CONF,
app_name + '_credential.conf', app_name + '.conf',
app_name + '_customized.conf', 'password.conf', 'passwords.conf'
])
_logger.debug("Local conf list: %s", local_confs)
files_to_clean = []
files_set_executable = []
for dir_name, subdir_list, files in os.walk(app_root_dir):
for file_name in files:
if file_name.endswith("pyc") or file_name.endswith(
"pyo") or file_name == "local.meta" or file_name == account_conf_file:
files_to_clean.append(os.path.join(dir_name, file_name))
elif os.path.split(dir_name.rstrip(os.path.sep))[
-1] == 'local' and file_name in local_confs:
files_to_clean.append(os.path.join(dir_name, file_name))
elif 'conf.orig.pre' in file_name:
# fix weird issue TAB-1506
files_to_clean.append(os.path.join(dir_name, file_name))
elif file_name.endswith(".py") or file_name.endswith(".sh"):
files_set_executable.append(os.path.join(dir_name, file_name))
for f in files_to_clean:
_logger.debug("Clean file %s", f)
os.remove(f)
for f in files_set_executable:
os.chmod(f, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
# rm the temp_certs folder: TAB-2167
temp_certs = os.path.join(app_root_dir, 'temp_certs')
if os.path.isdir(temp_certs):
shutil.rmtree(temp_certs)
# clean up the README folder, remove the empty spec files
README_DIR = os.path.join(app_root_dir, 'README')
if os.path.isdir(README_DIR):
for f in os.listdir(README_DIR):
if f.endswith('.conf.spec'):
p = TABConfigParser()
conf = os.path.join(README_DIR, f)
p.read(conf)
if p.fields_outside_stanza or p.sections():
continue
# remove the empty file
_logger.debug('remove the empty conf spec file:' + conf)
os.remove(conf)
def update_build_number(app_root_dir, build_number):
if not os.path.isdir(app_root_dir):
_logger.error("App root dir %s not found!", app_root_dir)
return
dft_conf = os.path.join(app_root_dir, "default", "app.conf")
if os.path.isfile(dft_conf):
parser = TABConfigParser()
parser.read(dft_conf)
if 'install' not in parser.sections():
parser.add_section('install')
parser.set('install', 'build', build_number)
with open(dft_conf, "w") as f:
parser.write(f)
def _set_is_configure(app_conf_file, is_configure):
parser = TABConfigParser()
parser.read(app_conf_file)
if 'install' not in parser.sections():
parser.add_section('install')
val = 1 if is_configure else 0
parser.set('install', 'is_configured', val)
with open(app_conf_file, "w") as f:
parser.write(f)
def _merge_conf_file(app_root_dir, conf_file_name):
'''
:param app_root_dir: the root directory full path
:param conf_file_name: the conf file short path, like 'app.conf'. It must endswith ".conf"
'''
if not conf_file_name.endswith('.conf'):
_logger.error('%s is not a conf file. Can not merge it.',
conf_file_name)
return
dft_conf = os.path.join(app_root_dir, "default", conf_file_name)
usr_conf = os.path.join(app_root_dir, "local", conf_file_name)
if os.path.isfile(dft_conf):
if os.path.isfile(usr_conf):
parser = TABConfigParser()
parser.read(usr_conf)
# for the inputs.conf, filter all the input instance stanzas in local conf
if usr_conf.split(os.path.sep)[-1] == 'inputs.conf':
to_be_delete_sections = [s for s in parser.sections() if len(s.split("://")) == 2]
if to_be_delete_sections:
_logger.info("Remove stanzas %s in conf %s", to_be_delete_sections, usr_conf)
for s in to_be_delete_sections:
parser.remove_section(s)
local_dict = parser.item_dict()
parser.read(dft_conf)
default_dict = parser.item_dict()
# overwrite the key values by local dict
for stanza, key_values in list(local_dict.items()):
if stanza not in default_dict:
parser.add_section(stanza)
for k, v in list(key_values.items()):
parser.set(stanza, k, v)
with open(dft_conf, "w") as conf_file:
parser.write(conf_file)
_logger.info("%s is merged to %s", usr_conf, dft_conf)
else:
_logger.debug("No need to merge. User Conf %s not found!",
usr_conf)
else:
if os.path.isfile(usr_conf):
p = TABConfigParser()
p.read(usr_conf)
if p.sections() or p.fields_outside_stanza:
shutil.copyfile(usr_conf, dft_conf)
_logger.info("copy %s to %s, because %s not found.", usr_conf, dft_conf, dft_conf)
else:
os.remove(usr_conf)
_logger.info('remove {} because it is a empty conf file.'.format(usr_conf))
return
else:
_logger.error(
"Both default conf %s and user conf %s are not found!",
dft_conf, usr_conf)
return
# if it is inputs.conf, set default disabled = 0
if (conf_file_name.split(os.path.sep)[-1]
) == 'inputs.conf' and os.path.isfile(dft_conf):
parser = TABConfigParser()
parser.read(dft_conf)
item_dict = parser.item_dict()
for section, key_values in list(item_dict.items()):
splits = section.split("://")
# it's default section
if len(splits) == 1:
parser.set(section, "disabled", 0)
with open(dft_conf, "w") as fp:
parser.write(fp)