def _updateCredentials(self, all_creds, ta_conf_mgr, backend_update):
all_origin_creds = ta_conf_mgr.all(return_acl=False)
if all_origin_creds is None:
all_origin_creds = {}
for name, settings in all_creds.iteritems():
settings[c.name] = name
if name not in all_origin_creds:
logger.info("new %s stanza", name)
ta_conf_mgr.create(settings)
else:
if not self._configChanges(settings, all_origin_creds[name]):
logger.debug("%s stanza is not changed", name)
continue
logger.info("%s stanza changes", name)
if utils.is_false(settings.get(c.removed)):
settings[c.removed] = 0
ta_conf_mgr.update(settings)
# Remove credentials
if backend_update:
return
stanzas = [
k for k, v in all_origin_creds.iteritems()
if k not in all_creds and utils.is_false(v.get(c.removed))
]
for stanza in stanzas:
logger.info("Remove %s", stanza)
ta_conf_mgr.update({c.name: stanza, c.removed: "1"})
def handleEdit(self, confInfo):
logger.info("start edit")
# all_settings = utils.escape_json_control_chars(
# self.callerArgs.data[c.all_settings][0])
all_settings = self.callerArgs.data[c.all_settings][0]
all_settings = json.loads(all_settings)
proxy_settings = all_settings[c.proxy_settings]
if utils.is_false(proxy_settings.get(c.proxy_enabled, None)):
del all_settings[c.proxy_settings]
# 1) Global settings / Proxy settings
ta_conf_mgr = ta_conf.TAConfManager(
c.myta_global_settings_conf, scc.getMgmtUri(),
self.getSessionKey())
ta_conf_mgr.set_encrypt_keys(self.encrypt_fields)
for k in (c.global_settings, c.proxy_settings):
settings = all_settings.get(k, {})
settings[c.name] = k
ta_conf_mgr.update(settings)
# 2) Credentials / Data collections
for name, conf_file in self.confs:
settings = all_settings.get(name, {})
ta_conf_mgr = ta_conf.TAConfManager(
conf_file, scc.getMgmtUri(), self.getSessionKey(),
appname=self.appName)
ta_conf_mgr.set_encrypt_keys(self.encrypt_fields)
self._updateCredentials(settings, ta_conf_mgr)
logger.info("end edit")
def _handleKafkaClusterList(self, caller_args, conf_info):
conf_mgr = conf.ConfManager(scc.getMgmtUri(), self.getSessionKey())
conf_mgr.reload_conf(c.myta_cred_conf)
all_creds = conf_mgr.all_stanzas_as_dicts(c.myta_cred_conf)
all_creds = {
k: v
for k, v in all_creds.iteritems()
if utils.is_false(v.get(c.removed))
}
# _ = caller_args[self.kafka_cluster_filter],
conf_info["kafka_clusters"].append("clusters", all_creds.keys())
def assert_creds(account_name, session_key, logger):
import splunk.clilib.cli_common as scc
stanza = {tac.aws_account: account_name}
metas = {tac.server_uri: scc.getMgmtUri(), tac.session_key: session_key}
key_id, secret_key = get_aws_creds(stanza, metas, {})
if not key_id:
mgr = cm.ConfManager(metas[tac.server_uri], session_key, "nobody",
"Splunk_TA_aws")
ext_info = mgr.get_stanza("aws_account_ext", account_name)
if scutil.is_false(ext_info.get("iam")):
logger.error("No AWS Account is configured. Setup App first.")
raise Exception("No AWS Account is configured. Setup App first.")
return key_id, secret_key
def _disable_tasks_for_deleted_forwarders(self, global_settings,
kafka_clusters, forwarders,
dispatched_tasks,
removed_clusters):
# 2) Delete tasks on remote forwarders by setting "removed = 1"
# for removed forwarders
logger.info("Delete tasks for deleted forwarders")
failed_clusters = set()
for stanza_name, forwarder in forwarders.iteritems():
if utils.is_false(forwarder.get(c.removed)):
continue
localhost = self.is_current_host(forwarder[c.hostname])
if localhost:
conf_mgr = cm.ConfManager(self._meta_configs[c.server_uri],
self._meta_configs[c.session_key],
app_name=self._appname)
all_stanzas = conf_mgr.all_data_input_stanzas(c.kafka_mod)
res = self._delete_data_inputs(kafka_clusters, global_settings,
all_stanzas)
for kafka_stanza in dispatched_tasks.keys():
forwarder_tasks = dispatched_tasks[kafka_stanza]
if stanza_name not in forwarder_tasks:
continue
if kafka_stanza in removed_clusters:
logger.info("%s cluster has already been disabled",
kafka_stanza)
continue
if not localhost:
res = self._do_disable(forwarder, removed_clusters,
kafka_stanza)
if res:
del forwarder_tasks[stanza_name]
logger.info(
"Remove forwarder=%s from kafka_cluster=%s in "
"ckpt", stanza_name, kafka_stanza)
if not dispatched_tasks[kafka_stanza]:
logger.info("Remove cluster=%s from ckpt",
kafka_stanza)
del dispatched_tasks[kafka_stanza]
else:
failed_clusters.add(kafka_stanza)
return failed_clusters
def _remove_tasks(self, clusters, global_settings):
conf_mgr = cm.ConfManager(self._meta_configs[c.server_uri],
self._meta_configs[c.session_key],
app_name=self._appname)
all_stanzas = conf_mgr.all_data_input_stanzas(c.kafka_mod)
stanzas_to_be_deleted = []
for cluster_stanza, cluster in clusters.iteritems():
if utils.is_false(cluster.get(c.removed)):
continue
logger.warn("Remove all tasks for cluster=%s", cluster_stanza)
for stanza in all_stanzas:
if stanza[c.kafka_cluster] != cluster_stanza:
continue
stanzas_to_be_deleted.append(stanza)
self._delete_data_inputs(clusters, global_settings,
stanzas_to_be_deleted)
def _merge_global_settings(self, global_settings, kafka_clusters):
task_configs = []
keys = [
c.index, c.name, c.kafka_partition_offset, c.kafka_partition,
c.kafka_topic_group
]
kafka_clients = {}
for stanza in self._stanza_configs:
with save_and_restore(stanza, keys):
valid_kafka = utils.is_false(
kafka_clusters[stanza[c.kafka_cluster]].get(c.removed))
if not valid_kafka:
continue
if stanza[c.kafka_cluster] not in kafka_clusters:
logger.warn(
"%s is refered by data input stanza %s, but "
"it is not found in kafka conf",
stanza[c.kafka_cluster], stanza[c.name])
continue
stanza.update(global_settings[c.global_settings])
stanza.update(kafka_clusters[stanza[c.kafka_cluster]])
stanza.update(self._meta_configs)
stanza[c.appname] = self._appname
self._set_offset(stanza)
# construct topic, partition list for KafkaDataLoader
topic = stanza[c.kafka_topic]
topic_parts = [[
topic, stanza[c.kafka_partition_offset], stanza[c.index]
]]
partitions = self._filter_partitions(
stanza, (topic, ), kafka_clusters[stanza[c.kafka_cluster]],
kafka_clients)
if partitions:
topic_parts = [[
topic, pid, stanza[c.kafka_partition_offset],
stanza[c.index]
] for pid in partitions]
stanza[c.kafka_topic] = topic_parts
task_configs.append(stanza)
return task_configs
def _disable_tasks_for_deleted_clusters(self, kafka_clusters, forwarders,
dispatched_tasks,
removed_clusters):
# 1) Delete tasks on remote forwarders by setting "removed = 1"
# for removed kafka clusters
logger.info("Delete tasks for deleted clusters")
failed_clusters = set()
for stanza_name, cluster in kafka_clusters.iteritems():
if utils.is_false(cluster.get(c.removed)):
continue
if stanza_name not in dispatched_tasks:
continue
for forwarder_stanza in dispatched_tasks[stanza_name].keys():
if forwarder_stanza not in forwarders:
logger.error(
"Forwarder=%s is in ckpt but not found in "
"forwarder conf", forwarder_stanza)
del dispatched_tasks[stanza_name][forwarder_stanza]
continue
forwarder = forwarders[forwarder_stanza]
res = self._do_disable(forwarder, removed_clusters,
stanza_name)
if res:
del dispatched_tasks[stanza_name][forwarder_stanza]
logger.info("Remove cluster=%s forwarder=%s from ckpt",
stanza_name, forwarder_stanza)
else:
failed_clusters.add(stanza_name)
if not dispatched_tasks[stanza_name]:
logger.info("Remove cluster=%s from ckpt", stanza_name)
del dispatched_tasks[stanza_name]
return failed_clusters
def _get_tasks(self):
conf_mgr = tcm.TAConfManager(self.data_collection_conf(),
self.metas[ggc.server_uri],
self.metas[ggc.session_key],
appname=self._appname)
conf_mgr.reload()
data_collections = conf_mgr.all(return_acl=False)
if not data_collections:
return []
data_collections = {
k: v
for k, v in data_collections.iteritems()
if utils.is_false(v.get(ggc.disabled))
}
global_settings = get_global_settings(self.metas[ggc.server_uri],
self.metas[ggc.session_key])
google_creds = get_google_creds(self.metas[ggc.server_uri],
self.metas[ggc.session_key])
return self._expand_tasks(global_settings, google_creds,
data_collections)
def _filter_out_removed(kvs):
return {
k: v
for k, v in kvs.iteritems() if utils.is_false(v.get(c.removed))
}
def is_valid_bool(val):
"""Check whether a string can be convert to bool.
:param val: value as string.
:return: `True` if value can be convert to bool else `False`.
"""
return util.is_true(val) or util.is_false(val)