def _do_send_safe(self, clusters, dispatched_tasks, forwarder_stanza,
forwarder, settings, localhost):
session_key = self._get_forwarder_session_key(forwarder)
if not localhost and c.global_settings in settings["config_settings"]:
logger.info("Config global settings for forwarder=%s",
forwarder[c.hostname])
uri = ("{}/servicesNS/nobody/{}/kafka_input_setup"
"/kafka_settings/kafka_settings")
url = uri.format(forwarder[c.hostname], self._appname)
payload = {c.all_settings: json.dumps(settings["config_settings"])}
resp, content = rest.splunkd_request(url,
session_key,
method="POST",
data=payload)
if resp is None or resp.status not in (200, 201):
logger.error(
"Failed to update config settings on forwarder=%s",
forwarder[c.hostname])
self._dispatch_errors = True
return
conf_mgr = cm.ConfManager(forwarder[c.hostname],
session_key,
app_name=self._appname)
for name, datainput in settings["data_input_settings"].iteritems():
res = self._create_or_update(clusters, dispatched_tasks,
forwarder_stanza, datainput, name,
forwarder, conf_mgr)
if not res:
self._dispatch_errors = True
def get_tasks(self):
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
tasks = self._get_config_rule_tasks(conf_mgr)
settings = conf_mgr.all_stanzas_as_dicts(self.conf_file,
do_reload=False)
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
# set proxy here for validating credentials
tacommon.set_proxy_env(proxy_info)
set_log_level(settings[tac.log_stanza][tac.log_level])
valid_tasks = []
for task in tasks:
try:
# validate credentials
tacommon.get_service_client(task, tac.config)
task[tac.log_level] = settings[tac.log_stanza][tac.log_level]
task.update(settings[tac.global_settings])
task.update(proxy_info)
valid_tasks.append(task)
except Exception as e:
input_name = scutil.extract_datainput_name(task[tac.name])
logger.exception(
'Failed to load credentials, ignore this input.',
datainput=input_name)
return tacommon.handle_hec(valid_tasks, "aws_config_rule")
def handleList(self, confInfo):
logger.info("start list")
conf_mgr = conf.ConfManager(scc.getMgmtUri(), self.getSessionKey())
conf_mgr.reload_conf(c.myta_conf)
conf_mgr.reload_conf(c.myta_credential_conf)
conf_mgr.reload_conf(c.myta_customized_conf)
# read globala and proxy settings
all_settings = conf_mgr.all_stanzas_as_dicts(c.myta_conf)
if not all_settings:
all_settings = {}
self._setNoneValues(all_settings.get(c.global_settings, {}))
# read account credential settings
for cred, cred_conf in self.cred_confs:
ta_conf_mgr = ta_conf.TAConfManager(cred_conf, scc.getMgmtUri(),
self.getSessionKey())
ta_conf_mgr.set_encrypt_keys(self.encrypt_fields_credential)
creds = ta_conf_mgr.all(return_acl=False)
if creds:
self._setNoneValues(creds)
all_settings.update({cred: creds})
# read customized settings
ta_conf_mgr = ta_conf.TAConfManager(c.myta_customized_conf,
scc.getMgmtUri(),
self.getSessionKey())
ta_conf_mgr.set_encrypt_keys(self.encrypt_fields_customized)
customized_settings = ta_conf_mgr.all(return_acl=False)
all_settings.update({c.myta_customized_settings: customized_settings})
self._clearPasswords(all_settings, self.cred_fields)
all_settings = json.dumps(all_settings)
all_settings = utils.escape_json_control_chars(all_settings)
confInfo[c.myta_settings].append(c.all_settings, all_settings)
logger.info("end list")
def __init__(self, meta_configs, stanza_configs):
self._meta_configs = meta_configs
self._stanza_configs = stanza_configs
self._task_configs = []
self._kafka_topics = None
self._server_info = sc.ServerInfo(self._meta_configs[c.server_uri],
self._meta_configs[c.session_key])
setup_signal_handler(None, self)
self._dispatch_errors = None
self._servername, self._port = extract_hostname_port(
self._meta_configs[c.server_uri])
self._store = create_state_store(self._meta_configs, self._appname,
self._server_info)
conf_mgr = cm.ConfManager(self._meta_configs[c.server_uri],
self._meta_configs[c.session_key])
global_settings = conf_mgr.all_stanzas_as_dicts(c.myta_conf,
do_reload=True)
kafka_clusters = self._get_kafka_clusters(conf_mgr, global_settings)
log.Logs().set_level(global_settings[c.global_settings].get(
c.log_level, "INFO"))
if kafka_clusters:
self._retry_dispatch_tasks(global_settings, kafka_clusters)
# Refresh kafka clusters
kafka_clusters = self._get_kafka_clusters(conf_mgr,
global_settings)
self._kafka_clusters = kafka_clusters
self._get_task_configs(global_settings, kafka_clusters)
else:
logger.info("No Kafka cluster are configured")
def handleList(self, confInfo):
logger.info("start list")
conf_mgr = conf.ConfManager(scc.getMgmtUri(),
self.getSessionKey(),
app_name=self.appName)
conf_mgr.reload_conf(c.myta_conf)
conf_mgr.reload_conf(c.myta_cred_conf)
conf_mgr.reload_conf(c.myta_forwarder_conf)
all_settings = conf_mgr.all_stanzas_as_dicts(c.myta_conf)
if not all_settings:
all_settings = {}
self._setNoneValues(all_settings.get(c.global_settings, {}))
for cred, cred_conf in self.cred_confs:
ta_conf_mgr = ta_conf.TAConfManager(cred_conf,
scc.getMgmtUri(),
self.getSessionKey(),
appname=self.appName)
ta_conf_mgr.set_encrypt_keys(self.encrypt_fields)
creds = ta_conf_mgr.all(return_acl=False)
if creds:
self._setNoneValues(creds)
all_settings.update({cred: creds})
self._clearPasswords(all_settings, self.cred_fields)
all_settings = json.dumps(all_settings)
all_settings = utils.escape_json_control_chars(all_settings)
confInfo[c.myta_settings].append(c.all_settings, all_settings)
logger.info("end list")
def _delete_data_inputs(self, clusters, global_settings,
stanzas_to_be_deleted):
"""
:return: false if something goes wrong, true if everything is good
"""
has_exception = False
global_settings = global_settings[c.global_settings]
store = ss.get_state_store(self._meta_configs,
self._appname,
use_kv_store=global_settings.get(
c.use_kv_store, False))
conf_mgr = cm.ConfManager(self._meta_configs[c.server_uri],
self._meta_configs[c.session_key],
app_name=self._appname)
kafka_clients = {}
for stanza in stanzas_to_be_deleted:
try:
self._delete_ckpts(stanza, clusters, kafka_clients, store)
conf_mgr.set_appname(stanza[c.appname])
conf_mgr.delete_data_input(c.kafka_mod, stanza[c.name])
except req.ConfNotExistsException:
pass
except Exception:
has_exception = True
logger.error("Failed to remove data input=%s, error=%s",
stanza[c.name], traceback.format_exc())
continue
if not has_exception:
conf_mgr.set_appname(self._appname)
conf_mgr.delete_stanza(c.myta_cred_conf,
stanza[c.kafka_cluster])
return not has_exception
def _get_tasks(self):
if not self.stanza_configs:
return None
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
logging_settings = conf_mgr.get_stanza(self.log_info,
asc.log_stanza,
do_reload=False)
# set the log level read from conf for our logger
set_log_level(logging_settings[asc.log_level])
# entry point for this stanza task, setup root logger here
# Generic S3 can be configured to be single-instance
# or multiple instance
# through env variable
stanza_name = ''
try:
if len(self.stanza_configs) == 1:
# only one stanza exists
stanza_name = self.stanza_configs[0].get('name', '')
except Exception:
logger.exception('Failed to get stanza name!')
stanza_name = extract_datainput_name(stanza_name)
logging.setup_root_logger(app_name=tac.splunk_ta_aws,
modular_name=asc.mod_name,
stanza_name=stanza_name)
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
tasks, creds = [], {}
for stanza in self.stanza_configs:
task = {}
task.update(stanza)
task.update(self.metas)
task.update(proxy_info)
task[tac.log_level] = logging_settings[asc.log_level]
task[tac.interval] = tacommon.get_interval(task, 3600)
task[tac.polling_interval] = task[tac.interval]
task[asc.max_retries] = int(task.get(asc.max_retries, 3))
task[asc.prefix] = task.get(asc.key_name)
task[asc.last_modified] = self._get_last_modified_time(
task[asc.initial_scan_datetime])
task[
asc.
terminal_scan_datetime] = self._convert_terminal_scan_datetime(
task.get(asc.terminal_scan_datetime))
input_name = scutil.extract_datainput_name(task[tac.name])
task[asc.data_input] = input_name
task[tac.sourcetype] = task.get(tac.sourcetype, "aws:s3")
task[asc.bucket_name] = str(task[asc.bucket_name])
if not task.get(asc.whitelist):
task[asc.whitelist] = s3common.sourcetype_to_keyname_regex.get(
task[tac.sourcetype])
tasks.append(task)
logger.info("Done with configuration read from conf.")
s3ckpt.handle_ckpts(tasks)
return tasks
def __init__(self, conf_file, splunkd_uri, session_key, appname=None):
if appname is None:
appname = utils.get_appname_from_path(op.abspath(__file__))
self._conf_file = conf.conf_file2name(conf_file)
self._conf_mgr = conf.ConfManager(splunkd_uri, session_key, app_name=appname)
self._cred_mgr = cred.CredentialManager(
splunkd_uri, session_key, app=appname, owner="nobody", realm=appname
)
self._keys = None
def _get_cached_conf_mgr(self, forwarder_stanza, forwarder,
cached_conf_mgrs):
if forwarder_stanza in cached_conf_mgrs:
return cached_conf_mgrs[forwarder_stanza]
session_key = self._get_forwarder_session_key(forwarder)
conf_mgr = cm.ConfManager(forwarder[c.hostname],
session_key,
app_name=self._appname)
cached_conf_mgrs[forwarder_stanza] = conf_mgr
return conf_mgr
def _handleKafkaClusterList(self, caller_args, conf_info):
conf_mgr = conf.ConfManager(scc.getMgmtUri(), self.getSessionKey())
conf_mgr.reload_conf(c.myta_cred_conf)
all_creds = conf_mgr.all_stanzas_as_dicts(c.myta_cred_conf)
all_creds = {
k: v
for k, v in all_creds.iteritems()
if utils.is_false(v.get(c.removed))
}
# _ = caller_args[self.kafka_cluster_filter],
conf_info["kafka_clusters"].append("clusters", all_creds.keys())
def assert_creds(account_name, session_key, logger):
import splunk.clilib.cli_common as scc
stanza = {tac.aws_account: account_name}
metas = {tac.server_uri: scc.getMgmtUri(), tac.session_key: session_key}
key_id, secret_key = get_aws_creds(stanza, metas, {})
if not key_id:
mgr = cm.ConfManager(metas[tac.server_uri], session_key, "nobody",
"Splunk_TA_aws")
ext_info = mgr.get_stanza("aws_account_ext", account_name)
if scutil.is_false(ext_info.get("iam")):
logger.error("No AWS Account is configured. Setup App first.")
raise Exception("No AWS Account is configured. Setup App first.")
return key_id, secret_key
def _handleKafkaTopicList(self, caller_args, conf_info):
conf_mgr = conf.ConfManager(scc.getMgmtUri(), self.getSessionKey())
conf_mgr.reload_conf(c.myta_cred_conf)
all_creds = conf_mgr.all_stanzas_as_dicts(c.myta_cred_conf)
kafka_cluster = all_creds.get(caller_args["kafka_cluster"][0])
if kafka_cluster:
client = kq.get_kafka_client(kafka_cluster)
topics = set(client.topic_partitions.keys())
topics = [topic for topic in topics]
conf_info["kafka_topics"].append("topics", topics)
else:
msg = "Unknown Kafka cluster {}".format(
caller_args["kafka_cluster"])
logger.error(msg)
raise Exception(msg)
def _disable_tasks_for_deleted_forwarders(self, global_settings,
kafka_clusters, forwarders,
dispatched_tasks,
removed_clusters):
# 2) Delete tasks on remote forwarders by setting "removed = 1"
# for removed forwarders
logger.info("Delete tasks for deleted forwarders")
failed_clusters = set()
for stanza_name, forwarder in forwarders.iteritems():
if utils.is_false(forwarder.get(c.removed)):
continue
localhost = self.is_current_host(forwarder[c.hostname])
if localhost:
conf_mgr = cm.ConfManager(self._meta_configs[c.server_uri],
self._meta_configs[c.session_key],
app_name=self._appname)
all_stanzas = conf_mgr.all_data_input_stanzas(c.kafka_mod)
res = self._delete_data_inputs(kafka_clusters, global_settings,
all_stanzas)
for kafka_stanza in dispatched_tasks.keys():
forwarder_tasks = dispatched_tasks[kafka_stanza]
if stanza_name not in forwarder_tasks:
continue
if kafka_stanza in removed_clusters:
logger.info("%s cluster has already been disabled",
kafka_stanza)
continue
if not localhost:
res = self._do_disable(forwarder, removed_clusters,
kafka_stanza)
if res:
del forwarder_tasks[stanza_name]
logger.info(
"Remove forwarder=%s from kafka_cluster=%s in "
"ckpt", stanza_name, kafka_stanza)
if not dispatched_tasks[kafka_stanza]:
logger.info("Remove cluster=%s from ckpt",
kafka_stanza)
del dispatched_tasks[kafka_stanza]
else:
failed_clusters.add(kafka_stanza)
return failed_clusters
def get_tasks(self):
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
tasks = self._get_cloudwatch_logs_tasks(conf_mgr)
logging_settings = conf_mgr.get_stanza(self.conf_file,
tac.log_stanza,
do_reload=False)
set_log_level(logging_settings[tac.log_level])
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
for task in tasks:
task[tac.log_level] = logging_settings[tac.log_level]
task.update(proxy_info)
return tasks
def _remove_tasks(self, clusters, global_settings):
conf_mgr = cm.ConfManager(self._meta_configs[c.server_uri],
self._meta_configs[c.session_key],
app_name=self._appname)
all_stanzas = conf_mgr.all_data_input_stanzas(c.kafka_mod)
stanzas_to_be_deleted = []
for cluster_stanza, cluster in clusters.iteritems():
if utils.is_false(cluster.get(c.removed)):
continue
logger.warn("Remove all tasks for cluster=%s", cluster_stanza)
for stanza in all_stanzas:
if stanza[c.kafka_cluster] != cluster_stanza:
continue
stanzas_to_be_deleted.append(stanza)
self._delete_data_inputs(clusters, global_settings,
stanzas_to_be_deleted)
def get_tasks(self):
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
all_tasks = self._get_kinesis_tasks(conf_mgr)
settings = conf_mgr.all_stanzas_as_dicts(self.conf_file,
do_reload=False)
# set logging level for our logger
set_log_level(settings[tac.log_stanza][tac.log_level])
for task in all_tasks:
task[tac.log_level] = settings[tac.log_stanza][tac.log_level]
task.update(settings[tac.global_settings])
ackpt.clean_up_ckpt_for_deleted_data_input(all_tasks)
return tacommon.handle_hec(all_tasks, "aws_kinesis")
def handleEdit(self, confInfo):
logger.info("start edit")
conf_mgr = conf.ConfManager(scc.getMgmtUri(),
self.getSessionKey(),
app_name=self.appName)
conf_mgr.reload_conf(c.myta_conf)
conf_mgr.reload_conf(c.myta_credential_conf)
conf_mgr.reload_conf(c.myta_customized_conf)
all_origin_settings = conf_mgr.all_stanzas_as_dicts(c.myta_conf)
all_settings = utils.escape_json_control_chars(
self.callerArgs.data[c.all_settings][0])
all_settings = json.loads(all_settings)
# write globala and proxy settings
self._updateGlobalSettings(c.global_settings, all_settings,
all_origin_settings, conf_mgr)
# write account credential settings
for cred, conf_file in self.cred_confs:
creds = all_settings.get(cred, {})
if creds == c.ignore_backend_req:
logger.info("Ignore backend rest request")
continue
ta_conf_mgr = ta_conf.TAConfManager(conf_file,
scc.getMgmtUri(),
self.getSessionKey(),
appname=self.appName)
ta_conf_mgr.set_encrypt_keys(self.encrypt_fields_credential)
self._updateCredentials(creds, ta_conf_mgr)
# write customized settings
customized_settings = all_settings.get(c.myta_customized_settings, {})
ta_conf_mgr = ta_conf.TAConfManager(c.myta_customized_conf,
scc.getMgmtUri(),
self.getSessionKey(),
appname=self.appName)
ta_conf_mgr.set_encrypt_keys(self.encrypt_fields_customized)
self._updateCredentials(customized_settings, ta_conf_mgr)
conf_mgr.reload_conf(c.myta_conf)
conf_mgr.reload_conf(c.myta_credential_conf)
conf_mgr.reload_conf(c.myta_customized_conf)
logger.info("end edit")
def handleEdit(self, confInfo):
logger.info("start edit")
conf_mgr = conf.ConfManager(scc.getMgmtUri(),
self.getSessionKey(),
app_name=self.appName)
conf_mgr.reload_conf(c.myta_conf)
conf_mgr.reload_conf(c.myta_cred_conf)
conf_mgr.reload_conf(c.myta_forwarder_conf)
all_origin_settings = conf_mgr.all_stanzas_as_dicts(c.myta_conf)
all_settings = utils.escape_json_control_chars(
self.callerArgs.data[c.all_settings][0])
all_settings = json.loads(all_settings)
for stanza in (c.global_settings, c.proxy_settings):
self._updateGlobalSettings(stanza, all_settings,
all_origin_settings, conf_mgr)
for cred, conf_file in self.cred_confs:
creds = all_settings.get(cred, {})
if creds == c.ignore_backend_req:
logger.info("Ignore backend rest request")
continue
update = False
if c.backend_update_req in creds:
update = True
del creds[c.backend_update_req]
ta_conf_mgr = ta_conf.TAConfManager(conf_file,
scc.getMgmtUri(),
self.getSessionKey(),
appname=self.appName)
ta_conf_mgr.set_encrypt_keys(self.encrypt_fields)
self._updateCredentials(creds, ta_conf_mgr, update)
conf_mgr.reload_conf(c.myta_conf)
conf_mgr.reload_conf(c.myta_cred_conf)
conf_mgr.reload_conf(c.myta_forwarder_conf)
logger.info("end edit")
def delete_ckpts_for_removed_data_inputs(tasks):
# All datainputs including disabled
mgr = cm.ConfManager(tasks[0][tac.server_uri], tasks[0][tac.session_key])
s3_stanzas = mgr.all_data_input_stanzas("aws_s3", do_reload=False)
new_data_inputs = set(stanza[tac.stanza_name] for stanza in s3_stanzas)
data_inputs = "data_inputs"
data_inputs_ckpt_key = "aws_s3_data_inputs.ckpt"
store = create_state_store(tasks[0])
state = store.get_state(data_inputs_ckpt_key)
if not state or not state.get(data_inputs):
store.update_state(data_inputs_ckpt_key,
{data_inputs: list(new_data_inputs)})
return
config = copy.copy(tasks[0])
old_data_inputs = set(state[data_inputs])
for removed_data_input in (old_data_inputs - new_data_inputs):
config[asc.data_input] = removed_data_input
index_ckpt = S3IndexCheckpointer(config, new=False)
if not index_ckpt.index_ckpt():
logger.warn("index ckpt for datainput=%s doesn't exist",
removed_data_input)
continue
# 1) Remove ckpts for keys
for key_name, key_ckpt in index_ckpt.keys().iteritems():
index_ckpt.delete_state(key_ckpt[asc.key_ckpt])
# 2) Remove index ckpt for data input
index_ckpt.delete()
logger.warn(
"Remove ckpt for datainput=%s bucket_name=%s, since the "
"datainput has been deleted", removed_data_input,
index_ckpt.bucket_name())
store.update_state(data_inputs_ckpt_key,
{data_inputs: list(new_data_inputs)})
def get_tasks(self):
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
tasks = self._get_description_tasks(conf_mgr)
logging_settings = conf_mgr.get_stanza(self.conf_file,
tac.log_stanza,
do_reload=False)
# set logging level for our logger
set_log_level(logging_settings[tac.log_level])
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
# Set proxy for loading credentials by boto3
tacommon.set_proxy_env(proxy_info)
for task in tasks:
task[tac.log_level] = logging_settings[tac.log_level]
task.update(proxy_info)
self._assign_source(tasks)
return tasks
def get_aws_creds(stanza, metas, creds):
account_name = stanza.get(tac.aws_account)
mgr = cm.ConfManager(metas[tac.server_uri], metas[tac.session_key],
"nobody", "Splunk_TA_aws")
ext_info = mgr.get_stanza("aws_account_ext", account_name)
if not account_name or scutil.is_true(ext_info.get("iam")):
# IAM role EC2
key_id, secret_key = None, None
else:
if account_name in creds:
key_id, secret_key = creds[account_name]
else:
km = taa.AwsAccessKeyManager(taa.KEY_NAMESPACE, taa.KEY_OWNER,
metas[tac.session_key])
acct = km.get_accesskey(name=account_name)
if not acct:
raise Exception("Failed to get creds for account=%s",
account_name)
key_id, secret_key = acct.key_id, acct.secret_key
creds[account_name] = [key_id, secret_key]
return key_id, secret_key
def get_tasks(self):
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
stanzas = conf_mgr.all_stanzas(self.task_file, do_reload=False)
settings = conf_mgr.all_stanzas_as_dicts(self.conf_file,
do_reload=False)
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
# set proxy here for validating credentials
tacommon.set_proxy_env(proxy_info)
level = settings[tac.log_stanza][tac.log_level]
set_log_level(level)
tasks = self._get_inspector_tasks(stanzas, settings, proxy_info)
config = dict()
config.update(self.metas)
config.update(settings[tac.global_settings])
_cleanup_checkpoints(tasks, config)
tasks = [
task for task in tasks if not scutil.is_true(task.get('disabled'))
]
return tacommon.handle_hec(tasks, "aws_inspector")
def handleEdit(self, confInfo):
args = self.callerArgs.data
# Validate interval
interval = args['interval'][0]
if not interval.isdigit():
raise admin.ArgValidationException("Interval should be digit")
conf_mgr = conf.ConfManager(splunkd_uri=scc.getMgmtUri(), session_key=self.getSessionKey(),
app_name=self.appName, owner='-')
script_enabled = int(args['script_enabled'][0])
stanza = {"interval": interval}
success = conf_mgr.update_data_input("script",
"$SPLUNK_HOME/etc/apps/Splunk_TA_symantec-ep/bin/malware_category_update.py",
stanza)
if not success:
logger.error("ERROR in updating script input")
if script_enabled == 1:
conf_mgr.enable_data_input("script",
"$SPLUNK_HOME/etc/apps/Splunk_TA_symantec-ep/bin/malware_category_update.py")
else:
conf_mgr.disable_data_input("script",
"$SPLUNK_HOME/etc/apps/Splunk_TA_symantec-ep/bin/malware_category_update.py")
def get_tasks(self):
if not self.stanza_configs:
return None
conf_mgr = cm.ConfManager(self.metas[tac.server_uri],
self.metas[tac.session_key])
logging = conf_mgr.get_stanza(self.log_info,
asc.log_stanza,
do_reload=True)
proxy_info = tpc.get_proxy_info(self.metas[tac.session_key])
tasks, creds = [], {}
for stanza in self.stanza_configs:
task = {}
task.update(stanza)
task.update(self.metas)
task.update(proxy_info)
key_id, secret_key = tacommon.get_aws_creds(
stanza, self.metas, creds)
task[tac.log_level] = logging[asc.log_level]
task[tac.key_id] = key_id
task[tac.secret_key] = secret_key
task[tac.interval] = tacommon.get_interval(task, 3600)
task[asc.max_retries] = int(task.get(asc.max_retries, 10))
task[asc.prefix] = task.get(asc.key_name)
task[asc.last_modified] = self._get_last_modified_time(
task[asc.initial_scan_datetime])
input_name = scutil.extract_datainput_name(task[tac.name])
task[asc.data_input] = input_name
task[tac.sourcetype] = task.get(tac.sourcetype, "aws:s3")
task[asc.bucket_name] = str(task[asc.bucket_name])
if not task.get(asc.whitelist):
task[asc.whitelist] = s3common.sourcetype_to_keyname_regex.get(
task[tac.sourcetype])
tasks.append(task)
s3ckpt.handle_ckpts(tasks)
return tasks
def handleEdit(self, confInfo):
args = self.callerArgs.data
for key, val in args.items():
if val[0] is None:
val[0] = ''
proxy_stanza = {}
proxy_port = args['proxy_port'][0]
proxy_enabled = args['proxy_enabled'][0]
proxy_url = args['proxy_url'][0]
# Validate args
if proxy_url != '' and proxy_port == '':
raise admin.ArgValidationException("Port should not be blank")
if proxy_url == '' and proxy_port != '':
raise admin.ArgValidationException("URL should not be blank")
if proxy_port != '' and not proxy_port.isdigit():
raise admin.ArgValidationException("Port should be digit")
# Proxy is enabled, but proxy url or port is empty
if proxy_enabled == '1' and (proxy_url == '' or proxy_port == ''):
raise admin.ArgValidationException("URL and port should not be blank")
# Password is filled but username is empty
if args['proxy_password'][0] != '' and args['proxy_username'][0] == '':
raise admin.ArgValidationException("Username should not be blank")
confDict = self.readConf("symantec_ep")
self._decrypt_username_password(confDict)
proxy = confDict['symantec_ep_proxy']
proxy_stanza['proxy_enabled'] = proxy_enabled
proxy_stanza['proxy_url'] = proxy_url
proxy_stanza['proxy_port'] = proxy_port
if args['proxy_password'][0] != '' or proxy['proxy_username'] != args['proxy_username'][0]:
stanza = {}
proxy = {'proxy_username': args['proxy_username'][0], 'proxy_password': args['proxy_password'][0]}
stanza['symantec_ep_proxy'] = proxy
cred_mgr = cred.CredentialManager(session_key=self.getSessionKey(),
app=self.appName, splunkd_uri=scc.getMgmtUri())
try:
result = cred_mgr.update(stanza)
logger.info("Update result:"+str(result))
except Exception:
logger.error("ERROR in creating cred stanza")
logger.error(traceback.format_exc())
if args['proxy_password'][0] != '' or args['proxy_username'][0] != '':
proxy_stanza['proxy_username'] = self.encrypted
proxy_stanza['proxy_password'] = self.encrypted
else:
proxy_stanza['proxy_username'] = ''
proxy_stanza['proxy_password'] = ''
conf_mgr = conf.ConfManager(splunkd_uri=scc.getMgmtUri(), session_key=self.getSessionKey(),
app_name=self.appName, owner='-')
success = conf_mgr.update_stanza('symantec_ep', 'symantec_ep_proxy', proxy_stanza)
if not success:
logger.error("ERROR in writing symantec_ep conf file.")
def handleEdit(self, confInfo):
args = self.callerArgs.data
for key, val in args.items():
if val[0] is None:
val[0] = ''
conf_mgr = conf.ConfManager(splunkd_uri=scc.getMgmtUri(),
session_key=self.getSessionKey(),
app_name=self.appName,
owner='-')
success = False
if self.callerArgs.id == 'okta_loglevel':
loglevel_stanza = {}
loglevel = args['loglevel'][0]
loglevel_stanza['loglevel'] = loglevel
success = conf_mgr.update_stanza('okta', 'okta_loglevel',
loglevel_stanza)
elif self.callerArgs.id == 'okta_proxy':
proxy_stanza = {}
proxy_enabled = args['proxy_enabled'][0]
proxy_stanza['proxy_enabled'] = proxy_enabled
if proxy_enabled.lower().strip() in ("1", "true", "yes", "t", "y"):
proxy_port = args['proxy_port'][0].strip()
proxy_url = args['proxy_url'][0].strip()
proxy_type = args['proxy_type'][0]
proxy_rdns = args['proxy_rdns'][0]
# Validate args
if proxy_url != '' and proxy_port == '':
raise admin.ArgValidationException(
"Port should not be blank")
if proxy_url == '' and proxy_port != '':
raise admin.ArgValidationException(
"URL should not be blank")
if proxy_port != '' and not proxy_port.isdigit():
raise admin.ArgValidationException("Port should be digit")
# Proxy is enabled, but proxy url or port is empty
if proxy_url == '' or proxy_port == '':
raise admin.ArgValidationException(
"URL and port should not be blank")
# Password is filled but username is empty
if args['proxy_password'][0] != '' and args['proxy_username'][
0] == '':
raise admin.ArgValidationException(
"Username should not be blank")
if proxy_type not in ('http', 'http_no_tunnel', 'socks4',
'socks5'):
raise admin.ArgValidationException(
"Unsupported proxy type")
confDict = self.readConf("okta")
self._decrypt_username_password(confDict)
proxy = confDict['okta_proxy']
proxy_stanza['proxy_url'] = proxy_url
proxy_stanza['proxy_port'] = proxy_port
proxy_stanza['proxy_type'] = proxy_type
proxy_stanza['proxy_rdns'] = proxy_rdns
cred_mgr = cred.CredentialManager(
session_key=self.getSessionKey(),
app=self.appName,
splunkd_uri=scc.getMgmtUri())
proxy_username = args['proxy_username'][0].strip()
proxy_password = args['proxy_username'][0].strip()
if not proxy_username:
try:
result = cred_mgr.delete('__Splunk_TA_okta_proxy__')
logger.info("Update result:" + str(result))
except Exception:
logger.warn("ERROR in deleting cred stanza")
logger.warn(traceback.format_exc())
elif proxy_password != '' or \
proxy['proxy_username'] != proxy_username:
stanza = {}
proxy = {
'proxy_username': proxy_username,
'proxy_password': proxy_password
}
stanza['__Splunk_TA_okta_proxy__'] = proxy
try:
result = cred_mgr.update(stanza)
logger.info("Update result:" + str(result))
except Exception:
logger.error("ERROR in creating cred stanza")
logger.error(traceback.format_exc())
raise admin.HandlerSetupException(
"fail to encrypt proxy username and password.")
if proxy_password != '' or proxy_username != '':
proxy_stanza['proxy_username'] = self.encrypted
proxy_stanza['proxy_password'] = self.encrypted
else:
proxy_stanza['proxy_username'] = ''
proxy_stanza['proxy_password'] = ''
success = conf_mgr.update_stanza('okta', 'okta_proxy',
proxy_stanza)
elif self.callerArgs.id == 'okta_server':
okta_server_stanza = {}
custom_cmd_enabled = args['custom_cmd_enabled'][0]
okta_server_stanza['custom_cmd_enabled'] = custom_cmd_enabled
if custom_cmd_enabled.lower().strip() in ("1", "true", "yes", "t",
"y"):
okta_server_url = args['okta_server_url'][0].strip()
okta_server_token = args['okta_server_token'][0].strip()
if okta_server_url != '' and okta_server_token == '':
raise admin.ArgValidationException(
"token should not be blank")
if okta_server_token != '' and okta_server_url == '':
raise admin.ArgValidationException(
"server url should not be blank")
confDict = self.readConf('okta')
okta_server = confDict['okta_server']
cred_mgr = cred.CredentialManager(
session_key=self.getSessionKey(),
app=self.appName,
splunkd_uri=scc.getMgmtUri())
if not okta_server_url:
try:
result = cred_mgr.delete('__Splunk_TA_okta_server__')
logger.info("Update result:" + str(result))
except Exception:
logger.warn("ERROR in deleting cred stanza")
logger.warn(traceback.format_exc())
elif okta_server_token != '' or \
okta_server['okta_server_url'] != okta_server_url:
stanza = {}
okta_server_info = {
'okta_server_url': okta_server_url,
'okta_server_token': okta_server_token
}
stanza['__Splunk_TA_okta_server__'] = okta_server_info
try:
result = cred_mgr.update(stanza)
logger.info("Update result:" + str(result))
except Exception:
logger.error("ERROR in creating cred stanza")
logger.error(traceback.format_exc())
raise admin.HandlerSetupException(
"fail to encrypt okta server url and token.")
if okta_server_url != '' or okta_server_token != '':
okta_server_stanza['okta_server_url'] = okta_server_url
okta_server_stanza['okta_server_token'] = self.encrypted
else:
okta_server_stanza['okta_server_url'] = ''
okta_server_stanza['okta_server_token'] = ''
success = conf_mgr.update_stanza('okta', 'okta_server',
okta_server_stanza)
if not success:
logger.error("ERROR in writing okta conf file.")
raise admin.HandlerSetupException(
"fail to store data into okta.conf file.")
def __init__(self, splunkd_uri, session_key):
self._conf_mgr = cm.ConfManager(splunkd_uri,
session_key,
app_name="splunk_httpinput")
def run():
logger.info("Script input start.")
logger.info("Start reading session key")
if sys.stdin.closed:
return
session_key = sys.stdin.read()
logger.info("End reading session key")
splunkd_uri = "https://localhost:8089/"
server_info = sc.ServerInfo(splunkd_uri, session_key)
if not server_info.is_shc_member() or server_info.is_captain():
logger.info(
"This is a single instance or cluster captain. Run the malare_category_update."
)
app_name = "Splunk_TA_symantec-ep"
conf_name = "symantec_ep"
stanza = "symantec_ep_proxy"
base_url = "http://www.symantec.com/xml/rss/azlistings.jsp"
encrypted = "******"
conf_manager = conf.ConfManager(splunkd_uri=splunkd_uri,
session_key=session_key,
owner='-',
app_name=app_name)
conf_manager.reload_conf(conf_name)
stanza_obj = conf_manager.get_conf(conf_name, stanza)
config = {
"username": "",
"password": "",
"proxy_url": "",
"proxy_port": "",
"proxy_username": "",
"proxy_password": "",
}
if stanza_obj["proxy_enabled"] == '1':
config["proxy_url"] = stanza_obj["proxy_url"]
config["proxy_port"] = int(stanza_obj["proxy_port"])
is_encrypted = all(
stanza_obj.get(k, None) == "******"
for k in ("proxy_username", "proxy_password"))
if is_encrypted:
logger.info("Decrypting")
cred_mgr = cred.CredentialManager(session_key=session_key,
app=app_name,
splunkd_uri=splunkd_uri)
user_creds = cred_mgr.get_clear_password("symantec_ep_proxy")
proxy_username = user_creds["symantec_ep_proxy"][
"proxy_username"]
proxy_password = user_creds["symantec_ep_proxy"][
"proxy_password"]
config["proxy_username"] = proxy_username
config["proxy_password"] = proxy_password
else:
if stanza_obj["proxy_username"] is not None and stanza_obj[
"proxy_password"] is not None:
logger.info("Encrypting")
config["proxy_username"] = stanza_obj["proxy_username"]
config["proxy_password"] = stanza_obj["proxy_password"]
cred_mgr = cred.CredentialManager(session_key=session_key,
app=app_name,
splunkd_uri=splunkd_uri)
try:
new_stanza = {}
proxy = {
'proxy_username': config["proxy_username"],
'proxy_password': config["proxy_password"]
}
new_stanza['symantec_ep_proxy'] = proxy
result = cred_mgr.update(new_stanza)
logger.info("Update result:" + str(result))
proxy_stanza = config.copy()
proxy_stanza['proxy_username'] = encrypted
proxy_stanza['proxy_password'] = encrypted
success = conf_manager.update_stanza(
'symantec_ep', 'symantec_ep_proxy', proxy_stanza)
if not success:
logger.error(
"ERROR in writing symantec_ep conf file.")
except Exception:
logger.error("ERROR in updating cred stanza")
logger.error(traceback.format_exc())
try:
try:
http = rest.build_http_connection(config, timeout=60)
except Exception:
logger.error("ERROR in building connection")
return
if http is not None:
url_list = construct_url(base_url)
item_list = extract_xml(http, url_list)
file_name = os.path.join(os.environ['SPLUNK_HOME'], 'etc',
'apps', app_name, 'lookups',
'symantec_ep_malware_categories.csv')
save_lookup_file(file_name, item_list)
logger.info("Start the SPL")
request_url = splunkd_uri + "services/search/jobs/export"
logger.info("Start the SPL")
args = {
"search":
"|inputlookup symantec_ep_malware_category_lookup|outputlookup symantec_ep_malware_category_lookup",
"output_mode": "raw"
}
response, _ = rest.splunkd_request(splunkd_uri=request_url,
data=args,
session_key=session_key,
method='POST')
if response.status not in (200, 201):
logger.error("The SPL is not executed corretcly.")
else:
logger.info("The SPL is executed correctly.")
else:
logger.error("Failed to create http object.")
except Exception:
logger.error("ERROR in updating malware category lookup table.")
else:
logger.info(
"This is not the cluster captain. Do not run the malare_category_update."
)
def get_tasks(self):
if not self.stanza_configs:
return None
conf_mgr = cm.ConfManager(self.server_uri, self.session_key)
settings = conf_mgr.all_stanzas_as_dicts(self.conf_file,
do_reload=False)
# set logging level for our logger
set_log_level(settings[tac.log_stanza][tac.log_level])
proxy_info = tpc.get_proxy_info(self.session_key)
tasks, creds = {}, {}
for stanza in self.stanza_configs:
input_name = scutil.extract_datainput_name(stanza[tac.name])
with logging.LogContext(datainput=input_name):
stanza[tac.interval] = tacommon.get_interval(stanza, 60)
stanza[tac.polling_interval] = stanza[tac.interval]
stanza[acc.period] = int(stanza[acc.period])
if stanza[acc.period] > 86400 or stanza[acc.period] < 60:
logger.error(
"Granularity(period) is not in range[60, 86400], ignore this input.",
Period=stanza[acc.period],
ErrorCode="ConfigurationError",
ErrorDetail=
"Invalid Granularity(period). It's out of range [60, 86400]."
)
continue
if stanza[tac.polling_interval] % stanza[acc.period]:
logger.error(
"Polling interval is not multiple of period, ignore this input.",
Period=stanza[acc.period],
ErrorCode="ConfigurationError",
ErrorDetail=
"Polling interval should be a multiple of granularity(period)."
)
continue
stanza[tac.datainput] = input_name
stanza[tac.sourcetype] = stanza.get(tac.sourcetype,
"aws:cloudwatch")
metric_names = stanza[acc.metric_names].strip()
if metric_names != ".*":
metric_names = json.loads(metric_names)
else:
metric_names = None
stanza[acc.metric_names] = metric_names
stanza[acc.metric_dimensions] = json.loads(
stanza[acc.metric_dimensions])
stanza[acc.statistics] = json.loads(stanza[acc.statistics])
stanza[tac.log_level] = settings[tac.log_stanza][tac.log_level]
stanza[tac.aws_account] = stanza.get('aws_account')
stanza[tac.aws_iam_role] = stanza.get('aws_iam_role')
stanza[acc.use_metric_format] = scutil.is_true(
stanza.get(acc.use_metric_format, False))
stanza.update(self.metas)
stanza.update(proxy_info)
stanza.update(settings[tac.global_settings])
stanza[tac.use_hec] = scutil.is_true(
stanza.get(tac.use_hec, False))
stanza[acc.max_api_saver_time] = \
int(stanza.get(acc.max_api_saver_time, 7200))
region_tasks = {}
tasks[stanza[tac.datainput]] = region_tasks
for region in stanza[tac.aws_region].split(","):
region = region.strip()
if not region:
continue
task = {}
task.update(stanza)
task[tac.aws_region] = region
task[tac.region] = region
num, rtasks = self._expand_task(task)
if rtasks:
region_tasks[region] = rtasks
stanza[region] = num
if not region_tasks:
logger.warning("No metric/dimension has been found.")
all_tasks = []
for region_tasks in tasks.itervalues():
for rtasks in region_tasks.itervalues():
all_tasks.extend(rtasks)
tacommon.handle_hec(all_tasks, "aws_cloudwatch")
return all_tasks
