def run(collector_cls,
settings,
checkpoint_cls=None,
config_cls=None,
log_suffix=None):
"""
Main loop. Run this TA forever
"""
# This is for stdout flush
utils.disable_stdout_buffer()
# http://bugs.python.org/issue7980
time.strptime("2016-01-01", "%Y-%m-%d")
tconfig = tc.create_ta_config(settings, config_cls or tc.TaConfig,
log_suffix)
stulog.set_log_level(tconfig.get_log_level())
task_configs = tconfig.get_task_configs()
if not task_configs:
stulog.logger.debug("No task and exiting...")
return
meta_config = tconfig.get_meta_config()
if tconfig.is_shc_but_not_captain():
# In SHC env, only captain is able to collect data
stulog.logger.debug("This search header is not captain, will exit.")
return
loader = dl.create_data_loader(meta_config)
jobs = [
tdc.create_data_collector(
loader,
tconfig,
meta_config,
task_config,
collector_cls,
checkpoint_cls=checkpoint_cls or cpmgr.TACheckPointMgr,
) for task_config in task_configs
]
# handle signal
_setup_signal_handler(loader, settings["basic"]["title"])
# monitor files to reboot
if settings["basic"].get("monitor_file"):
monitor = fm.FileMonitor(
_handle_file_changes(loader),
_get_conf_files(settings["basic"]["monitor_file"]),
)
loader.add_timer(monitor.check_changes, time.time(), 10)
# add orphan process handling, which will check each 1 second
orphan_checker = opm.OrphanProcessChecker(loader.tear_down)
loader.add_timer(orphan_checker.check_orphan, time.time(), 1)
loader.run(jobs)
def run(collector_cls,
settings,
checkpoint_cls=None,
config_cls=None,
log_suffix=None,
single_instance=True,
cc_json_file=None):
"""
Main loop. Run this TA forever
"""
ta_short_name = settings["meta"]["name"].lower()
# This is for stdout flush
utils.disable_stdout_buffer()
# http://bugs.python.org/issue7980
time.strptime('2016-01-01', '%Y-%m-%d')
loader = dl.create_data_loader()
# handle signal
_setup_signal_handler(loader, ta_short_name)
# monitor files to reboot
try:
monitor = fm.FileMonitor(_handle_file_changes(loader),
_get_conf_files(settings))
loader.add_timer(monitor.check_changes, time.time(), 10)
except Exception:
stulog.logger.exception("Fail to add files for monitoring")
# add orphan process handling, which will check each 1 second
orphan_checker = opm.OrphanProcessChecker(loader.tear_down)
loader.add_timer(orphan_checker.check_orphan, time.time(), 1)
tconfig = tc.create_ta_config(settings,
config_cls or tc.TaConfig,
log_suffix,
single_instance=single_instance)
task_configs = tconfig.get_task_configs()
if not task_configs:
stulog.logger.debug("No task and exiting...")
return
meta_config = tconfig.get_meta_config()
meta_config["cc_json_file"] = cc_json_file
if tconfig.is_shc_member():
# Don't support SHC env
stulog.logger.error(
"This host is in search head cluster environment , "
"will exit.")
return
# In this case, use file for checkpoint
if _is_checkpoint_dir_length_exceed_limit(tconfig,
meta_config["checkpoint_dir"]):
stulog.logger.error(
"The length of the checkpoint directory path: '{}' "
"is too long. The max length we support is {}",
meta_config["checkpoint_dir"], __CHECKPOINT_DIR_MAX_LEN__)
return
jobs = [
tdc.create_data_collector(loader,
tconfig,
meta_config,
task_config,
collector_cls,
checkpoint_cls=checkpoint_cls
or cpmgr.TACheckPointMgr)
for task_config in task_configs
]
loader.run(jobs)
import threading
import sys
import json
import traceback
import time
import copy
import datetime
from splunktalib.common import log
logger = log.Logs().get_logger("util")
import splunktalib.common.util as scutil
import splunktalib.hec_config as hecc
import splunktalib.rest as sr
scutil.disable_stdout_buffer()
class _ExtendedEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.datetime):
return str(obj)
return json.JSONEncoder.default(self, obj)
class ModinputEvent(object):
base_fmt = ("""<event{unbroken}>"""
"<index>{index}</index>"
"<host>{host}</host>"
"<source>{source}</source>"
from splunktalib.common import log
import pubsub_mod.google_pubsub_consts as gpc
logger = log.Logs(gpc.pubsub_ns).get_logger("main")
import splunktalib.common.util as utils
import splunktalib.common.pattern as gcp
import splunktalib.orphan_process_monitor as opm
import google_concurrent_data_loader as gcdl
import google_ta_common.ta_common as tacommon
import pubsub_mod.google_pubsub_conf as psconf
utils.remove_http_proxy_env_vars()
utils.disable_stdout_buffer()
def print_scheme():
title = "Splunk AddOn for Google"
description = "Collect and index PubSub data for Google"
tacommon.print_scheme(title, description)
@gcp.catch_all(logger)
def run():
"""
Main loop. Run this TA forever
"""
logger.info("Start google_pubsub")
import threading
import sys
import json
import traceback
import time
import splunktalib.common.util as scutil
from splunktalib.common import log
import splunktalib.rest as sr
logger = log.Logs().get_logger("util")
scutil.disable_stdout_buffer()
class ModinputEvent(object):
base_fmt = ("""<event{unbroken}>"""
"<index>{index}</index>"
"<host>{host}</host>"
"<source>{source}</source>"
"<sourcetype>{sourcetype}</sourcetype>"
"<time>{time}</time>"
"<data>{data}</data>{done}</event>")
event_fmt = base_fmt.replace("{unbroken}", "").replace("{done}", "")
unbroken_fmt = base_fmt.replace("{unbroken}", ' unbroken="1"').replace(
"{done}", "")
import time
import os
import os.path as op
import splunktalib.common.util as utils
import splunktalib.modinput as modinput
import splunktalib.file_monitor as fm
import splunktalib.orphan_process_monitor as opm
from splunktalib.common import log
import kafka_config as kc
import kafka_consts as c
import kafka_concurrent_data_loader as kcdl
utils.remove_http_proxy_env_vars()
utils.disable_stdout_buffer()
logger = log.Logs().get_logger("main")
def do_scheme():
"""
Feed splunkd the TA's scheme
"""
desc = ("Enable modular inputs to collect Kafka topic data. Use only if "
"managing inputs manually from individual forwarders. "
"See documentation for details.")
print """
<scheme>
<title>Splunk Add-on for {}</title>
