def run(collector_cls, settings, checkpoint_cls=None, config_cls=None, log_suffix=None): """ Main loop. Run this TA forever """ # This is for stdout flush utils.disable_stdout_buffer() # http://bugs.python.org/issue7980 time.strptime("2016-01-01", "%Y-%m-%d") tconfig = tc.create_ta_config(settings, config_cls or tc.TaConfig, log_suffix) stulog.set_log_level(tconfig.get_log_level()) task_configs = tconfig.get_task_configs() if not task_configs: stulog.logger.debug("No task and exiting...") return meta_config = tconfig.get_meta_config() if tconfig.is_shc_but_not_captain(): # In SHC env, only captain is able to collect data stulog.logger.debug("This search header is not captain, will exit.") return loader = dl.create_data_loader(meta_config) jobs = [ tdc.create_data_collector( loader, tconfig, meta_config, task_config, collector_cls, checkpoint_cls=checkpoint_cls or cpmgr.TACheckPointMgr, ) for task_config in task_configs ] # handle signal _setup_signal_handler(loader, settings["basic"]["title"]) # monitor files to reboot if settings["basic"].get("monitor_file"): monitor = fm.FileMonitor( _handle_file_changes(loader), _get_conf_files(settings["basic"]["monitor_file"]), ) loader.add_timer(monitor.check_changes, time.time(), 10) # add orphan process handling, which will check each 1 second orphan_checker = opm.OrphanProcessChecker(loader.tear_down) loader.add_timer(orphan_checker.check_orphan, time.time(), 1) loader.run(jobs)
def run(collector_cls, settings, checkpoint_cls=None, config_cls=None, log_suffix=None, single_instance=True, cc_json_file=None): """ Main loop. Run this TA forever """ ta_short_name = settings["meta"]["name"].lower() # This is for stdout flush utils.disable_stdout_buffer() # http://bugs.python.org/issue7980 time.strptime('2016-01-01', '%Y-%m-%d') loader = dl.create_data_loader() # handle signal _setup_signal_handler(loader, ta_short_name) # monitor files to reboot try: monitor = fm.FileMonitor(_handle_file_changes(loader), _get_conf_files(settings)) loader.add_timer(monitor.check_changes, time.time(), 10) except Exception: stulog.logger.exception("Fail to add files for monitoring") # add orphan process handling, which will check each 1 second orphan_checker = opm.OrphanProcessChecker(loader.tear_down) loader.add_timer(orphan_checker.check_orphan, time.time(), 1) tconfig = tc.create_ta_config(settings, config_cls or tc.TaConfig, log_suffix, single_instance=single_instance) task_configs = tconfig.get_task_configs() if not task_configs: stulog.logger.debug("No task and exiting...") return meta_config = tconfig.get_meta_config() meta_config["cc_json_file"] = cc_json_file if tconfig.is_shc_member(): # Don't support SHC env stulog.logger.error( "This host is in search head cluster environment , " "will exit.") return # In this case, use file for checkpoint if _is_checkpoint_dir_length_exceed_limit(tconfig, meta_config["checkpoint_dir"]): stulog.logger.error( "The length of the checkpoint directory path: '{}' " "is too long. The max length we support is {}", meta_config["checkpoint_dir"], __CHECKPOINT_DIR_MAX_LEN__) return jobs = [ tdc.create_data_collector(loader, tconfig, meta_config, task_config, collector_cls, checkpoint_cls=checkpoint_cls or cpmgr.TACheckPointMgr) for task_config in task_configs ] loader.run(jobs)
import threading import sys import json import traceback import time import copy import datetime from splunktalib.common import log logger = log.Logs().get_logger("util") import splunktalib.common.util as scutil import splunktalib.hec_config as hecc import splunktalib.rest as sr scutil.disable_stdout_buffer() class _ExtendedEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, datetime.datetime): return str(obj) return json.JSONEncoder.default(self, obj) class ModinputEvent(object): base_fmt = ("""<event{unbroken}>""" "<index>{index}</index>" "<host>{host}</host>" "<source>{source}</source>"
from splunktalib.common import log import pubsub_mod.google_pubsub_consts as gpc logger = log.Logs(gpc.pubsub_ns).get_logger("main") import splunktalib.common.util as utils import splunktalib.common.pattern as gcp import splunktalib.orphan_process_monitor as opm import google_concurrent_data_loader as gcdl import google_ta_common.ta_common as tacommon import pubsub_mod.google_pubsub_conf as psconf utils.remove_http_proxy_env_vars() utils.disable_stdout_buffer() def print_scheme(): title = "Splunk AddOn for Google" description = "Collect and index PubSub data for Google" tacommon.print_scheme(title, description) @gcp.catch_all(logger) def run(): """ Main loop. Run this TA forever """ logger.info("Start google_pubsub")
import threading import sys import json import traceback import time import splunktalib.common.util as scutil from splunktalib.common import log import splunktalib.rest as sr logger = log.Logs().get_logger("util") scutil.disable_stdout_buffer() class ModinputEvent(object): base_fmt = ("""<event{unbroken}>""" "<index>{index}</index>" "<host>{host}</host>" "<source>{source}</source>" "<sourcetype>{sourcetype}</sourcetype>" "<time>{time}</time>" "<data>{data}</data>{done}</event>") event_fmt = base_fmt.replace("{unbroken}", "").replace("{done}", "") unbroken_fmt = base_fmt.replace("{unbroken}", ' unbroken="1"').replace( "{done}", "")
import time import os import os.path as op import splunktalib.common.util as utils import splunktalib.modinput as modinput import splunktalib.file_monitor as fm import splunktalib.orphan_process_monitor as opm from splunktalib.common import log import kafka_config as kc import kafka_consts as c import kafka_concurrent_data_loader as kcdl utils.remove_http_proxy_env_vars() utils.disable_stdout_buffer() logger = log.Logs().get_logger("main") def do_scheme(): """ Feed splunkd the TA's scheme """ desc = ("Enable modular inputs to collect Kafka topic data. Use only if " "managing inputs manually from individual forwarders. " "See documentation for details.") print """ <scheme> <title>Splunk Add-on for {}</title>