def __init__(self, confd_path, agentConfig): self.confd_path = confd_path self.agentConfig = agentConfig self.logging_config = get_logging_config() self.check_frequency = DEFAULT_CHECK_FREQUENCY self.jmx_process = None
def __init__(self, config): parser = argparse.ArgumentParser() parser.add_argument('--from-cron', action='store_true') parser.add_argument('--debug', action='store_true') args = parser.parse_args() log_level = logging.INFO if args.from_cron: log_level = logging.WARN elif args.debug: log_level = logging.DEBUG log_format = config.get_log_format(self.__class__.__name__) logging.basicConfig( filename=config.get_logging_config().get('collector_log_file'), format=log_format, level=log_level, ) if not args.from_cron: console = logging.StreamHandler() console.setLevel(log_level) console.setFormatter(logging.Formatter(log_format)) logging.getLogger('').addHandler(console) try: agent_config = get_config() if 'api_key' in agent_config: api_key = agent_config['api_key'] except: api_key = os.environ['DD_API_KEY'] initialize(api_key=api_key)
def init6(agent_config=None, config_path=None, args=None): if (not agent_config['use_dogstatsd'] and (args and args[0] in ['start', 'restart'] or not args)): log.info("Dogstatsd is disabled. Exiting") # We're exiting purposefully, so exit with zero (supervisor's expected # code). HACK: Sleep a little bit so supervisor thinks we've started cleanly # and thus can exit cleanly. sleep(4) sys.exit(0) env = copy.deepcopy(os.environ) if agent_config.get('api_key'): env['DD_API_KEY'] = str(agent_config['api_key']) if agent_config.get('dogstatsd_port'): env['DD_DOGSTATSD_PORT'] = str(agent_config['dogstatsd_port']) if agent_config.get('dd_url'): env['DD_DD_URL'] = str(agent_config['dd_url']) if agent_config.get('non_local_traffic'): env['DD_DOGSTATSD_NON_LOCAL_TRAFFIC'] = str(agent_config['non_local_traffic']) if agent_config.get('dogstatsd_socket'): env['DD_DOGSTATSD_SOCKET'] = str(agent_config['dogstatsd_socket']) if agent_config.get('dogstatsd6_stats_port'): env['DD_DOGSTATSD_STATS_PORT'] = str(agent_config['dogstatsd6_stats_port']) env['DD_LOG_LEVEL'] = agent_config.get('log_level', 'info') env['DD_CONF_PATH'] = os.path.join( os.path.dirname(get_config_path(cfg_path=config_path)), "stackstate.yaml") # metadata is sent by the collector, disable it in dogstatsd6 to avoid sending conflicting metadata env['DD_ENABLE_METADATA_COLLECTION'] = 'false' legacy_dogstatsd_log = get_logging_config().get('dogstatsd_log_file') if legacy_dogstatsd_log: env['DD_LOG_FILE'] = os.path.join( os.path.dirname(legacy_dogstatsd_log), '{}.log'.format(Dogstatsd6.DSD6_BIN_NAME)) return Dogstatsd6._get_dsd6_path(), env
def init6(agent_config=None, config_path=None, args=None): if (not agent_config['use_sdstatsd'] and (args and args[0] in ['start', 'restart'] or not args)): log.info("Sdstatsd is disabled. Exiting") # We're exiting purposefully, so exit with zero (supervisor's expected # code). HACK: Sleep a little bit so supervisor thinks we've started cleanly # and thus can exit cleanly. sleep(4) sys.exit(0) env = copy.deepcopy(os.environ) if agent_config.get('agent_key'): env['SD_AGENT_KEY'] = str(agent_config['agent_key']) if agent_config.get('sdstatsd_port'): env['SD_SDSTATSD_PORT'] = str(agent_config['sdstatsd_port']) if agent_config.get('sd_url'): env['SD_SD_URL'] = str(agent_config['sd_url']) if agent_config.get('non_local_traffic'): env['SD_SDSTATSD_NON_LOCAL_TRAFFIC'] = str(agent_config['non_local_traffic']) if agent_config.get('sdstatsd_socket'): env['SD_SDSTATSD_SOCKET'] = str(agent_config['sdstatsd_socket']) if agent_config.get('sdstatsd6_stats_port'): env['SD_SDSTATSD_STATS_PORT'] = str(agent_config['sdstatsd6_stats_port']) env['SD_LOG_LEVEL'] = agent_config.get('log_level', 'info') env['SD_CONF_PATH'] = os.path.join( os.path.dirname(get_config_path(cfg_path=config_path)), "config.cfg") # metadata is sent by the collector, disable it in sdstatsd6 to avoid sending conflicting metadata env['SD_ENABLE_METADATA_COLLECTION'] = 'false' legacy_sdstatsd_log = get_logging_config().get('sdstatsd_log_file') if legacy_sdstatsd_log: env['SD_LOG_FILE'] = os.path.join( os.path.dirname(legacy_sdstatsd_log), '{}.log'.format(Sdstatsd6.SDSD6_BIN_NAME)) return Sdstatsd6._get_sdsd6_path(), env
def _save_logs_path(self): config = get_logging_config() self._collector_log = config.get('collector_log_file') self._forwarder_log = config.get('forwarder_log_file') self._dogstatsd_log = config.get('dogstatsd_log_file') self._jmxfetch_log = config.get('jmxfetch_log_file') self._gometro_log = config.get('go-metro_log_file') self._trace_agent_log = config.get('trace-agent_log_file')
def _save_logs_path(self): prefix = "" if Platform.is_windows(): prefix = "windows_" config = get_logging_config() self._collector_log = config.get("{0}collector_log_file".format(prefix)) self._forwarder_log = config.get("{0}forwarder_log_file".format(prefix)) self._dogstatsd_log = config.get("{0}dogstatsd_log_file".format(prefix)) self._jmxfetch_log = config.get("jmxfetch_log_file")
def __init__(self, confd_path, agentConfig): self.confd_path = confd_path self.agentConfig = agentConfig self.logging_config = get_logging_config() self.check_frequency = DEFAULT_CHECK_FREQUENCY self.service_discovery = _is_affirmative(self.agentConfig.get('sd_jmx_enable', False)) self.jmx_process = None self.jmx_checks = None
def _save_logs_path(self): prefix = '' if Platform.is_windows(): prefix = 'windows_' config = get_logging_config() self._collector_log = config.get('{0}collector_log_file'.format(prefix)) self._forwarder_log = config.get('{0}forwarder_log_file'.format(prefix)) self._dogstatsd_log = config.get('{0}dogstatsd_log_file'.format(prefix)) self._jmxfetch_log = config.get('jmxfetch_log_file')
def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) pid_file = PidFile('dogstatsd') self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "jmx_yamls")) JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15)
def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) pid_file = PidFile('dogstatsd') self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "jmx_yamls")) JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15, JMX_COLLECT_COMMAND)
def setUp(self): aggregator = MetricsAggregator("test_host") self.server = Server(aggregator, "localhost", STATSD_PORT) pid_file = PidFile('dogstatsd') self.reporter = DummyReporter(aggregator) self.t1 = threading.Thread(target=self.server.start) self.t1.start() confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml') JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15, JMX_COLLECT_COMMAND)
def __init__(self, parent=None): log_conf = get_logging_config() QSplitter.__init__(self, parent) self.setWindowTitle(MAIN_WINDOW_TITLE) self.setWindowIcon(get_icon("agent.svg")) self.sysTray = SystemTray(self) self.connect(self.sysTray, SIGNAL("activated(QSystemTrayIcon::ActivationReason)"), self.__icon_activated) checks = get_checks() datadog_conf = DatadogConf(get_config_path()) self.create_logs_files_windows(log_conf) listwidget = QListWidget(self) listwidget.addItems([ osp.basename(check.module_name).replace("_", " ").title() for check in checks ]) self.properties = PropertiesWidget(self) self.setting_button = QPushButton(get_icon("info.png"), "Logs and Status", self) self.menu_button = QPushButton(get_icon("settings.png"), "Actions", self) self.settings = [ ("Forwarder Logs", lambda: [ self.properties.set_log_file(self.forwarder_log_file), self.show_html(self.properties.group_code, self.properties. html_window, False) ]), ("Collector Logs", lambda: [ self.properties.set_log_file(self.collector_log_file), self.show_html(self.properties.group_code, self.properties. html_window, False) ]), ("Dogstatsd Logs", lambda: [ self.properties.set_log_file(self.dogstatsd_log_file), self.show_html(self.properties.group_code, self.properties. html_window, False) ]), ("JMX Fetch Logs", lambda: [ self.properties.set_log_file(self.jmxfetch_log_file), self.show_html(self.properties.group_code, self.properties. html_window, False) ]), ] if Platform.is_windows(): self.settings.extend([ ("Service Logs", lambda: [ self.properties.set_log_file(self.service_log_file), self.show_html(self.properties.group_code, self.properties. html_window, False) ]), ]) self.settings.extend([ ("Agent Status", lambda: [ self.properties.html_window.setHtml(self.properties.html_window .latest_status()), self.show_html(self.properties.group_code, self.properties. html_window, True), self.properties.set_status() ]), ]) self.agent_settings = QPushButton(get_icon("edit.png"), "Settings", self) self.connect( self.agent_settings, SIGNAL("clicked()"), lambda: [ self.properties.set_datadog_conf(datadog_conf), self.show_html(self.properties.group_code, self.properties. html_window, False) ]) self.setting_menu = SettingMenu(self.settings) self.connect( self.setting_button, SIGNAL("clicked()"), lambda: self.setting_menu.popup( self.setting_button.mapToGlobal(QPoint(0, 0)))) self.manager_menu = Menu(self) self.connect( self.menu_button, SIGNAL("clicked()"), lambda: self.manager_menu.popup( self.menu_button.mapToGlobal(QPoint(0, 0)))) holdingBox = QGroupBox("", self) Box = QVBoxLayout(self) Box.addWidget(self.agent_settings) Box.addWidget(self.setting_button) Box.addWidget(self.menu_button) Box.addWidget(listwidget) holdingBox.setLayout(Box) self.addWidget(holdingBox) self.addWidget(self.properties) self.connect(self.properties.enable_button, SIGNAL("clicked()"), lambda: enable_check(self.properties)) self.connect(self.properties.disable_button, SIGNAL("clicked()"), lambda: disable_check(self.properties)) self.connect(self.properties.save_button, SIGNAL("clicked()"), lambda: save_file(self.properties)) self.connect( self.properties.refresh_button, SIGNAL("clicked()"), lambda: [ self.properties.set_log_file(self.properties.current_file), self.properties.html_window.setHtml(self.properties.html_window .latest_status()) ]) self.connect( listwidget, SIGNAL('currentRowChanged(int)'), lambda row: [ self.properties.set_item(checks[row]), self.show_html(self.properties.group_code, self.properties. html_window, False) ]) listwidget.setCurrentRow(0) self.setSizes([150, 1]) self.setStretchFactor(1, 1) self.resize(QSize(950, 600)) self.properties.set_datadog_conf(datadog_conf) self.do_refresh()
print "#" * 80 print "JMX tool to be used to help configuring your JMX checks." print "See http://docs.datadoghq.com/integrations/java/ for more information" print "#" * 80 print "\n" print "You have to specify one of the following commands:" for command, desc in JMX_LIST_COMMANDS.iteritems(): print " - %s [OPTIONAL: LIST OF CHECKS]: %s" % (command, desc) print "Example: sudo /etc/init.d/datadog-agent jmx list_matching_attributes tomcat jmx solr" print "\n" else: jmx_command = args[1] checks_list = args[2:] confd_directory = get_confd_path(get_os()) should_run = JMXFetch.init(confd_directory, agentConfig, get_logging_config(), 15, jmx_command, checks_list, reporter="console") if not should_run: print "Couldn't find any valid JMX configuration in your conf.d directory: %s" % confd_directory print "Have you enabled any JMX check ?" print "If you think it's not normal please get in touch with Datadog Support" return 0 if __name__ == '__main__': try: sys.exit(main()) except StandardError: # Try our best to log the error. try:
import time import os import sys import subprocess import re import socket import logging import logging.config import numpy as np import tensorflow as tf slim = tf.contrib.slim logging.config.dictConfig(get_logging_config(args.run_name)) log = logging.getLogger() train_dir = CKPT_ROOT + args.run_name pretrain_dir = CKPT_ROOT + args.pretrained_net def extract_batch(data_provider, classes): with tf.device("/cpu:0"): im, bbox, gt, proposals = data_provider.get(['image', 'object/bbox', 'object/label', 'proposal/bbox']) im = tf.to_float(im)/255 gt = tf.to_int32(gt) gt_mask = tf.reduce_any(tf.equal(tf.expand_dims(gt, 1), tf.expand_dims(tf.constant(classes), 0)), axis=1) gt = tf.boolean_mask(gt, gt_mask) bbox = tf.boolean_mask(bbox, gt_mask)
import logging import logging.config import time from config import get_logging_config, args, evaluation_logfile, train_dir from config import config as net_config from paths import CKPT_ROOT from utils import decode_bboxes, batch_iou from skimage.transform import resize as imresize from resnet import ResNet from boxer import PriorBoxGrid from voc_loader import VOCLoader logging.config.dictConfig(get_logging_config(args.run_name)) log = logging.getLogger() def main(argv=None): # pylint: disable=unused-argument net = ResNet depth = 50 loader = VOCLoader('07', 'test') net = net(config=net_config, depth=depth, training=False) num_classes = 21 batch_size = args.batch_size img_size = args.image_size image_ph = tf.placeholder(shape=[1, img_size, img_size, 3],
def run(self): handlers = [ (r"/intake/?", AgentInputHandler), (r"/intake/metrics?", MetricsAgentInputHandler), (r"/intake/metadata?", MetadataAgentInputHandler), (r"/api/v1/series/?", ApiInputHandler), (r"/api/v1/check_run/?", ApiCheckRunHandler), (r"/status/?", StatusHandler), ] settings = dict( cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", xsrf_cookies=False, debug=False, log_function=self.log_request ) non_local_traffic = self._agentConfig.get("non_local_traffic", False) tornado.web.Application.__init__(self, handlers, **settings) http_server = tornado.httpserver.HTTPServer(self) try: # non_local_traffic must be == True to match, not just some non-false value if non_local_traffic is True: http_server.listen(self._port) else: # localhost in lieu of 127.0.0.1 to support IPv6 try: http_server.listen(self._port, address=self._agentConfig['bind_host']) except gaierror: log.warning("localhost seems undefined in your host file, using 127.0.0.1 instead") http_server.listen(self._port, address="127.0.0.1") except socket_error as e: if "Errno 99" in str(e): log.warning("IPv6 doesn't seem to be fully supported. Falling back to IPv4") http_server.listen(self._port, address="127.0.0.1") else: raise except socket_error as e: log.exception("Socket error %s. Is another application listening on the same port ? Exiting", e) sys.exit(1) except Exception as e: log.exception("Uncaught exception. Forwarder is exiting.") sys.exit(1) log.info("Listening on port %d" % self._port) # Register callbacks self.mloop = get_tornado_ioloop() logging.getLogger().setLevel(get_logging_config()['log_level'] or logging.INFO) def flush_trs(): if self._watchdog: self._watchdog.reset() self._postMetrics() self._tr_manager.flush() tr_sched = tornado.ioloop.PeriodicCallback(flush_trs, TRANSACTION_FLUSH_INTERVAL, io_loop=self.mloop) # Register optional Graphite listener gport = self._agentConfig.get("graphite_listen_port", None) if gport is not None: log.info("Starting graphite listener on port %s" % gport) from graphite import GraphiteServer gs = GraphiteServer(self, get_hostname(self._agentConfig), io_loop=self.mloop) if non_local_traffic is True: gs.listen(gport) else: gs.listen(gport, address="localhost") # Start everything if self._watchdog: self._watchdog.reset() tr_sched.start() self.mloop.start() log.info("Stopped")
def run(self): handlers = [ (r"/intake/?", AgentInputHandler), (r"/api/v1/series/?", ApiInputHandler), (r"/status/?", StatusHandler), ] settings = dict( cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", xsrf_cookies=False, debug=False, log_function=self.log_request ) non_local_traffic = self._agentConfig.get("non_local_traffic", False) tornado.web.Application.__init__(self, handlers, **settings) http_server = tornado.httpserver.HTTPServer(self) # non_local_traffic must be == True to match, not just some non-false value if non_local_traffic is True: http_server.listen(self._port) else: # localhost in lieu of 127.0.0.1 to support IPv6 try: http_server.listen(self._port, address = "localhost") except gaierror: log.warning("Warning localhost seems undefined in your host file, using 127.0.0.1 instead") http_server.listen(self._port, address = "127.0.0.1") log.info("Listening on port %d" % self._port) # Register callbacks self.mloop = tornado.ioloop.IOLoop.instance() logging.getLogger().setLevel(get_logging_config()['log_level'] or logging.INFO) def flush_trs(): if self._watchdog: self._watchdog.reset() self._postMetrics() self._tr_manager.flush() tr_sched = tornado.ioloop.PeriodicCallback(flush_trs,TRANSACTION_FLUSH_INTERVAL, io_loop = self.mloop) # Register optional Graphite listener gport = self._agentConfig.get("graphite_listen_port", None) if gport is not None: log.info("Starting graphite listener on port %s" % gport) from graphite import GraphiteServer gs = GraphiteServer(self, get_hostname(self._agentConfig), io_loop=self.mloop) if non_local_traffic is True: gs.listen(gport) else: gs.listen(gport, address = "localhost") # Start everything if self._watchdog: self._watchdog.reset() tr_sched.start() self.mloop.start() log.info("Stopped")
def __init__(self): self.logging_config = get_logging_config() self._process = None self._running = True
print "All yaml files passed. You can now run the Datadog agent." else: print("Fix the invalid yaml files above in order to start the Datadog agent. " "A useful external tool for yaml parsing can be found at " "http://yaml-online-parser.appspot.com/") elif 'jmx' == command: from jmxfetch import JMX_LIST_COMMANDS, JMXFetch if len(args) < 2 or args[1] not in JMX_LIST_COMMANDS: print "You have to specify one of the following command %s" % JMX_LIST_COMMANDS else: jmx_command = args[1] from config import get_confd_path, get_logging_config from util import get_os JMXFetch.init(get_confd_path(get_os()), agentConfig, get_logging_config(), 15, jmx_command) return 0 if __name__ == '__main__': try: sys.exit(main()) except StandardError: # Try our best to log the error. try: log.exception("Uncaught error running the Agent") except Exception: pass raise
print "#" * 80 print "\n" print "You have to specify one of the following command:" for command, desc in JMX_LIST_COMMANDS.iteritems(): print " - %s [OPTIONAL: LIST OF CHECKS]: %s" % (command, desc) print "Example: sudo /etc/init.d/datadog-agent jmx list_matching_attributes tomcat jmx solr" print "\n" else: jmx_command = args[1] checks_list = args[2:] confd_directory = get_confd_path(get_os()) should_run = JMXFetch.init(confd_directory, agentConfig, get_logging_config(), 15, jmx_command, checks_list, reporter="console") if not should_run: print "Couldn't find any valid JMX configuration in your conf.d directory: %s" % confd_directory print "Have you enabled any JMX check ?" print "If you think it's not normal please get in touch with Datadog Support" return 0 if __name__ == '__main__': try: sys.exit(main())
def __init__(self, parent=None): prefix_conf = '' if Platform.is_windows(): prefix_conf = 'windows_' log_conf = get_logging_config() QSplitter.__init__(self, parent) self.setWindowTitle(MAIN_WINDOW_TITLE) self.setWindowIcon(get_icon("agent.svg")) self.sysTray = SystemTray(self) self.connect(self.sysTray, SIGNAL("activated(QSystemTrayIcon::ActivationReason)"), self.__icon_activated) checks = get_checks() datadog_conf = DatadogConf(get_config_path()) self.create_logs_files_windows(log_conf, prefix_conf) listwidget = QListWidget(self) listwidget.addItems([osp.basename(check.module_name).replace("_", " ").title() for check in checks]) self.properties = PropertiesWidget(self) self.setting_button = QPushButton(get_icon("info.png"), "Logs and Status", self) self.menu_button = QPushButton(get_icon("settings.png"), "Actions", self) self.settings = [ ("Forwarder Logs", lambda: [self.properties.set_log_file(self.forwarder_log_file), self.show_html(self.properties.group_code, self.properties.html_window, False)]), ("Collector Logs", lambda: [self.properties.set_log_file(self.collector_log_file), self.show_html(self.properties.group_code, self.properties.html_window, False)]), ("Dogstatsd Logs", lambda: [self.properties.set_log_file(self.dogstatsd_log_file), self.show_html(self.properties.group_code, self.properties.html_window, False)]), ("JMX Fetch Logs", lambda: [self.properties.set_log_file(self.jmxfetch_log_file), self.show_html(self.properties.group_code, self.properties.html_window, False)]), ("Agent Status", lambda: [self.properties.html_window.setHtml(self.properties.html_window.latest_status()), self.show_html(self.properties.group_code, self.properties.html_window, True), self.properties.set_status()]), ] self.agent_settings = QPushButton(get_icon("edit.png"), "Settings", self) self.connect(self.agent_settings, SIGNAL("clicked()"), lambda: [self.properties.set_datadog_conf(datadog_conf), self.show_html(self.properties.group_code, self.properties.html_window, False)]) self.setting_menu = SettingMenu(self.settings) self.connect(self.setting_button, SIGNAL("clicked()"), lambda: self.setting_menu.popup(self.setting_button.mapToGlobal(QPoint(0, 0)))) self.manager_menu = Menu(self) self.connect(self.menu_button, SIGNAL("clicked()"), lambda: self.manager_menu.popup(self.menu_button.mapToGlobal(QPoint(0, 0)))) holdingBox = QGroupBox("", self) Box = QVBoxLayout(self) Box.addWidget(self.agent_settings) Box.addWidget(self.setting_button) Box.addWidget(self.menu_button) Box.addWidget(listwidget) holdingBox.setLayout(Box) self.addWidget(holdingBox) self.addWidget(self.properties) self.connect(self.properties.enable_button, SIGNAL("clicked()"), lambda: enable_check(self.properties)) self.connect(self.properties.disable_button, SIGNAL("clicked()"), lambda: disable_check(self.properties)) self.connect(self.properties.save_button, SIGNAL("clicked()"), lambda: save_file(self.properties)) self.connect(self.properties.refresh_button, SIGNAL("clicked()"), lambda: [self.properties.set_log_file(self.properties.current_file), self.properties.html_window.setHtml(self.properties.html_window.latest_status())]) self.connect(listwidget, SIGNAL('currentRowChanged(int)'), lambda row: [self.properties.set_item(checks[row]), self.show_html(self.properties.group_code, self.properties.html_window, False)]) listwidget.setCurrentRow(0) self.setSizes([150, 1]) self.setStretchFactor(1, 1) self.resize(QSize(950, 600)) self.properties.set_datadog_conf(datadog_conf) self.do_refresh()
def run(self): handlers = [ (r"/intake/?", AgentInputHandler), (r"/intake/metrics?", MetricsAgentInputHandler), (r"/intake/metadata?", MetadataAgentInputHandler), (r"/api/v1/series/?", ApiInputHandler), (r"/api/v1/check_run/?", ApiCheckRunHandler), (r"/status/?", StatusHandler), ] settings = dict( cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", xsrf_cookies=False, debug=False, log_function=self.log_request) non_local_traffic = self._agentConfig.get("non_local_traffic", False) tornado.web.Application.__init__(self, handlers, **settings) http_server = tornado.httpserver.HTTPServer(self) try: # non_local_traffic must be == True to match, not just some non-false value if non_local_traffic is True: http_server.listen(self._port) else: # localhost in lieu of 127.0.0.1 to support IPv6 try: http_server.listen(self._port, address=self._agentConfig['bind_host']) except gaierror: log.warning( "localhost seems undefined in your host file, using 127.0.0.1 instead" ) http_server.listen(self._port, address="127.0.0.1") except socket_error as e: if "Errno 99" in str(e): log.warning( "IPv6 doesn't seem to be fully supported. Falling back to IPv4" ) http_server.listen(self._port, address="127.0.0.1") else: raise except socket_error as e: log.exception( "Socket error %s. Is another application listening on the same port ? Exiting", e) sys.exit(1) except Exception as e: log.exception("Uncaught exception. Forwarder is exiting.") sys.exit(1) log.info("Listening on port %d" % self._port) # Register callbacks self.mloop = tornado.ioloop.IOLoop.current() logging.getLogger().setLevel(get_logging_config()['log_level'] or logging.INFO) def flush_trs(): if self._watchdog: self._watchdog.reset() self._postMetrics() self._tr_manager.flush() tr_sched = tornado.ioloop.PeriodicCallback(flush_trs, TRANSACTION_FLUSH_INTERVAL, io_loop=self.mloop) # Register optional Graphite listener gport = self._agentConfig.get("graphite_listen_port", None) if gport is not None: log.info("Starting graphite listener on port %s" % gport) from graphite import GraphiteServer gs = GraphiteServer(self, get_hostname(self._agentConfig), io_loop=self.mloop) if non_local_traffic is True: gs.listen(gport) else: gs.listen(gport, address="localhost") # Start everything if self._watchdog: self._watchdog.reset() tr_sched.start() self.mloop.start() log.info("Stopped")
import tornado.httpserver import tornado.ioloop import tornado.web from tornado.escape import json_decode from tornado.options import define, parse_command_line, options # agent import from util import Watchdog, get_uuid, get_hostname, json from emitter import http_emitter, format_body from config import get_config from checks.check_status import ForwarderStatus from transaction import Transaction, TransactionManager import modules log = logging.getLogger("forwarder") log.setLevel(get_logging_config()["log_level"] or logging.INFO) TRANSACTION_FLUSH_INTERVAL = 5000 # Every 5 seconds WATCHDOG_INTERVAL_MULTIPLIER = 10 # 10x flush interval # Maximum delay before replaying a transaction MAX_WAIT_FOR_REPLAY = timedelta(seconds=90) # Maximum queue size in bytes (when this is reached, old messages are dropped) MAX_QUEUE_SIZE = 30 * 1024 * 1024 # 30MB THROTTLING_DELAY = timedelta(microseconds=1000000 / 2) # 2 msg/second class EmitterThread(threading.Thread): def __init__(self, *args, **kwargs):
) import modules from transaction import Transaction, TransactionManager from util import ( get_hostname, get_tornado_ioloop, get_uuid, json, Watchdog, ) from utils.logger import RedactedLogRecord logging.LogRecord = RedactedLogRecord log = logging.getLogger('forwarder') log.setLevel(get_logging_config()['log_level'] or logging.INFO) DD_ENDPOINT = "dd_url" TRANSACTION_FLUSH_INTERVAL = 5000 # Every 5 seconds WATCHDOG_INTERVAL_MULTIPLIER = 10 # 10x flush interval HEADERS_TO_REMOVE = [ 'Host', 'Content-Length', ] # Maximum delay before replaying a transaction MAX_WAIT_FOR_REPLAY = timedelta(seconds=90) # Maximum queue size in bytes (when this is reached, old messages are dropped)
def run(self): handlers = [ (r"/intake/?", AgentInputHandler), (r"/api/v1/series/?", ApiInputHandler), (r"/status/?", StatusHandler), ] settings = dict( cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", xsrf_cookies=False, debug=False, ) non_local_traffic = self._agentConfig.get("non_local_traffic", False) tornado.web.Application.__init__(self, handlers, **settings) http_server = tornado.httpserver.HTTPServer(self) # set the root logger to warn so tornado is less chatty logging.getLogger().setLevel(logging.WARNING) # but keep the forwarder logger at the original level forwarder_logger = logging.getLogger('forwarder') log_config = get_logging_config() forwarder_logger.setLevel(log_config['log_level'] or logging.INFO) # non_local_traffic must be == True to match, not just some non-false value if non_local_traffic is True: http_server.listen(self._port) else: # localhost in lieu of 127.0.0.1 to support IPv6 try: http_server.listen(self._port, address = "localhost") except gaierror: log.warning("Warning localhost seems undefined in your host file, using 127.0.0.1 instead") http_server.listen(self._port, address = "127.0.0.1") log.info("Listening on port %d" % self._port) # Register callbacks self.mloop = tornado.ioloop.IOLoop.instance() def flush_trs(): if self._watchdog: self._watchdog.reset() self._postMetrics() self._tr_manager.flush() tr_sched = tornado.ioloop.PeriodicCallback(flush_trs,TRANSACTION_FLUSH_INTERVAL, io_loop = self.mloop) # Register optional Graphite listener gport = self._agentConfig.get("graphite_listen_port", None) if gport is not None: log.info("Starting graphite listener on port %s" % gport) from graphite import GraphiteServer gs = GraphiteServer(self, gethostname(self._agentConfig), io_loop=self.mloop) if non_local_traffic is True: gs.listen(gport) else: gs.listen(gport, address = "localhost") # Start everything if self._watchdog: self._watchdog.reset() tr_sched.start() self.mloop.start() log.info("Stopped")