Exemplo n.º 1
0
    def __init__(self, confd_path, agentConfig):
        self.confd_path = confd_path
        self.agentConfig = agentConfig
        self.logging_config = get_logging_config()
        self.check_frequency = DEFAULT_CHECK_FREQUENCY

        self.jmx_process = None
Exemplo n.º 2
0
    def __init__(self, config):
        parser = argparse.ArgumentParser()
        parser.add_argument('--from-cron', action='store_true')
        parser.add_argument('--debug', action='store_true')
        args = parser.parse_args()

        log_level = logging.INFO
        if args.from_cron:
            log_level = logging.WARN
        elif args.debug:
            log_level = logging.DEBUG

        log_format = config.get_log_format(self.__class__.__name__)
        logging.basicConfig(
            filename=config.get_logging_config().get('collector_log_file'),
            format=log_format,
            level=log_level,
        )

        if not args.from_cron:
            console = logging.StreamHandler()
            console.setLevel(log_level)
            console.setFormatter(logging.Formatter(log_format))
            logging.getLogger('').addHandler(console)

        try:
            agent_config = get_config()
            if 'api_key' in agent_config:
                api_key = agent_config['api_key']
        except:
            api_key = os.environ['DD_API_KEY']

        initialize(api_key=api_key)
Exemplo n.º 3
0
def init6(agent_config=None, config_path=None, args=None):
    if (not agent_config['use_dogstatsd'] and
            (args and args[0] in ['start', 'restart'] or not args)):
        log.info("Dogstatsd is disabled. Exiting")
        # We're exiting purposefully, so exit with zero (supervisor's expected
        # code). HACK: Sleep a little bit so supervisor thinks we've started cleanly
        # and thus can exit cleanly.
        sleep(4)
        sys.exit(0)

    env = copy.deepcopy(os.environ)
    if agent_config.get('api_key'):
        env['DD_API_KEY'] = str(agent_config['api_key'])
    if agent_config.get('dogstatsd_port'):
        env['DD_DOGSTATSD_PORT'] = str(agent_config['dogstatsd_port'])
    if agent_config.get('dd_url'):
        env['DD_DD_URL'] = str(agent_config['dd_url'])
    if agent_config.get('non_local_traffic'):
        env['DD_DOGSTATSD_NON_LOCAL_TRAFFIC'] = str(agent_config['non_local_traffic'])
    if agent_config.get('dogstatsd_socket'):
        env['DD_DOGSTATSD_SOCKET'] = str(agent_config['dogstatsd_socket'])
    if agent_config.get('dogstatsd6_stats_port'):
        env['DD_DOGSTATSD_STATS_PORT'] = str(agent_config['dogstatsd6_stats_port'])
    env['DD_LOG_LEVEL'] = agent_config.get('log_level', 'info')
    env['DD_CONF_PATH'] = os.path.join(
        os.path.dirname(get_config_path(cfg_path=config_path)), "stackstate.yaml")
    # metadata is sent by the collector, disable it in dogstatsd6 to avoid sending conflicting metadata
    env['DD_ENABLE_METADATA_COLLECTION'] = 'false'

    legacy_dogstatsd_log = get_logging_config().get('dogstatsd_log_file')
    if legacy_dogstatsd_log:
        env['DD_LOG_FILE'] = os.path.join(
            os.path.dirname(legacy_dogstatsd_log), '{}.log'.format(Dogstatsd6.DSD6_BIN_NAME))

    return Dogstatsd6._get_dsd6_path(), env
Exemplo n.º 4
0
    def __init__(self, confd_path, agentConfig):
        self.confd_path = confd_path
        self.agentConfig = agentConfig
        self.logging_config = get_logging_config()
        self.check_frequency = DEFAULT_CHECK_FREQUENCY

        self.jmx_process = None
Exemplo n.º 5
0
def init6(agent_config=None, config_path=None, args=None):
    if (not agent_config['use_sdstatsd'] and
            (args and args[0] in ['start', 'restart'] or not args)):
        log.info("Sdstatsd is disabled. Exiting")
        # We're exiting purposefully, so exit with zero (supervisor's expected
        # code). HACK: Sleep a little bit so supervisor thinks we've started cleanly
        # and thus can exit cleanly.
        sleep(4)
        sys.exit(0)

    env = copy.deepcopy(os.environ)
    if agent_config.get('agent_key'):
        env['SD_AGENT_KEY'] = str(agent_config['agent_key'])
    if agent_config.get('sdstatsd_port'):
        env['SD_SDSTATSD_PORT'] = str(agent_config['sdstatsd_port'])
    if agent_config.get('sd_url'):
        env['SD_SD_URL'] = str(agent_config['sd_url'])
    if agent_config.get('non_local_traffic'):
        env['SD_SDSTATSD_NON_LOCAL_TRAFFIC'] = str(agent_config['non_local_traffic'])
    if agent_config.get('sdstatsd_socket'):
        env['SD_SDSTATSD_SOCKET'] = str(agent_config['sdstatsd_socket'])
    if agent_config.get('sdstatsd6_stats_port'):
        env['SD_SDSTATSD_STATS_PORT'] = str(agent_config['sdstatsd6_stats_port'])
    env['SD_LOG_LEVEL'] = agent_config.get('log_level', 'info')
    env['SD_CONF_PATH'] = os.path.join(
        os.path.dirname(get_config_path(cfg_path=config_path)), "config.cfg")
    # metadata is sent by the collector, disable it in sdstatsd6 to avoid sending conflicting metadata
    env['SD_ENABLE_METADATA_COLLECTION'] = 'false'

    legacy_sdstatsd_log = get_logging_config().get('sdstatsd_log_file')
    if legacy_sdstatsd_log:
        env['SD_LOG_FILE'] = os.path.join(
            os.path.dirname(legacy_sdstatsd_log), '{}.log'.format(Sdstatsd6.SDSD6_BIN_NAME))

    return Sdstatsd6._get_sdsd6_path(), env
Exemplo n.º 6
0
 def _save_logs_path(self):
     config = get_logging_config()
     self._collector_log = config.get('collector_log_file')
     self._forwarder_log = config.get('forwarder_log_file')
     self._dogstatsd_log = config.get('dogstatsd_log_file')
     self._jmxfetch_log = config.get('jmxfetch_log_file')
     self._gometro_log = config.get('go-metro_log_file')
     self._trace_agent_log = config.get('trace-agent_log_file')
Exemplo n.º 7
0
 def _save_logs_path(self):
     config = get_logging_config()
     self._collector_log = config.get('collector_log_file')
     self._forwarder_log = config.get('forwarder_log_file')
     self._dogstatsd_log = config.get('dogstatsd_log_file')
     self._jmxfetch_log = config.get('jmxfetch_log_file')
     self._gometro_log = config.get('go-metro_log_file')
     self._trace_agent_log = config.get('trace-agent_log_file')
Exemplo n.º 8
0
 def _save_logs_path(self):
     prefix = ""
     if Platform.is_windows():
         prefix = "windows_"
     config = get_logging_config()
     self._collector_log = config.get("{0}collector_log_file".format(prefix))
     self._forwarder_log = config.get("{0}forwarder_log_file".format(prefix))
     self._dogstatsd_log = config.get("{0}dogstatsd_log_file".format(prefix))
     self._jmxfetch_log = config.get("jmxfetch_log_file")
Exemplo n.º 9
0
    def __init__(self, confd_path, agentConfig):
        self.confd_path = confd_path
        self.agentConfig = agentConfig
        self.logging_config = get_logging_config()
        self.check_frequency = DEFAULT_CHECK_FREQUENCY
        self.service_discovery = _is_affirmative(self.agentConfig.get('sd_jmx_enable', False))

        self.jmx_process = None
        self.jmx_checks = None
Exemplo n.º 10
0
 def _save_logs_path(self):
     prefix = ''
     if Platform.is_windows():
         prefix = 'windows_'
     config = get_logging_config()
     self._collector_log = config.get('{0}collector_log_file'.format(prefix))
     self._forwarder_log = config.get('{0}forwarder_log_file'.format(prefix))
     self._dogstatsd_log = config.get('{0}dogstatsd_log_file'.format(prefix))
     self._jmxfetch_log = config.get('jmxfetch_log_file')
Exemplo n.º 11
0
    def __init__(self, confd_path, agentConfig):
        self.confd_path = confd_path
        self.agentConfig = agentConfig
        self.logging_config = get_logging_config()
        self.check_frequency = DEFAULT_CHECK_FREQUENCY
        self.service_discovery = _is_affirmative(self.agentConfig.get('sd_jmx_enable', False))

        self.jmx_process = None
        self.jmx_checks = None
Exemplo n.º 12
0
 def _save_logs_path(self):
     prefix = ''
     if Platform.is_windows():
         prefix = 'windows_'
     config = get_logging_config()
     self._collector_log = config.get('{0}collector_log_file'.format(prefix))
     self._forwarder_log = config.get('{0}forwarder_log_file'.format(prefix))
     self._dogstatsd_log = config.get('{0}dogstatsd_log_file'.format(prefix))
     self._jmxfetch_log = config.get('jmxfetch_log_file')
Exemplo n.º 13
0
    def setUp(self):
        aggregator = MetricsAggregator("test_host")
        self.server = Server(aggregator, "localhost", STATSD_PORT)
        pid_file = PidFile('dogstatsd')
        self.reporter = DummyReporter(aggregator)
        
        self.t1 = threading.Thread(target=self.server.start)
        self.t1.start()

        confd_path = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "jmx_yamls"))
        JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15)
Exemplo n.º 14
0
    def setUp(self):
        aggregator = MetricsAggregator("test_host")
        self.server = Server(aggregator, "localhost", STATSD_PORT)
        pid_file = PidFile('dogstatsd')
        self.reporter = DummyReporter(aggregator)
        
        self.t1 = threading.Thread(target=self.server.start)
        self.t1.start()

        confd_path = os.path.realpath(os.path.join(os.path.abspath(__file__), "..", "jmx_yamls"))
        JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15, JMX_COLLECT_COMMAND)
Exemplo n.º 15
0
    def setUp(self):
        aggregator = MetricsAggregator("test_host")
        self.server = Server(aggregator, "localhost", STATSD_PORT)
        pid_file = PidFile('dogstatsd')
        self.reporter = DummyReporter(aggregator)

        self.t1 = threading.Thread(target=self.server.start)
        self.t1.start()

        confd_path = os.path.join(os.environ['VOLATILE_DIR'], 'jmx_yaml')
        JMXFetch.init(confd_path, {'dogstatsd_port':STATSD_PORT}, get_logging_config(), 15, JMX_COLLECT_COMMAND)
Exemplo n.º 16
0
    def __init__(self, parent=None):
        log_conf = get_logging_config()

        QSplitter.__init__(self, parent)
        self.setWindowTitle(MAIN_WINDOW_TITLE)
        self.setWindowIcon(get_icon("agent.svg"))

        self.sysTray = SystemTray(self)

        self.connect(self.sysTray,
                     SIGNAL("activated(QSystemTrayIcon::ActivationReason)"),
                     self.__icon_activated)

        checks = get_checks()
        datadog_conf = DatadogConf(get_config_path())
        self.create_logs_files_windows(log_conf)

        listwidget = QListWidget(self)
        listwidget.addItems([
            osp.basename(check.module_name).replace("_", " ").title()
            for check in checks
        ])

        self.properties = PropertiesWidget(self)

        self.setting_button = QPushButton(get_icon("info.png"),
                                          "Logs and Status", self)
        self.menu_button = QPushButton(get_icon("settings.png"), "Actions",
                                       self)
        self.settings = [
            ("Forwarder Logs", lambda: [
                self.properties.set_log_file(self.forwarder_log_file),
                self.show_html(self.properties.group_code, self.properties.
                               html_window, False)
            ]),
            ("Collector Logs", lambda: [
                self.properties.set_log_file(self.collector_log_file),
                self.show_html(self.properties.group_code, self.properties.
                               html_window, False)
            ]),
            ("Dogstatsd Logs", lambda: [
                self.properties.set_log_file(self.dogstatsd_log_file),
                self.show_html(self.properties.group_code, self.properties.
                               html_window, False)
            ]),
            ("JMX Fetch Logs", lambda: [
                self.properties.set_log_file(self.jmxfetch_log_file),
                self.show_html(self.properties.group_code, self.properties.
                               html_window, False)
            ]),
        ]

        if Platform.is_windows():
            self.settings.extend([
                ("Service Logs", lambda: [
                    self.properties.set_log_file(self.service_log_file),
                    self.show_html(self.properties.group_code, self.properties.
                                   html_window, False)
                ]),
            ])

        self.settings.extend([
            ("Agent Status", lambda: [
                self.properties.html_window.setHtml(self.properties.html_window
                                                    .latest_status()),
                self.show_html(self.properties.group_code, self.properties.
                               html_window, True),
                self.properties.set_status()
            ]),
        ])

        self.agent_settings = QPushButton(get_icon("edit.png"), "Settings",
                                          self)
        self.connect(
            self.agent_settings, SIGNAL("clicked()"), lambda: [
                self.properties.set_datadog_conf(datadog_conf),
                self.show_html(self.properties.group_code, self.properties.
                               html_window, False)
            ])

        self.setting_menu = SettingMenu(self.settings)
        self.connect(
            self.setting_button, SIGNAL("clicked()"),
            lambda: self.setting_menu.popup(
                self.setting_button.mapToGlobal(QPoint(0, 0))))

        self.manager_menu = Menu(self)
        self.connect(
            self.menu_button, SIGNAL("clicked()"),
            lambda: self.manager_menu.popup(
                self.menu_button.mapToGlobal(QPoint(0, 0))))

        holdingBox = QGroupBox("", self)
        Box = QVBoxLayout(self)
        Box.addWidget(self.agent_settings)
        Box.addWidget(self.setting_button)
        Box.addWidget(self.menu_button)
        Box.addWidget(listwidget)
        holdingBox.setLayout(Box)

        self.addWidget(holdingBox)
        self.addWidget(self.properties)

        self.connect(self.properties.enable_button, SIGNAL("clicked()"),
                     lambda: enable_check(self.properties))

        self.connect(self.properties.disable_button, SIGNAL("clicked()"),
                     lambda: disable_check(self.properties))

        self.connect(self.properties.save_button, SIGNAL("clicked()"),
                     lambda: save_file(self.properties))

        self.connect(
            self.properties.refresh_button, SIGNAL("clicked()"), lambda: [
                self.properties.set_log_file(self.properties.current_file),
                self.properties.html_window.setHtml(self.properties.html_window
                                                    .latest_status())
            ])

        self.connect(
            listwidget, SIGNAL('currentRowChanged(int)'), lambda row: [
                self.properties.set_item(checks[row]),
                self.show_html(self.properties.group_code, self.properties.
                               html_window, False)
            ])

        listwidget.setCurrentRow(0)

        self.setSizes([150, 1])
        self.setStretchFactor(1, 1)
        self.resize(QSize(950, 600))
        self.properties.set_datadog_conf(datadog_conf)

        self.do_refresh()
Exemplo n.º 17
0
            print "#" * 80
            print "JMX tool to be used to help configuring your JMX checks."
            print "See http://docs.datadoghq.com/integrations/java/ for more information"
            print "#" * 80
            print "\n"
            print "You have to specify one of the following commands:"
            for command, desc in JMX_LIST_COMMANDS.iteritems():
                print "      - %s [OPTIONAL: LIST OF CHECKS]: %s" % (command, desc)
            print "Example: sudo /etc/init.d/datadog-agent jmx list_matching_attributes tomcat jmx solr"
            print "\n"

        else:
            jmx_command = args[1]
            checks_list = args[2:]
            confd_directory = get_confd_path(get_os())
            should_run  = JMXFetch.init(confd_directory, agentConfig, get_logging_config(), 15, jmx_command, checks_list, reporter="console")
            if not should_run:
                print "Couldn't find any valid JMX configuration in your conf.d directory: %s" % confd_directory
                print "Have you enabled any JMX check ?"
                print "If you think it's not normal please get in touch with Datadog Support"


    return 0


if __name__ == '__main__':
    try:
        sys.exit(main())
    except StandardError:
        # Try our best to log the error.
        try:
Exemplo n.º 18
0
import time
import os
import sys
import subprocess
import re
import socket

import logging
import logging.config

import numpy as np
import tensorflow as tf

slim = tf.contrib.slim

logging.config.dictConfig(get_logging_config(args.run_name))
log = logging.getLogger()

train_dir = CKPT_ROOT + args.run_name
pretrain_dir = CKPT_ROOT + args.pretrained_net


def extract_batch(data_provider, classes):
    with tf.device("/cpu:0"):
        im, bbox, gt, proposals = data_provider.get(['image', 'object/bbox', 'object/label', 'proposal/bbox'])
        im = tf.to_float(im)/255
        gt = tf.to_int32(gt)
        gt_mask = tf.reduce_any(tf.equal(tf.expand_dims(gt, 1), tf.expand_dims(tf.constant(classes), 0)), axis=1)
        gt = tf.boolean_mask(gt, gt_mask)
        bbox = tf.boolean_mask(bbox, gt_mask)
Exemplo n.º 19
0
import logging
import logging.config
import time

from config import get_logging_config, args, evaluation_logfile, train_dir
from config import config as net_config
from paths import CKPT_ROOT
from utils import decode_bboxes, batch_iou

from skimage.transform import resize as imresize

from resnet import ResNet
from boxer import PriorBoxGrid
from voc_loader import VOCLoader

logging.config.dictConfig(get_logging_config(args.run_name))
log = logging.getLogger()


def main(argv=None):  # pylint: disable=unused-argument
    net = ResNet
    depth = 50

    loader = VOCLoader('07', 'test')

    net = net(config=net_config, depth=depth, training=False)

    num_classes = 21
    batch_size = args.batch_size
    img_size = args.image_size
    image_ph = tf.placeholder(shape=[1, img_size, img_size, 3],
Exemplo n.º 20
0
    def run(self):
        handlers = [
            (r"/intake/?", AgentInputHandler),
            (r"/intake/metrics?", MetricsAgentInputHandler),
            (r"/intake/metadata?", MetadataAgentInputHandler),
            (r"/api/v1/series/?", ApiInputHandler),
            (r"/api/v1/check_run/?", ApiCheckRunHandler),
            (r"/status/?", StatusHandler),
        ]

        settings = dict(
            cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
            xsrf_cookies=False,
            debug=False,
            log_function=self.log_request
        )

        non_local_traffic = self._agentConfig.get("non_local_traffic", False)

        tornado.web.Application.__init__(self, handlers, **settings)
        http_server = tornado.httpserver.HTTPServer(self)

        try:
            # non_local_traffic must be == True to match, not just some non-false value
            if non_local_traffic is True:
                http_server.listen(self._port)
            else:
                # localhost in lieu of 127.0.0.1 to support IPv6
                try:
                    http_server.listen(self._port, address=self._agentConfig['bind_host'])
                except gaierror:
                    log.warning("localhost seems undefined in your host file, using 127.0.0.1 instead")
                    http_server.listen(self._port, address="127.0.0.1")
                except socket_error as e:
                    if "Errno 99" in str(e):
                        log.warning("IPv6 doesn't seem to be fully supported. Falling back to IPv4")
                        http_server.listen(self._port, address="127.0.0.1")
                    else:
                        raise
        except socket_error as e:
            log.exception("Socket error %s. Is another application listening on the same port ? Exiting", e)
            sys.exit(1)
        except Exception as e:
            log.exception("Uncaught exception. Forwarder is exiting.")
            sys.exit(1)

        log.info("Listening on port %d" % self._port)

        # Register callbacks
        self.mloop = get_tornado_ioloop()

        logging.getLogger().setLevel(get_logging_config()['log_level'] or logging.INFO)

        def flush_trs():
            if self._watchdog:
                self._watchdog.reset()
            self._postMetrics()
            self._tr_manager.flush()

        tr_sched = tornado.ioloop.PeriodicCallback(flush_trs, TRANSACTION_FLUSH_INTERVAL,
                                                   io_loop=self.mloop)

        # Register optional Graphite listener
        gport = self._agentConfig.get("graphite_listen_port", None)
        if gport is not None:
            log.info("Starting graphite listener on port %s" % gport)
            from graphite import GraphiteServer
            gs = GraphiteServer(self, get_hostname(self._agentConfig), io_loop=self.mloop)
            if non_local_traffic is True:
                gs.listen(gport)
            else:
                gs.listen(gport, address="localhost")

        # Start everything
        if self._watchdog:
            self._watchdog.reset()
        tr_sched.start()

        self.mloop.start()
        log.info("Stopped")
Exemplo n.º 21
0
    def run(self):
        handlers = [
            (r"/intake/?", AgentInputHandler),
            (r"/api/v1/series/?", ApiInputHandler),
            (r"/status/?", StatusHandler),
        ]

        settings = dict(
            cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
            xsrf_cookies=False,
            debug=False,
            log_function=self.log_request
        )

        non_local_traffic = self._agentConfig.get("non_local_traffic", False)

        tornado.web.Application.__init__(self, handlers, **settings)
        http_server = tornado.httpserver.HTTPServer(self)

        # non_local_traffic must be == True to match, not just some non-false value
        if non_local_traffic is True:
            http_server.listen(self._port)
        else:
            # localhost in lieu of 127.0.0.1 to support IPv6
            try:
                http_server.listen(self._port, address = "localhost")
            except gaierror:
                log.warning("Warning localhost seems undefined in your host file, using 127.0.0.1 instead")
                http_server.listen(self._port, address = "127.0.0.1")

        log.info("Listening on port %d" % self._port)

        # Register callbacks
        self.mloop = tornado.ioloop.IOLoop.instance()

        logging.getLogger().setLevel(get_logging_config()['log_level'] or logging.INFO)

        def flush_trs():
            if self._watchdog:
                self._watchdog.reset()
            self._postMetrics()
            self._tr_manager.flush()

        tr_sched = tornado.ioloop.PeriodicCallback(flush_trs,TRANSACTION_FLUSH_INTERVAL,
            io_loop = self.mloop)

        # Register optional Graphite listener
        gport = self._agentConfig.get("graphite_listen_port", None)
        if gport is not None:
            log.info("Starting graphite listener on port %s" % gport)
            from graphite import GraphiteServer
            gs = GraphiteServer(self, get_hostname(self._agentConfig), io_loop=self.mloop)
            if non_local_traffic is True:
                gs.listen(gport)
            else:
                gs.listen(gport, address = "localhost")

        # Start everything
        if self._watchdog:
            self._watchdog.reset()
        tr_sched.start()

        self.mloop.start()
        log.info("Stopped")
Exemplo n.º 22
0
 def __init__(self):
     self.logging_config = get_logging_config()
     self._process = None
     self._running = True
Exemplo n.º 23
0
            print "All yaml files passed. You can now run the Datadog agent."
        else:
            print("Fix the invalid yaml files above in order to start the Datadog agent. "
                    "A useful external tool for yaml parsing can be found at "
                    "http://yaml-online-parser.appspot.com/")

    elif 'jmx' == command:
        from jmxfetch import JMX_LIST_COMMANDS, JMXFetch
       
        if len(args) < 2 or args[1] not in JMX_LIST_COMMANDS:
            print "You have to specify one of the following command %s" % JMX_LIST_COMMANDS
        else:
            jmx_command = args[1]
            from config import get_confd_path, get_logging_config
            from util import get_os
            JMXFetch.init(get_confd_path(get_os()), agentConfig, get_logging_config(), 15, jmx_command)


    return 0


if __name__ == '__main__':
    try:
        sys.exit(main())
    except StandardError:
        # Try our best to log the error.
        try:
            log.exception("Uncaught error running the Agent")
        except Exception:
            pass
        raise
Exemplo n.º 24
0
            print "#" * 80
            print "\n"
            print "You have to specify one of the following command:"
            for command, desc in JMX_LIST_COMMANDS.iteritems():
                print "      - %s [OPTIONAL: LIST OF CHECKS]: %s" % (command,
                                                                     desc)
            print "Example: sudo /etc/init.d/datadog-agent jmx list_matching_attributes tomcat jmx solr"
            print "\n"

        else:
            jmx_command = args[1]
            checks_list = args[2:]
            confd_directory = get_confd_path(get_os())
            should_run = JMXFetch.init(confd_directory,
                                       agentConfig,
                                       get_logging_config(),
                                       15,
                                       jmx_command,
                                       checks_list,
                                       reporter="console")
            if not should_run:
                print "Couldn't find any valid JMX configuration in your conf.d directory: %s" % confd_directory
                print "Have you enabled any JMX check ?"
                print "If you think it's not normal please get in touch with Datadog Support"

    return 0


if __name__ == '__main__':
    try:
        sys.exit(main())
Exemplo n.º 25
0
    def __init__(self, parent=None):
        prefix_conf = ''

        if Platform.is_windows():
            prefix_conf = 'windows_'

        log_conf = get_logging_config()

        QSplitter.__init__(self, parent)
        self.setWindowTitle(MAIN_WINDOW_TITLE)
        self.setWindowIcon(get_icon("agent.svg"))

        self.sysTray = SystemTray(self)

        self.connect(self.sysTray, SIGNAL("activated(QSystemTrayIcon::ActivationReason)"), self.__icon_activated)

        checks = get_checks()
        datadog_conf = DatadogConf(get_config_path())
        self.create_logs_files_windows(log_conf, prefix_conf)

        listwidget = QListWidget(self)
        listwidget.addItems([osp.basename(check.module_name).replace("_", " ").title() for check in checks])

        self.properties = PropertiesWidget(self)

        self.setting_button = QPushButton(get_icon("info.png"),
                                          "Logs and Status", self)
        self.menu_button = QPushButton(get_icon("settings.png"),
                                       "Actions", self)
        self.settings = [
            ("Forwarder Logs", lambda: [self.properties.set_log_file(self.forwarder_log_file),
                self.show_html(self.properties.group_code, self.properties.html_window, False)]),
            ("Collector Logs", lambda: [self.properties.set_log_file(self.collector_log_file),
                self.show_html(self.properties.group_code, self.properties.html_window, False)]),
            ("Dogstatsd Logs", lambda: [self.properties.set_log_file(self.dogstatsd_log_file),
                self.show_html(self.properties.group_code, self.properties.html_window, False)]),
            ("JMX Fetch Logs", lambda: [self.properties.set_log_file(self.jmxfetch_log_file),
                self.show_html(self.properties.group_code, self.properties.html_window, False)]),
            ("Agent Status", lambda: [self.properties.html_window.setHtml(self.properties.html_window.latest_status()),
                self.show_html(self.properties.group_code, self.properties.html_window, True),
                self.properties.set_status()]),
        ]

        self.agent_settings = QPushButton(get_icon("edit.png"),
                                          "Settings", self)
        self.connect(self.agent_settings, SIGNAL("clicked()"),
                     lambda: [self.properties.set_datadog_conf(datadog_conf),
                     self.show_html(self.properties.group_code, self.properties.html_window, False)])

        self.setting_menu = SettingMenu(self.settings)
        self.connect(self.setting_button, SIGNAL("clicked()"),
                     lambda: self.setting_menu.popup(self.setting_button.mapToGlobal(QPoint(0, 0))))

        self.manager_menu = Menu(self)
        self.connect(self.menu_button, SIGNAL("clicked()"),
                     lambda: self.manager_menu.popup(self.menu_button.mapToGlobal(QPoint(0, 0))))

        holdingBox = QGroupBox("", self)
        Box = QVBoxLayout(self)
        Box.addWidget(self.agent_settings)
        Box.addWidget(self.setting_button)
        Box.addWidget(self.menu_button)
        Box.addWidget(listwidget)
        holdingBox.setLayout(Box)

        self.addWidget(holdingBox)
        self.addWidget(self.properties)

        self.connect(self.properties.enable_button, SIGNAL("clicked()"),
                     lambda: enable_check(self.properties))

        self.connect(self.properties.disable_button, SIGNAL("clicked()"),
                     lambda: disable_check(self.properties))

        self.connect(self.properties.save_button, SIGNAL("clicked()"),
                     lambda: save_file(self.properties))

        self.connect(self.properties.refresh_button, SIGNAL("clicked()"),
                     lambda: [self.properties.set_log_file(self.properties.current_file),
                     self.properties.html_window.setHtml(self.properties.html_window.latest_status())])

        self.connect(listwidget, SIGNAL('currentRowChanged(int)'),
                     lambda row: [self.properties.set_item(checks[row]),
                     self.show_html(self.properties.group_code, self.properties.html_window, False)])

        listwidget.setCurrentRow(0)

        self.setSizes([150, 1])
        self.setStretchFactor(1, 1)
        self.resize(QSize(950, 600))
        self.properties.set_datadog_conf(datadog_conf)

        self.do_refresh()
Exemplo n.º 26
0
    def run(self):
        handlers = [
            (r"/intake/?", AgentInputHandler),
            (r"/intake/metrics?", MetricsAgentInputHandler),
            (r"/intake/metadata?", MetadataAgentInputHandler),
            (r"/api/v1/series/?", ApiInputHandler),
            (r"/api/v1/check_run/?", ApiCheckRunHandler),
            (r"/status/?", StatusHandler),
        ]

        settings = dict(
            cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
            xsrf_cookies=False,
            debug=False,
            log_function=self.log_request)

        non_local_traffic = self._agentConfig.get("non_local_traffic", False)

        tornado.web.Application.__init__(self, handlers, **settings)
        http_server = tornado.httpserver.HTTPServer(self)

        try:
            # non_local_traffic must be == True to match, not just some non-false value
            if non_local_traffic is True:
                http_server.listen(self._port)
            else:
                # localhost in lieu of 127.0.0.1 to support IPv6
                try:
                    http_server.listen(self._port,
                                       address=self._agentConfig['bind_host'])
                except gaierror:
                    log.warning(
                        "localhost seems undefined in your host file, using 127.0.0.1 instead"
                    )
                    http_server.listen(self._port, address="127.0.0.1")
                except socket_error as e:
                    if "Errno 99" in str(e):
                        log.warning(
                            "IPv6 doesn't seem to be fully supported. Falling back to IPv4"
                        )
                        http_server.listen(self._port, address="127.0.0.1")
                    else:
                        raise
        except socket_error as e:
            log.exception(
                "Socket error %s. Is another application listening on the same port ? Exiting",
                e)
            sys.exit(1)
        except Exception as e:
            log.exception("Uncaught exception. Forwarder is exiting.")
            sys.exit(1)

        log.info("Listening on port %d" % self._port)

        # Register callbacks
        self.mloop = tornado.ioloop.IOLoop.current()

        logging.getLogger().setLevel(get_logging_config()['log_level']
                                     or logging.INFO)

        def flush_trs():
            if self._watchdog:
                self._watchdog.reset()
            self._postMetrics()
            self._tr_manager.flush()

        tr_sched = tornado.ioloop.PeriodicCallback(flush_trs,
                                                   TRANSACTION_FLUSH_INTERVAL,
                                                   io_loop=self.mloop)

        # Register optional Graphite listener
        gport = self._agentConfig.get("graphite_listen_port", None)
        if gport is not None:
            log.info("Starting graphite listener on port %s" % gport)
            from graphite import GraphiteServer
            gs = GraphiteServer(self,
                                get_hostname(self._agentConfig),
                                io_loop=self.mloop)
            if non_local_traffic is True:
                gs.listen(gport)
            else:
                gs.listen(gport, address="localhost")

        # Start everything
        if self._watchdog:
            self._watchdog.reset()
        tr_sched.start()

        self.mloop.start()
        log.info("Stopped")
Exemplo n.º 27
0
import tornado.httpserver
import tornado.ioloop
import tornado.web
from tornado.escape import json_decode
from tornado.options import define, parse_command_line, options

# agent import
from util import Watchdog, get_uuid, get_hostname, json
from emitter import http_emitter, format_body
from config import get_config
from checks.check_status import ForwarderStatus
from transaction import Transaction, TransactionManager
import modules

log = logging.getLogger("forwarder")
log.setLevel(get_logging_config()["log_level"] or logging.INFO)

TRANSACTION_FLUSH_INTERVAL = 5000  # Every 5 seconds
WATCHDOG_INTERVAL_MULTIPLIER = 10  # 10x flush interval

# Maximum delay before replaying a transaction
MAX_WAIT_FOR_REPLAY = timedelta(seconds=90)

# Maximum queue size in bytes (when this is reached, old messages are dropped)
MAX_QUEUE_SIZE = 30 * 1024 * 1024  # 30MB

THROTTLING_DELAY = timedelta(microseconds=1000000 / 2)  # 2 msg/second


class EmitterThread(threading.Thread):
    def __init__(self, *args, **kwargs):
Exemplo n.º 28
0
)
import modules
from transaction import Transaction, TransactionManager
from util import (
    get_hostname,
    get_tornado_ioloop,
    get_uuid,
    json,
    Watchdog,
)
from utils.logger import RedactedLogRecord


logging.LogRecord = RedactedLogRecord
log = logging.getLogger('forwarder')
log.setLevel(get_logging_config()['log_level'] or logging.INFO)

DD_ENDPOINT = "dd_url"

TRANSACTION_FLUSH_INTERVAL = 5000  # Every 5 seconds
WATCHDOG_INTERVAL_MULTIPLIER = 10  # 10x flush interval
HEADERS_TO_REMOVE = [
    'Host',
    'Content-Length',
]


# Maximum delay before replaying a transaction
MAX_WAIT_FOR_REPLAY = timedelta(seconds=90)

# Maximum queue size in bytes (when this is reached, old messages are dropped)
Exemplo n.º 29
0
    def run(self):
        handlers = [
            (r"/intake/?", AgentInputHandler),
            (r"/api/v1/series/?", ApiInputHandler),
            (r"/status/?", StatusHandler),
        ]

        settings = dict(
            cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
            xsrf_cookies=False,
            debug=False,
        )

        non_local_traffic = self._agentConfig.get("non_local_traffic", False)

        tornado.web.Application.__init__(self, handlers, **settings)
        http_server = tornado.httpserver.HTTPServer(self)

        # set the root logger to warn so tornado is less chatty
        logging.getLogger().setLevel(logging.WARNING)

        # but keep the forwarder logger at the original level
        forwarder_logger = logging.getLogger('forwarder')
        log_config = get_logging_config()
        forwarder_logger.setLevel(log_config['log_level'] or logging.INFO)

        # non_local_traffic must be == True to match, not just some non-false value
        if non_local_traffic is True:
            http_server.listen(self._port)
        else:
            # localhost in lieu of 127.0.0.1 to support IPv6
            try:
                http_server.listen(self._port, address = "localhost")
            except gaierror:
                log.warning("Warning localhost seems undefined in your host file, using 127.0.0.1 instead")
                http_server.listen(self._port, address = "127.0.0.1")

        log.info("Listening on port %d" % self._port)

        # Register callbacks
        self.mloop = tornado.ioloop.IOLoop.instance()

        def flush_trs():
            if self._watchdog:
                self._watchdog.reset()
            self._postMetrics()
            self._tr_manager.flush()

        tr_sched = tornado.ioloop.PeriodicCallback(flush_trs,TRANSACTION_FLUSH_INTERVAL,
            io_loop = self.mloop)

        # Register optional Graphite listener
        gport = self._agentConfig.get("graphite_listen_port", None)
        if gport is not None:
            log.info("Starting graphite listener on port %s" % gport)
            from graphite import GraphiteServer
            gs = GraphiteServer(self, gethostname(self._agentConfig), io_loop=self.mloop)
            if non_local_traffic is True:
                gs.listen(gport)
            else:
                gs.listen(gport, address = "localhost")

        # Start everything
        if self._watchdog:
            self._watchdog.reset()
        tr_sched.start()

        self.mloop.start()
        log.info("Stopped")