예제 #1
0
파일: config.py 프로젝트: msalib/datadogpy
def get_config(cfg_path=None, options=None):
    agentConfig = {}

    # Config handling
    try:
        # Find the right config file
        path = os.path.realpath(__file__)
        path = os.path.dirname(path)

        config_path = get_config_path(cfg_path, os_name=get_os())
        config = configparser.ConfigParser()
        with open(config_path) as config_file:
            if is_p3k():
                config.read_file(skip_leading_wsp(config_file))
            else:
                config.readfp(skip_leading_wsp(config_file))

        # bulk import
        for option in config.options('Main'):
            agentConfig[option] = config.get('Main', option)

    except Exception:
        raise CfgNotFound

    return agentConfig
    def send(self, payload):
        if is_p3k():
            assert type(payload) == bytes
        else:
            assert type(payload) == str

        self.payloads.append(payload)
예제 #3
0
    def send(self, payload):
        if is_p3k():
            assert isinstance(payload, bytes)
        else:
            assert isinstance(payload, str)

        self.payloads.append(payload)
    def send(self, payload):
        if is_p3k():
            assert type(payload) == bytes
        else:
            assert type(payload) == str

        self.payloads.append(payload)
예제 #5
0
def get_temp_file():
    """Return a (fn, fp) pair"""
    if is_p3k():
        fn = "/tmp/{0}-{1}".format(time.time(), random.random())
        return (fn, open(fn, 'w+'))
    else:
        tf = tempfile.NamedTemporaryFile()
        return (tf.name, tf)
예제 #6
0
def get_temp_file():
    """Return a (fn, fp) pair"""
    if is_p3k():
        fn = "/tmp/{0}-{1}".format(time.time(), random.random())
        return (fn, open(fn, "w+"))
    else:
        tf = tempfile.NamedTemporaryFile()
        return (tf.name, tf)
예제 #7
0
    def test_parse_options(self):
        options, cmd = parse_options([])
        self.assertEqual(cmd, '')

        # The output of parse_args is already unicode in python 3, so don't encode the input
        if is_p3k():
            arg = u'helløøééé'
        else:
            arg = u'helløøééé'.encode('utf-8')

        options, cmd = parse_options([
            '-n', 'name', '-k', 'key', '-m', 'all', '-p', 'low', '-t', '123',
            '--sigterm_timeout', '456', '--sigkill_timeout', '789',
            '--proc_poll_interval', '1.5', '--notify_success', 'success',
            '--notify_error', 'error', '-b', '--tags', 'k1:v1,k2:v2', 'echo',
            arg
        ])
        self.assertEqual(cmd, u'echo helløøééé')
        self.assertEqual(options.name, 'name')
        self.assertEqual(options.api_key, 'key')
        self.assertEqual(options.submit_mode, 'all')
        self.assertEqual(options.priority, 'low')
        self.assertEqual(options.timeout, 123)
        self.assertEqual(options.sigterm_timeout, 456)
        self.assertEqual(options.sigkill_timeout, 789)
        self.assertEqual(options.proc_poll_interval, 1.5)
        self.assertEqual(options.notify_success, 'success')
        self.assertEqual(options.notify_error, 'error')
        self.assertTrue(options.buffer_outs)
        self.assertEqual(options.tags, 'k1:v1,k2:v2')

        with self.assertRaises(SystemExit):
            parse_options(['-m', 'invalid'])

        with self.assertRaises(SystemExit):
            parse_options(['-p', 'invalid'])

        with self.assertRaises(SystemExit):
            parse_options(['-t', 'invalid'])

        with self.assertRaises(SystemExit):
            parse_options(['--sigterm_timeout', 'invalid'])

        with self.assertRaises(SystemExit):
            parse_options(['--sigkill_timeout', 'invalid'])

        with self.assertRaises(SystemExit):
            parse_options(['--proc_poll_interval', 'invalid'])

        with mock.patch.dict(os.environ,
                             values={"DD_API_KEY": "the_key"},
                             clear=True):
            options, _ = parse_options([])
            self.assertEqual(options.api_key, "the_key")
예제 #8
0
def dog(api, vcr_cassette):
    """Record communication with Datadog API."""
    from datadog.util.compat import is_p3k
    if not is_p3k() and vcr_cassette.record_mode != "all":
        pytest.skip("Can not replay responses on Python 2")

    old_host_name = api._host_name
    api._host_name = "test.host"

    yield api

    api._host_name = old_host_name
예제 #9
0
 def _get_hostname_unix():
     try:
         # try fqdn
         p = subprocess.Popen(["/bin/hostname", "-f"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
         out, err = p.communicate()
         if p.returncode == 0:
             if is_p3k():
                 return out.decode("utf-8").strip()
             else:
                 return out.strip()
     except Exception:
         return None
예제 #10
0
 def _get_hostname_unix():
     try:
         # try fqdn
         p = subprocess.Popen(['/bin/hostname', '-f'], stdout=subprocess.PIPE)
         out, err = p.communicate()
         if p.returncode == 0:
             if is_p3k():
                 return out.decode("utf-8").strip()
             else:
                 return out.strip()
     except Exception:
         return None
예제 #11
0
    def test_get_hostname(self, mock_config_path):
        # Generate a fake agent config
        tmpfilepath = os.path.join(tempfile.gettempdir(), "tmp-agentconfig")
        with open(tmpfilepath, "wb") as f:
            if is_p3k():
                f.write(bytes("[Main]\n", 'UTF-8'))
                f.write(bytes("hostname: {0}\n".format(HOST_NAME), 'UTF-8'))
            else:
                f.write("[Main]\n")
                f.write("hostname: {0}\n".format(HOST_NAME))
        # Mock get_config_path to return this fake agent config
        mock_config_path.return_value = tmpfilepath

        initialize()
        assert api._host_name == HOST_NAME, api._host_name
예제 #12
0
파일: helper.py 프로젝트: msalib/datadogpy
    def load_request_response(self,
                              status_code=200,
                              response_body='{}',
                              raise_for_status=False):
        """
        Load the repsonse body from the given payload
        """
        mock_response = MockResponse(raise_for_status=raise_for_status)
        if is_p3k():
            mock_response.raw = BytesIO(bytes(response_body, 'utf-8'))
        else:
            mock_response.raw = BytesIO(response_body)
        mock_response.status_code = status_code

        self.request_mock.request = Mock(return_value=mock_response)
예제 #13
0
    def test_get_hostname(self, mock_config_path):
        # Generate a fake agent config
        tmpfilepath = os.path.join(tempfile.gettempdir(), "tmp-agentconfig")
        with open(tmpfilepath, "wb") as f:
            if is_p3k():
                f.write(bytes("[Main]\n", 'UTF-8'))
                f.write(bytes("hostname: {0}\n".format(HOST_NAME), 'UTF-8'))
            else:
                f.write("[Main]\n")
                f.write("hostname: {0}\n".format(HOST_NAME))
        # Mock get_config_path to return this fake agent config
        mock_config_path.return_value = tmpfilepath

        initialize()
        assert api._host_name == HOST_NAME, api._host_name
예제 #14
0
    def test_data_type_support(self):
        """
        `Metric` API supports `real` numerical data types.
        """
        from decimal import Decimal
        from fractions import Fraction

        m_long = int(1)  # long in Python 3.x

        if not is_p3k():
            m_long = long(1)  # noqa: F821

        supported_data_types = [1, 1.0, m_long, Decimal(1), Fraction(1, 2)]

        for point in supported_data_types:
            serie = dict(metric='metric.numerical', points=point)
            self.submit_and_assess_metric_payload(serie)
예제 #15
0
파일: test_api.py 프로젝트: adqio/datadogpy
    def test_data_type_support(self):
        """
        `Metric` API supports `real` numerical data types.
        """
        from decimal import Decimal
        from fractions import Fraction

        m_long = int(1)  # long in Python 3.x

        if not is_p3k():
            m_long = long(1)

        supported_data_types = [1, 1.0, m_long, Decimal(1), Fraction(1, 2)]

        for point in supported_data_types:
            serie = dict(metric='metric.numerical', points=point)
            self.submit_and_assess_metric_payload(serie)
예제 #16
0
    def test_get_hostname(self, mock_config_path):
        """
        API hostname parameter fallback with Datadog Agent hostname when available.
        """
        # Generate a fake agent config
        tmpfilepath = os.path.join(tempfile.gettempdir(), "tmp-agentconfig")
        with open(tmpfilepath, "wb") as f:
            if is_p3k():
                f.write(bytes("[Main]\n", 'UTF-8'))
                f.write(bytes("hostname: {0}\n".format(HOST_NAME), 'UTF-8'))
            else:
                f.write("[Main]\n")
                f.write("hostname: {0}\n".format(HOST_NAME))
        # Mock get_config_path to return this fake agent config
        mock_config_path.return_value = tmpfilepath

        initialize()
        self.assertEqual(api._host_name, HOST_NAME, api._host_name)
예제 #17
0
파일: test_api.py 프로젝트: adqio/datadogpy
    def test_get_hostname(self, mock_config_path):
        """
        API hostname parameter fallback with Datadog Agent hostname when available.
        """
        # Generate a fake agent config
        tmpfilepath = os.path.join(tempfile.gettempdir(), "tmp-agentconfig")
        with open(tmpfilepath, "wb") as f:
            if is_p3k():
                f.write(bytes("[Main]\n", 'UTF-8'))
                f.write(bytes("hostname: {0}\n".format(HOST_NAME), 'UTF-8'))
            else:
                f.write("[Main]\n")
                f.write("hostname: {0}\n".format(HOST_NAME))
        # Mock get_config_path to return this fake agent config
        mock_config_path.return_value = tmpfilepath

        initialize()
        self.assertEquals(api._host_name, HOST_NAME, api._host_name)
예제 #18
0
파일: helper.py 프로젝트: yannmh/datadogpy
    def request_called_with(self, method, url, data=None, params=None):
        (req_method, req_url), others = self.request_mock.request.call_args
        assert method == req_method, req_method
        assert url == req_url, req_url

        if data:
            assert 'data' in others
            assert json.dumps(data) == others['data'], others['data']

        if params:
            assert 'params' in others
            if is_p3k():
                for (k, v) in params.items():
                    assert k in others['params'], others['params']
                    assert v == others['params'][k]
            else:
                for (k, v) in params.iteritems():
                    assert k in others['params'], others['params']
                    assert v == others['params'][k]
예제 #19
0
def main():
    options, cmd = parse_options()

    # If silent is checked we force the outputs to be buffered (and therefore
    # not forwarded to the Terminal streams) and we just avoid printing the
    # buffers at the end
    returncode, stdout, stderr, duration = execute(cmd, options.timeout,
                                                   options.sigterm_timeout,
                                                   options.sigkill_timeout,
                                                   options.proc_poll_interval,
                                                   options.buffer_outs)

    initialize(api_key=options.api_key)
    host = api._host_name

    if returncode == 0:
        alert_type = SUCCESS
        event_priority = 'low'
        event_title = u'[%s] %s succeeded in %.2fs' % (host, options.name,
                                                       duration)
    else:
        alert_type = ERROR
        event_priority = 'normal'

        if returncode is Timeout:
            event_title = u'[%s] %s timed out after %.2fs' % (
                host, options.name, duration)
            returncode = -1
        else:
            event_title = u'[%s] %s failed in %.2fs' % (host, options.name,
                                                        duration)

    notifications = ""
    if alert_type == SUCCESS and options.notify_success:
        notifications = options.notify_success
    elif alert_type == ERROR and options.notify_error:
        notifications = options.notify_error

    if options.tags:
        tags = [t.strip() for t in options.tags.split(',')]
    else:
        tags = None

    event_body = build_event_body(cmd, returncode, stdout, stderr,
                                  notifications)

    event = {
        'alert_type': alert_type,
        'aggregation_key': options.name,
        'host': host,
        'priority': options.priority or event_priority,
        'tags': tags
    }

    if options.buffer_outs:
        if is_p3k():
            stderr = stderr.decode('utf-8')
            stdout = stdout.decode('utf-8')

        print(stderr.strip(), file=sys.stderr)
        print(stdout.strip(), file=sys.stdout)

    if options.submit_mode == 'all' or returncode != 0:
        api.Event.create(title=event_title, text=event_body, **event)

    sys.exit(returncode)
예제 #20
0
    def request(cls, method, path, body=None, attach_host_name=False, response_formatter=None,
                error_formatter=None, **params):
        """
        Make an HTTP API request

        :param method: HTTP method to use to contact API endpoint
        :type method: HTTP method string

        :param path: API endpoint url
        :type path: url

        :param body: dictionnary to be sent in the body of the request
        :type body: dictionary

        :param response_formatter: function to format JSON response from HTTP API request
        :type response_formatter: JSON input function

        :param error_formatter: function to format JSON error response from HTTP API request
        :type error_formatter: JSON input function

        :param attach_host_name: link the new resource object to the host name
        :type attach_host_name: bool

        :param params: dictionnary to be sent in the query string of the request
        :type params: dictionary

        :returns: JSON or formated response from HTTP API request
        """

        try:
            # Check if it's ok to submit
            if not cls._should_submit():
                raise HttpBackoff("Too many timeouts. Won't try again for {1} seconds."
                                  .format(*cls._backoff_status()))

            # Import API, User and HTTP settings
            from datadog.api import _api_key, _application_key, _api_host, \
                _mute, _host_name, _proxies, _max_retries, _timeout, \
                _cacert

            # Check keys and add then to params
            if _api_key is None:
                raise ApiNotInitialized("API key is not set."
                                        " Please run 'initialize' method first.")
            params['api_key'] = _api_key
            if _application_key:
                params['application_key'] = _application_key

            # Construct the url
            url = "%s/api/%s/%s" % (_api_host, cls._api_version, path.lstrip("/"))

            # Attach host name to body
            if attach_host_name and body:
                # Is it a 'series' list of objects ?
                if 'series' in body:
                    # Adding the host name to all objects
                    for obj_params in body['series']:
                        if 'host' not in obj_params:
                            obj_params['host'] = _host_name
                else:
                    if 'host' not in body:
                        body['host'] = _host_name

            # If defined, make sure tags are defined as a comma-separated string
            if 'tags' in params and isinstance(params['tags'], list):
                params['tags'] = ','.join(params['tags'])

            # Process the body, if necessary
            headers = {}
            if isinstance(body, dict):
                body = json.dumps(body)
                headers['Content-Type'] = 'application/json'

            # Process requesting
            start_time = time.time()
            try:
                # Use a session to set a max_retries parameters
                s = requests.Session()
                http_adapter = requests.adapters.HTTPAdapter(max_retries=_max_retries)
                s.mount('https://', http_adapter)

                # Request
                result = s.request(
                    method,
                    url,
                    headers=headers,
                    params=params,
                    data=body,
                    timeout=_timeout,
                    proxies=_proxies,
                    verify=_cacert)

                result.raise_for_status()
            except requests.ConnectionError as e:
                raise ClientError("Could not request %s %s%s: %s" % (method, _api_host, url, e))
            except requests.exceptions.Timeout as e:
                cls._timeout_counter += 1
                raise HttpTimeout('%s %s timed out after %d seconds.' % (method, url, _timeout))
            except requests.exceptions.HTTPError as e:
                if e.response.status_code in (400, 403, 404):
                    # This gets caught afterwards and raises an ApiError exception
                    pass
                else:
                    raise
            except TypeError as e:
                raise TypeError(
                    "Your installed version of 'requests' library seems not compatible with"
                    "Datadog's usage. We recommand upgrading it ('pip install -U requests')."
                    "If you need help or have any question, please contact [email protected]")

            # Request succeeded: log it and reset the timeout counter
            duration = round((time.time() - start_time) * 1000., 4)
            log.info("%s %s %s (%sms)" % (result.status_code, method, url, duration))
            cls._timeout_counter = 0

            # Format response content
            content = result.content

            if content:
                try:
                    if is_p3k():
                        response_obj = json.loads(content.decode('utf-8'))
                    else:
                        response_obj = json.loads(content)
                except ValueError:
                    raise ValueError('Invalid JSON response: {0}'.format(content))

                if response_obj and 'errors' in response_obj:
                    raise ApiError(response_obj)
            else:
                response_obj = None
            if response_formatter is None:
                return response_obj
            else:
                return response_formatter(response_obj)

        except ClientError as e:
            if _mute:
                log.error(str(e))
                if error_formatter is None:
                    return {'errors': e.args[0]}
                else:
                    return error_formatter({'errors': e.args[0]})
            else:
                raise
        except ApiError as e:
            if _mute:
                for error in e.args[0]['errors']:
                    log.error(str(error))
                if error_formatter is None:
                    return e.args[0]
                else:
                    return error_formatter(e.args[0])
            else:
                raise
예제 #21
0
파일: base.py 프로젝트: DataDog/datadogpy
    def event(
        self,
        title,
        text,
        alert_type=None,
        aggregation_key=None,
        source_type_name=None,
        date_happened=None,
        priority=None,
        tags=None,
        hostname=None,
    ):
        """
        Send an event. Attributes are the same as the Event API.
            http://docs.datadoghq.com/api/

        >>> statsd.event("Man down!", "This server needs assistance.")
        >>> statsd.event("The web server restarted", "The web server is up again", alert_type="success")  # NOQA
        """
        title = self._escape_event_content(title)
        text = self._escape_event_content(text)

        if not is_p3k():
            if not isinstance(title, unicode):  # noqa: F821
                title = unicode(self._escape_event_content(title),
                                'utf8')  # noqa: F821
            if not isinstance(text, unicode):  # noqa: F821
                text = unicode(self._escape_event_content(text),
                               'utf8')  # noqa: F821

        # Append all client level tags to every event
        tags = self._add_constant_tags(tags)

        string = u"_e{{{},{}}}:{}|{}".format(
            len(title.encode('utf8', 'replace')),
            len(text.encode('utf8', 'replace')),
            title,
            text,
        )

        if date_happened:
            string = "%s|d:%d" % (string, date_happened)
        if hostname:
            string = "%s|h:%s" % (string, hostname)
        if aggregation_key:
            string = "%s|k:%s" % (string, aggregation_key)
        if priority:
            string = "%s|p:%s" % (string, priority)
        if source_type_name:
            string = "%s|s:%s" % (string, source_type_name)
        if alert_type:
            string = "%s|t:%s" % (string, alert_type)
        if tags:
            string = "%s|#%s" % (string, ",".join(tags))

        if len(string) > 8 * 1024:
            raise Exception(u'Event "%s" payload is too big (more than 8KB), '
                            "event discarded" % title)

        if self._telemetry:
            self.events_count += 1
        self._send(string)
예제 #22
0
    def submit(cls, method, path, body=None, attach_host_name=False, response_formatter=None,
               error_formatter=None, **params):
        """
        Make an HTTP API request

        :param method: HTTP method to use to contact API endpoint
        :type method: HTTP method string

        :param path: API endpoint url
        :type path: url

        :param body: dictionary to be sent in the body of the request
        :type body: dictionary

        :param response_formatter: function to format JSON response from HTTP API request
        :type response_formatter: JSON input function

        :param error_formatter: function to format JSON error response from HTTP API request
        :type error_formatter: JSON input function

        :param attach_host_name: link the new resource object to the host name
        :type attach_host_name: bool

        :param params: dictionary to be sent in the query string of the request
        :type params: dictionary

        :returns: JSON or formated response from HTTP API request
        """
        try:
            # Check if it's ok to submit
            if not cls._should_submit():
                _, backoff_time_left = cls._backoff_status()
                raise HttpBackoff(backoff_time_left)

            # Import API, User and HTTP settings
            from datadog.api import _api_key, _application_key, _api_host, \
                _mute, _host_name, _proxies, _max_retries, _timeout, \
                _cacert

            # Check keys and add then to params
            if _api_key is None:
                raise ApiNotInitialized("API key is not set."
                                        " Please run 'initialize' method first.")
            params['api_key'] = _api_key
            if _application_key:
                params['application_key'] = _application_key

            # Attach host name to body
            if attach_host_name and body:
                # Is it a 'series' list of objects ?
                if 'series' in body:
                    # Adding the host name to all objects
                    for obj_params in body['series']:
                        if obj_params.get('host', "") == "":
                            obj_params['host'] = _host_name
                else:
                    if body.get('host', "") == "":
                        body['host'] = _host_name

            # If defined, make sure tags are defined as a comma-separated string
            if 'tags' in params and isinstance(params['tags'], list):
                params['tags'] = ','.join(params['tags'])

            # Process the body, if necessary
            headers = {}
            if isinstance(body, dict):
                body = json.dumps(body)
                headers['Content-Type'] = 'application/json'

            # Construct the URL
            url = "{api_host}/api/{api_version}/{path}".format(
                  api_host=_api_host,
                  api_version=cls._api_version,
                  path=path.lstrip("/"),
            )

            # Process requesting
            start_time = time.time()

            result = cls._get_http_client().request(
                method=method, url=url,
                headers=headers, params=params, data=body,
                timeout=_timeout, max_retries=_max_retries,
                proxies=_proxies, verify=_cacert
            )

            # Request succeeded: log it and reset the timeout counter
            duration = round((time.time() - start_time) * 1000., 4)
            log.info("%s %s %s (%sms)" % (result.status_code, method, url, duration))
            cls._timeout_counter = 0

            # Format response content
            content = result.content

            if content:
                try:
                    if is_p3k():
                        response_obj = json.loads(content.decode('utf-8'))
                    else:
                        response_obj = json.loads(content)
                except ValueError:
                    raise ValueError('Invalid JSON response: {0}'.format(content))

                if response_obj and 'errors' in response_obj:
                    raise ApiError(response_obj)
            else:
                response_obj = None
            if response_formatter is None:
                return response_obj
            else:
                return response_formatter(response_obj)

        except HttpTimeout:
            cls._timeout_counter += 1
            raise
        except ClientError as e:
            if _mute:
                log.error(str(e))
                if error_formatter is None:
                    return {'errors': e.args[0]}
                else:
                    return error_formatter({'errors': e.args[0]})
            else:
                raise
        except ApiError as e:
            if _mute:
                for error in e.args[0]['errors']:
                    log.error(str(error))
                if error_formatter is None:
                    return e.args[0]
                else:
                    return error_formatter(e.args[0])
            else:
                raise
예제 #23
0
파일: common.py 프로젝트: dcrosta/datadogpy
def print_err(msg):
    if is_p3k():
        print('ERROR: ' + msg + '\n', file=sys.stderr)
    else:
        sys.stderr.write(msg + '\n')
예제 #24
0
    def submit(cls, method, path, api_version=None, body=None, attach_host_name=False,
               response_formatter=None, error_formatter=None, suppress_response_errors_on_codes=None,
               compress_payload=False, **params):
        """
        Make an HTTP API request

        :param method: HTTP method to use to contact API endpoint
        :type method: HTTP method string

        :param path: API endpoint url
        :type path: url

        :param api_version: The API version used

        :param body: dictionary to be sent in the body of the request
        :type body: dictionary

        :param response_formatter: function to format JSON response from HTTP API request
        :type response_formatter: JSON input function

        :param error_formatter: function to format JSON error response from HTTP API request
        :type error_formatter: JSON input function

        :param attach_host_name: link the new resource object to the host name
        :type attach_host_name: bool

        :param suppress_response_errors_on_codes: suppress ApiError on `errors` key in the response for the given HTTP
                                                  status codes
        :type suppress_response_errors_on_codes: None|list(int)

        :param compress_payload: compress the payload using zlib
        :type compress_payload: bool

        :param params: dictionary to be sent in the query string of the request
        :type params: dictionary

        :returns: JSON or formated response from HTTP API request
        """
        try:
            # Check if it's ok to submit
            if not cls._should_submit():
                _, backoff_time_left = cls._backoff_status()
                raise HttpBackoff(backoff_time_left)

            # Import API, User and HTTP settings
            from datadog.api import _api_key, _application_key, _api_host, \
                _mute, _host_name, _proxies, _max_retries, _timeout, \
                _cacert, _return_raw_response

            # Check keys and add then to params
            if _api_key is None:
                raise ApiNotInitialized("API key is not set."
                                        " Please run 'initialize' method first.")

            # Set api and app keys in headers
            headers = {}
            headers['DD-API-KEY'] = _api_key
            if _application_key:
                headers['DD-APPLICATION-KEY'] = _application_key

            # Check if the api_version is provided
            if not api_version:
                api_version = _api_version

            # set api and app keys in params only for some endpoints and thus remove keys from headers
            # as they cannot be set in both params and headers
            if cls._set_api_and_app_keys_in_params(api_version, path):
                params['api_key'] = _api_key
                del headers['DD-API-KEY']
                if _application_key:
                    params['application_key'] = _application_key
                    del headers['DD-APPLICATION-KEY']

            # Attach host name to body
            if attach_host_name and body:
                # Is it a 'series' list of objects ?
                if 'series' in body:
                    # Adding the host name to all objects
                    for obj_params in body['series']:
                        if obj_params.get('host', "") == "":
                            obj_params['host'] = _host_name
                else:
                    if body.get('host', "") == "":
                        body['host'] = _host_name

            # If defined, make sure tags are defined as a comma-separated string
            if 'tags' in params and isinstance(params['tags'], list):
                tag_list = normalize_tags(params['tags'])
                params['tags'] = ','.join(tag_list)

            # If defined, make sure monitor_ids are defined as a comma-separated string
            if 'monitor_ids' in params and isinstance(params['monitor_ids'], list):
                params['monitor_ids'] = ','.join(str(i) for i in params['monitor_ids'])

            # Process the body, if necessary
            if isinstance(body, dict):
                body = json.dumps(body, sort_keys=cls._sort_keys)
                headers['Content-Type'] = 'application/json'

            if compress_payload:
                body = zlib.compress(body.encode("utf-8"))
                headers["Content-Encoding"] = "deflate"

            # Construct the URL
            url = construct_url(_api_host, api_version, path)

            # Process requesting
            start_time = time.time()

            result = cls._get_http_client().request(
                method=method, url=url,
                headers=headers, params=params, data=body,
                timeout=_timeout, max_retries=_max_retries,
                proxies=_proxies, verify=_cacert
            )

            # Request succeeded: log it and reset the timeout counter
            duration = round((time.time() - start_time) * 1000., 4)
            log.info("%s %s %s (%sms)" % (result.status_code, method, url, duration))
            cls._timeout_counter = 0

            # Format response content
            content = result.content

            if content:
                try:
                    if is_p3k():
                        response_obj = json.loads(content.decode('utf-8'))
                    else:
                        response_obj = json.loads(content)
                except ValueError:
                    raise ValueError('Invalid JSON response: {0}'.format(content))

                # response_obj can be a bool and not a dict
                if isinstance(response_obj, dict):
                    if response_obj and 'errors' in response_obj:
                        # suppress ApiError when specified and just return the response
                        if not (suppress_response_errors_on_codes and
                                result.status_code in suppress_response_errors_on_codes):
                            raise ApiError(response_obj)
            else:
                response_obj = None

            if response_formatter is not None:
                response_obj = response_formatter(response_obj)

            if _return_raw_response:
                return response_obj, result
            else:
                return response_obj

        except HttpTimeout:
            cls._timeout_counter += 1
            raise
        except ClientError as e:
            if _mute:
                log.error(str(e))
                if error_formatter is None:
                    return {'errors': e.args[0]}
                else:
                    return error_formatter({'errors': e.args[0]})
            else:
                raise
        except ApiError as e:
            if _mute:
                for error in (e.args[0].get('errors') or []):
                    log.error(error)
                if error_formatter is None:
                    return e.args[0]
                else:
                    return error_formatter(e.args[0])
            else:
                raise
예제 #25
0
def print_err(msg):
    if is_p3k():
        print(msg + "\n", file=sys.stderr)
    else:
        sys.stderr.write(msg + "\n")
    sys.stderr.flush()
예제 #26
0
def print_err(msg):
    if is_p3k():
        print(msg + '\n', file=sys.stderr)
    else:
        sys.stderr.write(msg + '\n')
예제 #27
0
파일: wrap.py 프로젝트: yesyouken/datadogpy
def execute(cmd, cmd_timeout, sigterm_timeout, sigkill_timeout,
            proc_poll_interval, buffer_outs):
    '''
    Launches the process and monitors its outputs
    '''
    start_time = time.time()
    returncode = -1
    stdout = b''
    stderr = b''
    try:
        proc = subprocess.Popen(cmd,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                shell=True)
    except Exception:
        print(u"Failed to execute %s" % (repr(cmd)), file=sys.stderr)
        raise
    try:
        # Let's that the threads collecting the output from the command in the
        # background
        stdout_buffer = sys.stdout.buffer if is_p3k() else sys.stdout
        stderr_buffer = sys.stderr.buffer if is_p3k() else sys.stderr
        out_reader = OutputReader(proc.stdout,
                                  stdout_buffer if not buffer_outs else None)
        err_reader = OutputReader(proc.stderr,
                                  stderr_buffer if not buffer_outs else None)
        out_reader.start()
        err_reader.start()

        # Let's quietly wait from the program's completion here to get the exit
        # code when it finishes
        returncode = poll_proc(proc, proc_poll_interval, cmd_timeout)
    except Timeout:
        returncode = Timeout
        sigterm_start = time.time()
        print("Command timed out after %.2fs, killing with SIGTERM" %
              (time.time() - start_time),
              file=sys.stderr)
        try:
            proc.terminate()
            try:
                poll_proc(proc, proc_poll_interval, sigterm_timeout)
            except Timeout:
                print(
                    "SIGTERM timeout failed after %.2fs, killing with SIGKILL"
                    % (time.time() - sigterm_start),
                    file=sys.stderr)
                sigkill_start = time.time()
                proc.kill()
                try:
                    poll_proc(proc, proc_poll_interval, sigkill_timeout)
                except Timeout:
                    print("SIGKILL timeout failed after %.2fs, exiting" %
                          (time.time() - sigkill_start),
                          file=sys.stderr)
        except OSError as e:
            # Ignore OSError 3: no process found.
            if e.errno != 3:
                raise

    # Let's harvest the outputs collected by our background threads
    # after making sure they're done reading it.
    out_reader.join()
    err_reader.join()
    stdout = out_reader.content
    stderr = err_reader.content

    duration = time.time() - start_time

    return returncode, stdout, stderr, duration
예제 #28
0
파일: wrap.py 프로젝트: yesyouken/datadogpy
def main():
    options, cmd = parse_options()

    # If silent is checked we force the outputs to be buffered (and therefore
    # not forwarded to the Terminal streams) and we just avoid printing the
    # buffers at the end
    returncode, stdout, stderr, duration = execute(cmd, options.timeout,
                                                   options.sigterm_timeout,
                                                   options.sigkill_timeout,
                                                   options.proc_poll_interval,
                                                   options.buffer_outs)

    if options.site in ('datadoghq.eu', 'eu'):
        api_host = 'https://api.datadoghq.eu'
    else:
        api_host = 'https://api.datadoghq.com'

    initialize(api_key=options.api_key, api_host=api_host)
    host = api._host_name

    warning_codes = None

    if options.warning_codes:
        # Convert warning codes from string to int since return codes will evaluate the latter
        warning_codes = list(map(int, options.warning_codes))

    if returncode == 0:
        alert_type = SUCCESS
        event_priority = 'low'
        event_title = u'[%s] %s succeeded in %.2fs' % (host, options.name,
                                                       duration)
    elif returncode != 0 and options.submit_mode == 'warnings':
        if not warning_codes:
            # the list of warning codes is empty - the option was not specified
            print("A comma separated list of exit codes need to be provided")
            sys.exit()
        elif returncode in warning_codes:
            alert_type = WARNING
            event_priority = 'normal'
            event_title = u'[%s] %s failed in %.2fs' % (host, options.name,
                                                        duration)
        else:
            print(
                "Command exited with a different exit code that the one(s) provided"
            )
            sys.exit()
    else:
        alert_type = ERROR
        event_priority = 'normal'

        if returncode is Timeout:
            event_title = u'[%s] %s timed out after %.2fs' % (
                host, options.name, duration)
            returncode = -1
        else:
            event_title = u'[%s] %s failed in %.2fs' % (host, options.name,
                                                        duration)

    notifications = ""

    if alert_type == SUCCESS and options.notify_success:
        notifications = options.notify_success
    elif alert_type == ERROR and options.notify_error:
        notifications = options.notify_error
    elif alert_type == WARNING and options.notify_warning:
        notifications = options.notify_warning

    if options.tags:
        tags = [t.strip() for t in options.tags.split(',')]
    else:
        tags = None

    event_body = build_event_body(cmd, returncode, stdout, stderr,
                                  notifications)

    event = {
        'alert_type': alert_type,
        'aggregation_key': options.name,
        'host': host,
        'priority': options.priority or event_priority,
        'tags': tags
    }

    if options.buffer_outs:
        if is_p3k():
            stderr = stderr.decode('utf-8')
            stdout = stdout.decode('utf-8')

        print(stderr.strip(), file=sys.stderr)
        print(stdout.strip(), file=sys.stdout)

    if options.submit_mode == 'all' or returncode != 0:
        if options.send_metric:
            event_name_tag = "event_name:{}".format(options.name)
            if tags:
                duration_tags = tags + [event_name_tag]
            else:
                duration_tags = [event_name_tag]
            api.Metric.send(metric='dogwrap.duration',
                            points=duration,
                            tags=duration_tags,
                            type="gauge")
        api.Event.create(title=event_title, text=event_body, **event)

    sys.exit(returncode)
예제 #29
0
    def submit(cls,
               method,
               path,
               body=None,
               attach_host_name=False,
               response_formatter=None,
               error_formatter=None,
               **params):
        """
        Make an HTTP API request

        :param method: HTTP method to use to contact API endpoint
        :type method: HTTP method string

        :param path: API endpoint url
        :type path: url

        :param body: dictionary to be sent in the body of the request
        :type body: dictionary

        :param response_formatter: function to format JSON response from HTTP API request
        :type response_formatter: JSON input function

        :param error_formatter: function to format JSON error response from HTTP API request
        :type error_formatter: JSON input function

        :param attach_host_name: link the new resource object to the host name
        :type attach_host_name: bool

        :param params: dictionary to be sent in the query string of the request
        :type params: dictionary

        :returns: JSON or formated response from HTTP API request
        """
        try:
            # Check if it's ok to submit
            if not cls._should_submit():
                _, backoff_time_left = cls._backoff_status()
                raise HttpBackoff(backoff_time_left)

            # Import API, User and HTTP settings
            from datadog.api import _api_key, _application_key, _api_host, \
                _mute, _host_name, _proxies, _max_retries, _timeout, \
                _cacert

            # Check keys and add then to params
            if _api_key is None:
                raise ApiNotInitialized(
                    "API key is not set."
                    " Please run 'initialize' method first.")
            params['api_key'] = _api_key
            if _application_key:
                params['application_key'] = _application_key

            # Attach host name to body
            if attach_host_name and body:
                # Is it a 'series' list of objects ?
                if 'series' in body:
                    # Adding the host name to all objects
                    for obj_params in body['series']:
                        if obj_params.get('host', "") == "":
                            obj_params['host'] = _host_name
                else:
                    if body.get('host', "") == "":
                        body['host'] = _host_name

            # If defined, make sure tags are defined as a comma-separated string
            if 'tags' in params and isinstance(params['tags'], list):
                params['tags'] = ','.join(params['tags'])

            # Process the body, if necessary
            headers = {}
            if isinstance(body, dict):
                body = json.dumps(body)
                headers['Content-Type'] = 'application/json'

            # Construct the URL
            url = "{api_host}/api/{api_version}/{path}".format(
                api_host=_api_host,
                api_version=cls._api_version,
                path=path.lstrip("/"),
            )

            # Process requesting
            start_time = time.time()

            result = cls._get_http_client().request(method=method,
                                                    url=url,
                                                    headers=headers,
                                                    params=params,
                                                    data=body,
                                                    timeout=_timeout,
                                                    max_retries=_max_retries,
                                                    proxies=_proxies,
                                                    verify=_cacert)

            # Request succeeded: log it and reset the timeout counter
            duration = round((time.time() - start_time) * 1000., 4)
            log.info("%s %s %s (%sms)" %
                     (result.status_code, method, url, duration))
            cls._timeout_counter = 0

            # Format response content
            content = result.content

            if content:
                try:
                    if is_p3k():
                        response_obj = json.loads(content.decode('utf-8'))
                    else:
                        response_obj = json.loads(content)
                except ValueError:
                    raise ValueError(
                        'Invalid JSON response: {0}'.format(content))

                if response_obj and 'errors' in response_obj:
                    raise ApiError(response_obj)
            else:
                response_obj = None
            if response_formatter is None:
                return response_obj
            else:
                return response_formatter(response_obj)

        except HttpTimeout:
            cls._timeout_counter += 1
            raise
        except ClientError as e:
            if _mute:
                log.error(str(e))
                if error_formatter is None:
                    return {'errors': e.args[0]}
                else:
                    return error_formatter({'errors': e.args[0]})
            else:
                raise
        except ApiError as e:
            if _mute:
                for error in e.args[0]['errors']:
                    log.error(str(error))
                if error_formatter is None:
                    return e.args[0]
                else:
                    return error_formatter(e.args[0])
            else:
                raise
예제 #30
0
파일: base.py 프로젝트: kuzmich/datadogpy
    def request(cls,
                method,
                path,
                body=None,
                attach_host_name=False,
                response_formatter=None,
                error_formatter=None,
                **params):
        """
        Make an HTTP API request

        :param method: HTTP method to use to contact API endpoint
        :type method: HTTP method string

        :param path: API endpoint url
        :type path: url

        :param body: dictionnary to be sent in the body of the request
        :type body: dictionary

        :param response_formatter: function to format JSON response from HTTP API request
        :type response_formatter: JSON input function

        :param error_formatter: function to format JSON error response from HTTP API request
        :type error_formatter: JSON input function

        :param attach_host_name: link the new resource object to the host name
        :type attach_host_name: bool

        :param params: dictionnary to be sent in the query string of the request
        :type params: dictionary

        :returns: JSON or formated response from HTTP API request
        """

        try:
            # Check if it's ok to submit
            if not cls._should_submit():
                raise HttpBackoff(
                    "Too many timeouts. Won't try again for {1} seconds.".
                    format(*cls._backoff_status()))

            # Import API, User and HTTP settings
            from datadog.api import _api_key, _application_key, _api_host, \
                _swallow, _host_name, _proxies, _max_retries, _timeout

            # Check keys and add then to params
            if _api_key is None:
                raise ApiNotInitialized(
                    "API key is not set."
                    " Please run 'initialize' method first.")
            params['api_key'] = _api_key
            if _application_key:
                params['application_key'] = _application_key

            # Construct the url
            url = "%s/api/%s/%s" % (_api_host, cls._api_version,
                                    path.lstrip("/"))

            # Attach host name to body
            if attach_host_name and body:
                # Is it a 'series' list of objects ?
                if 'series' in body:
                    # Adding the host name to all objects
                    for obj_params in body['series']:
                        if 'host' not in obj_params:
                            obj_params['host'] = _host_name
                else:
                    if 'host' not in body:
                        body['host'] = _host_name

            # If defined, make sure tags are defined as a comma-separated string
            if 'tags' in params and isinstance(params['tags'], list):
                params['tags'] = ','.join(params['tags'])

            # Process the body, if necessary
            headers = {}
            if isinstance(body, dict):
                body = json.dumps(body)
                headers['Content-Type'] = 'application/json'

            # Process requesting
            start_time = time.time()
            try:
                # Use a session to set a max_retries parameters
                s = requests.Session()
                http_adapter = requests.adapters.HTTPAdapter(
                    max_retries=_max_retries)
                s.mount('https://', http_adapter)

                # Request
                result = s.request(method,
                                   url,
                                   headers=headers,
                                   params=params,
                                   data=body,
                                   timeout=_timeout,
                                   proxies=_proxies)

                result.raise_for_status()
            except requests.ConnectionError as e:
                raise ClientError("Could not request %s %s%s: %s" %
                                  (method, _api_host, url, e))
            except requests.exceptions.Timeout as e:
                cls._timeout_counter += 1
                raise HttpTimeout('%s %s timed out after %d seconds.' %
                                  (method, url, _timeout))
            except requests.exceptions.HTTPError as e:
                if e.response.status_code == 404 or e.response.status_code == 400:
                    pass
                else:
                    raise
            except TypeError as e:
                raise TypeError(
                    "Your installed version of 'requests' library seems not compatible with"
                    "Datadog's usage. We recommand upgrading it ('pip install -U requests')."
                    "If you need help or have any question, please contact [email protected]"
                )

            # Request succeeded: log it and reset the timeout counter
            duration = round((time.time() - start_time) * 1000., 4)
            log.info("%s %s %s (%sms)" %
                     (result.status_code, method, url, duration))
            cls._timeout_counter = 0

            # Format response content
            content = result.content

            if content:
                try:
                    if is_p3k():
                        response_obj = json.loads(content.decode('utf-8'))
                    else:
                        response_obj = json.loads(content)
                except ValueError:
                    raise ValueError(
                        'Invalid JSON response: {0}'.format(content))

                if response_obj and 'errors' in response_obj:
                    raise ApiError(response_obj)
            else:
                response_obj = None
            if response_formatter is None:
                return response_obj
            else:
                return response_formatter(response_obj)

        except ClientError as e:
            if _swallow:
                log.error(str(e))
                if error_formatter is None:
                    return {'errors': e.args[0]}
                else:
                    return error_formatter({'errors': e.args[0]})
            else:
                raise
        except ApiError as e:
            if _swallow:
                for error in e.args[0]['errors']:
                    log.error(str(error))
                if error_formatter is None:
                    return e.args[0]
                else:
                    return error_formatter(e.args[0])
            else:
                raise
예제 #31
0
파일: config.py 프로젝트: msalib/datadogpy
def skip_leading_wsp(f):
    "Works on a file, returns a file-like object"
    if is_p3k():
        return StringIO("\n".join(x.strip(" ") for x in f.readlines()))
    else:
        return StringIO("\n".join(map(string.strip, f.readlines())))
예제 #32
0
파일: wrap.py 프로젝트: yesyouken/datadogpy
def parse_options(raw_args=None):
    '''
    Parse the raw command line options into an options object and the remaining command string
    '''
    parser = optparse.OptionParser(
        usage="%prog -n [event_name] -k [api_key] --submit_mode \
[ all | errors | warnings] [options] \"command\". \n\nNote that you need to enclose your command in \
quotes to prevent python executing as soon as there is a space in your command. \n \nNOTICE: In \
normal mode, the whole stderr is printed before stdout, in flush_live mode they will be mixed but \
there is not guarantee that messages sent by the command on both stderr and stdout are printed in \
the order they were sent.",
        version="%prog {0}".format(__version__),
        option_class=DogwrapOption)

    parser.add_option('-n',
                      '--name',
                      action='store',
                      type='string',
                      help="the name of the event \
as it should appear on your Datadog stream")
    parser.add_option('-k',
                      '--api_key',
                      action='store',
                      type='string',
                      help="your DataDog API Key",
                      default=os.environ.get("DD_API_KEY"))
    parser.add_option('-s',
                      '--site',
                      action='store',
                      type='choice',
                      default='datadoghq.com',
                      choices=['datadoghq.com', 'us', 'datadoghq.eu', 'eu'],
                      help="The site \
to send data, us (datadoghq.com) or eu (datadoghq.eu), default: us")
    parser.add_option('-m',
                      '--submit_mode',
                      action='store',
                      type='choice',
                      default='errors',
                      choices=['errors', 'warnings', 'all'],
                      help="[ all | errors | warnings ] if set \
to error, an event will be sent only of the command exits with a non zero exit status or if it \
times out. If set to warning, a list of exit codes need to be provided")
    parser.add_option(
        '--warning_codes',
        action='store',
        type='warning_codes',
        dest='warning_codes',
        help="comma separated list of warning codes, e.g: 127,255")
    parser.add_option('-p',
                      '--priority',
                      action='store',
                      type='choice',
                      choices=['normal', 'low'],
                      help="the priority of the event (default: 'normal')")
    parser.add_option(
        '-t',
        '--timeout',
        action='store',
        type='int',
        default=60 * 60 * 24,
        help=
        "(in seconds)  a timeout after which your command must be aborted. An \
event will be sent to your DataDog stream (default: 24hours)")
    parser.add_option('--sigterm_timeout',
                      action='store',
                      type='int',
                      default=60 * 2,
                      help="(in seconds)  When your command times out, the \
process it triggers is sent a SIGTERM. If this sigterm_timeout is reached, it will be sent a \
SIGKILL signal. (default: 2m)")
    parser.add_option(
        '--sigkill_timeout',
        action='store',
        type='int',
        default=60,
        help="(in seconds) how long to wait at most after SIGKILL \
                              has been sent (default: 60s)")
    parser.add_option(
        '--proc_poll_interval',
        action='store',
        type='float',
        default=0.5,
        help="(in seconds). interval at which your command will be polled \
(default: 500ms)")
    parser.add_option(
        '--notify_success',
        action='store',
        type='string',
        default='',
        help="a message string and @people directives to send notifications in \
case of success.")
    parser.add_option(
        '--notify_error',
        action='store',
        type='string',
        default='',
        help="a message string and @people directives to send notifications in \
case of error.")
    parser.add_option(
        '--notify_warning',
        action='store',
        type='string',
        default='',
        help="a message string and @people directives to send notifications in \
    case of warning.")
    parser.add_option(
        '-b',
        '--buffer_outs',
        action='store_true',
        dest='buffer_outs',
        default=False,
        help="displays the stderr and stdout of the command only once it has \
returned (the command outputs remains buffered in dogwrap meanwhile)")
    parser.add_option('--send_metric',
                      action='store_true',
                      dest='send_metric',
                      default=False,
                      help="sends a metric for event duration")
    parser.add_option('--tags',
                      action='store',
                      type='string',
                      dest='tags',
                      default='',
                      help="comma separated list of tags")

    options, args = parser.parse_args(args=raw_args)

    if is_p3k():
        cmd = ' '.join(args)
    else:
        cmd = b' '.join(args).decode('utf-8')

    return options, cmd
예제 #33
0
파일: config.py 프로젝트: kuzmich/datadogpy
def skip_leading_wsp(f):
    "Works on a file, returns a file-like object"
    if is_p3k():
        return StringIO("\n".join(x.strip(" ") for x in f.readlines()))
    else:
        return StringIO("\n".join(map(string.strip, f.readlines())))