Exemplo n.º 1
0
class KeystoneProbe(wsgi.Middleware):
    def __init__(self, app, conf):
        self.app = app
        self.config = conf
        self.log = logging.getLogger(__name__)
        self.statsd = Statsd(conf)

    def __call__(self, environ, start_response):
        return self.process_request(environ, start_response)

    def str2bool(self, str):
        if not str:
            return False
        if str in ['false', 'False', 'no', '0']:
            return False
        return True

    def get_username(self, environ):
        '''Extracts auth details from requests and returns a tuple (username, password) if found,
        otherwise returns None'''
        try:
            auth = environ['openstack.params']['auth']['passwordCredentials']
            username = auth['username']
        except KeyError:
            self.log.debug('No authentication context in request')
            return None
        return username

    def statsd_event(self, environ):
        username = self.get_username(environ)
        status_int = environ['statsd.status']
        if username:
            # Authentication request
            username = username.replace('.', '_')
            self.statsd.increment('auth.{0}.{1}'.format(username, status_int))

        path = environ['PATH_INFO']
        method = environ['REQUEST_METHOD']
        # Comment this out for now, generates a lot of excess metrics
        #self.statsd.increment('req.{0}.{1}.{2}'.format(path, method, status_int))

    def process_request(self, environ, start_response):
        def _start_response(status, headers, exc_info=None):
            """start_response wrapper to grab headers and status code"""
            # Convert all headers to lower case
            new_h = [(k.lower(), v) for k,v in headers]
            environ['statsd.headers'] = new_h
            environ['statsd.status'] = int(status.split(' ', 1)[0])
            start_response(status, headers, exc_info)

        # Register a post-hook to be called after the request completes
        if not 'eventlet.posthooks' in environ:
            environ['eventlet.posthooks'] = []
        req = Request(environ)
        environ['eventlet.posthooks'].append((self.statsd_event, (), {}))
        return self.app(environ, _start_response)
Exemplo n.º 2
0
import string
import sys

from kafka import KafkaConsumer as Consumer, TopicPartition as Partition
from statsd import StatsClient as Statsd

if __name__ == '__main__':

    state = json.loads(''.join(sys.stdin))

    try:

        js = json.loads(os.environ['KONTROL_ANNOTATIONS'])
        broker = random.choice(js['kafka.unity3d.com/brokers'].split(','))
        topic = js['kafka.unity3d.com/topic']
        statsd = Statsd('127.0.0.1', 8125)
        consumer = Consumer(bootstrap_servers=broker)
        partition = Partition(topic, 0)
        consumer.assign([partition])

        if not state['last']:

            #
            # - the produce/consume loop has just started
            # - reset to the head
            #
            consumer.seek_to_end()

        else:

            #
Exemplo n.º 3
0
 def __init__(self, app, conf):
     self.app = app
     self.config = conf
     self.log = logging.getLogger(__name__)
     self.statsd = Statsd(conf)
Exemplo n.º 4
0
 def __init__(self, app, conf, *args, **kwargs):
     self.statsd = Statsd(conf)
     self.logger = get_logger(conf, log_route='probe')
     self.app = app
     self.pp = PrettyPrinter(indent=4)
Exemplo n.º 5
0
class ProbeMiddleware(object):
    """
    Probe middleware used for monitoring and statistics gathering.

    It will sit in the request pipeline doing nothing, but offers a place to
    send metrics to a system.
    """

    def __init__(self, app, conf, *args, **kwargs):
        self.statsd = Statsd(conf)
        self.logger = get_logger(conf, log_route='probe')
        self.app = app
        self.pp = PrettyPrinter(indent=4)

    def GET(self, req):
        """Returns a 200 response with "OK" in the body."""
        return Response(request=req, body="OK", content_type="text/plain")

    def statsd_event(self, env, req):
        try:
            request_time = time() - env['swprobe.start_time']
            headers = dict(env['swprobe.headers'])
            response = getattr(req, 'response', None)
            if getattr(req, 'client_disconnect', False) or \
                       getattr(response, 'client_disconnect', False):
                status_int = 499
            else:
                status_int = env['swprobe.status']
            duration = (time() - env['swprobe.start_time']) * 1000

            # Find out how much bytes were transferred. For PUTs we can get this from the request object,
            # but for GETs we look at the Content-Length header of the response
            # Don't know how to find out # bytes transferred for aborted transfers
            transferred = getattr(req, 'bytes_transferred', 0)
            if transferred == 0 and 'CONTENT_LENGTH' in env.keys():
                transferred = env['CONTENT_LENGTH']
            transferred = 0 if transferred == '-' else int(transferred)
            if transferred == 0 and status_int != 499 and req.method == "GET":
                transferred = headers['content-length']
            if req.path.startswith("/auth"):
                # Time how long auth request takes
                self.statsd.increment("req.auth")
                self.statsd.timing("auth", duration)
                return None
            # Find out for which account the request was made
            try:
                if 'keystone.identity' in env.keys():
                    # If authenticated by keystone
                    swift_account = env['keystone.identity']['tenant'][1]
                else:
                    # If authenticated by swauth or externally
                    swift_account = env["REMOTE_USER"].split(",")[1]
            except:
                swift_account = "anonymous"
            self.statsd.increment("req.%s.%s.%s" %(swift_account, req.method, status_int))
            if status_int >= 200 and status_int < 400:
                # Log timers for succesful requests
                self.statsd.timing("%s" %(req.method), duration)
            # Upload and download size statistics
            if req.method == "PUT":
                self.statsd.update_stats("xfer.%s.bytes_uploaded" % swift_account, transferred)
            elif req.method == "GET":
                self.statsd.update_stats("xfer.%s.bytes_downloaded" % swift_account, transferred)
        except Exception as e:
            try:
                self.logger.exception(_("Encountered error in statsd_event"))
                self.logger.exception(e)
            except Exception:
                pass



    def __call__(self, env, start_response):
        """WSGI callable"""

        def _start_response(status, headers, exc_info=None):
            """start_response wrapper to grab headers and status code"""
            # Convert all headers to lower case
            new_h = [(k.lower(), v) for k,v in headers]
            env['swprobe.headers'] = new_h
            env['swprobe.status'] = int(status.split(' ', 1)[0])
            start_response(status, headers, exc_info)

        req = Request(env)
        env['swprobe.start_time'] = time()
        if not 'eventlet.posthooks' in env:
            env['eventlet.posthooks'] = []
        env['eventlet.posthooks'].append((self.statsd_event, (req,), {}))
        return self.app(env, _start_response)