Пример #1
0
    def getChildWithDefault(self, path, request):
        headers = request.requestHeaders

        # Construct and endpoint from the requested host and port and the
        # passed service name.
        host = request.getHost()

        endpoint = Endpoint(host.host, host.port, self._service_name)

        # Construct the trace using the headers X-B3-* headers that the
        # TracingAgent will send.
        trace = Trace(
            request.method,
            int_or_none(headers.getRawHeaders('X-B3-TraceId', [None])[0]),
            int_or_none(headers.getRawHeaders('X-B3-SpanId', [None])[0]),
            int_or_none(headers.getRawHeaders('X-B3-ParentSpanId', [None])[0]))

        trace.set_endpoint(endpoint)

        # twisted.web.server.Request is a subclass of Componentized this
        # allows us to use ITrace(request) to get the trace and object (and
        # create children of this trace) in the wrapped resource.
        request.setComponent(ITrace, trace)

        trace.record(Annotation.server_recv())

        def _record_finish(_ignore):
            trace.record(Annotation.server_send())

        # notifyFinish returns a deferred that fires when the request is
        # finished this will allow us to record server_send regardless of if
        # the wrapped resource us using deferred rendering.
        request.notifyFinish().addCallback(_record_finish)

        return self._wrapped.getChildWithDefault(path, request)
Пример #2
0
    def render_POST(self, request):
        print "For CORS by john"

        request.setHeader('Access-Control-Allow-Origin', '*')
        request.setHeader('Access-Control-Allow-Methods', 'POST')
        request.setHeader('Access-Control-Allow-Headers', 'authorization, content-type')

        request.responseHeaders.setRawHeaders(
            'content-type', ['application/json'])

        body = request.content.read()

        try:
            spans = json.loads(body)
        except ValueError:
            log.err(None, 'Failed to decode request body')
            msg = 'Could not decode request body (invalid JSON)'
            return json.dumps({'error': msg})

        succeeded, failed = 0, 0

        for json_span in spans:
            trace_id = None
            span_id = None

            try:
                trace_id = decode_hex_number('trace_id', json_span['trace_id'])
                span_id = decode_hex_number('span_id', json_span['span_id'])
                parent_span_id = json_span.get('parent_span_id', None)

                if parent_span_id is not None:
                    parent_span_id = decode_hex_number('parent_span_id',
                                                       parent_span_id)

                t = Trace(json_span['name'], trace_id, span_id, parent_span_id)

                for json_annotation in json_span['annotations']:
                    annotation = Annotation(
                        json_annotation['key'],
                        json_annotation['value'],
                        json_annotation['type'])

                    host = json_annotation.get('host', None)

                    if host:
                        annotation.endpoint = Endpoint(
                            host['ipv4'], host['port'], host['service_name'])

                    t.record(annotation)
                    succeeded = succeeded + 1
            except Exception:
                log.err(None,
                        'Failed to insert a trace: trace_id=%r,span_id=%r' %
                        (trace_id, span_id))

                failed = failed + 1
                continue

        return json.dumps({'succeeded': succeeded, 'failed': failed})
Пример #3
0
    def test_Trace_child(self):
        t = Trace('test_trace', trace_id=1, span_id=1)

        c = t.child('child_test_trace')

        self.assertEqual(c.trace_id, 1)
        self.assertEqual(c.parent_span_id, 1)
        self.assertNotEqual(c.span_id, 1)
Пример #4
0
    def test_record_invokes_tracer(self):
        tracer = mock.Mock()

        t = Trace('test_trace', trace_id=1, span_id=1, tracers=[tracer])
        annotation = Annotation.client_send(timestamp=0)
        t.record(annotation)

        tracer.record.assert_called_with([(t, (annotation,))])
Пример #5
0
    def test_Trace_child(self):
        t = Trace('test_trace', trace_id=1, span_id=1)

        c = t.child('child_test_trace')

        self.assertEqual(c.trace_id, 1)
        self.assertEqual(c.parent_span_id, 1)
        self.assertNotEqual(c.span_id, 1)
Пример #6
0
    def test_record_invokes_tracer(self):
        tracer = mock.Mock()

        t = Trace('test_trace', trace_id=1, span_id=1, tracers=[tracer])
        annotation = Annotation.client_send(timestamp=0)
        t.record(annotation)

        tracer.record.assert_called_with([(t, (annotation, ))])
Пример #7
0
    def render_POST(self, request):
        print "For CORS by john"

        request.setHeader('Access-Control-Allow-Origin', '*')
        request.setHeader('Access-Control-Allow-Methods', 'POST')

        request.responseHeaders.setRawHeaders('content-type',
                                              ['application/json'])

        body = request.content.read()

        try:
            spans = json.loads(body)
        except ValueError:
            log.err(None, 'Failed to decode request body')
            msg = 'Could not decode request body (invalid JSON)'
            return json.dumps({'error': msg})

        succeeded, failed = 0, 0

        for json_span in spans:
            trace_id = None
            span_id = None

            try:
                trace_id = decode_hex_number('trace_id', json_span['trace_id'])
                span_id = decode_hex_number('span_id', json_span['span_id'])
                parent_span_id = json_span.get('parent_span_id', None)

                if parent_span_id is not None:
                    parent_span_id = decode_hex_number('parent_span_id',
                                                       parent_span_id)

                t = Trace(json_span['name'], trace_id, span_id, parent_span_id)

                for json_annotation in json_span['annotations']:
                    annotation = Annotation(json_annotation['key'],
                                            json_annotation['value'],
                                            json_annotation['type'])

                    host = json_annotation.get('host', None)

                    if host:
                        annotation.endpoint = Endpoint(host['ipv4'],
                                                       host['port'],
                                                       host['service_name'])

                    t.record(annotation)
                    succeeded = succeeded + 1
            except Exception:
                log.err(
                    None, 'Failed to insert a trace: trace_id=%r,span_id=%r' %
                    (trace_id, span_id))

                failed = failed + 1
                continue

        return json.dumps({'succeeded': succeeded, 'failed': failed})
Пример #8
0
    def test_logs_to_scribe_immediately(self):
        tracer = RawZipkinTracer(self.scribe)
        t = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])

        t.record(Annotation.client_send(1))
        self.scribe.log.assert_called_once_with(
            'zipkin',
            ['CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
             'MAAAAAQoAAQAA\nAAAAAAABCwACAAAAAmNzAA8ACAwAAAAAAA=='])
Пример #9
0
    def test_logs_to_scribe_immediately(self):
        tracer = RawZipkinTracer(self.scribe)
        t = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])

        t.record(Annotation.client_send(1))
        self.scribe.log.assert_called_once_with('zipkin', [
            'CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
            'MAAAAAQoAAQAA\nAAAAAAABCwACAAAAAmNzAA8ACAwAAAAAAA=='
        ])
Пример #10
0
    def test_logs_to_scribe_with_non_default_category(self):
        tracer = RawZipkinTracer(self.scribe, 'not-zipkin')
        t = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])

        t.record(Annotation.client_send(1), Annotation.client_recv(2))

        self.scribe.log.assert_called_once_with('not-zipkin', [
            'CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
            'MAAAAAgoAAQAA\nAAAAAAABCwACAAAAAmNzAAoAAQAAAAAAAAACCwACAAAAAmNy'
            'AA8ACAwAAAAAAA=='
        ])
Пример #11
0
    def test_logs_to_scribe_with_non_default_category(self):
        tracer = RawZipkinTracer(self.scribe, 'not-zipkin')
        t = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])

        t.record(Annotation.client_send(1), Annotation.client_recv(2))

        self.scribe.log.assert_called_once_with(
            'not-zipkin',
            ['CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
             'MAAAAAgoAAQAA\nAAAAAAABCwACAAAAAmNzAAoAAQAAAAAAAAACCwACAAAAAmNy'
             'AA8ACAwAAAAAAA=='])
Пример #12
0
    def test_writes_trace(self):
        t = Trace('test', 1, 2, tracers=[self.tracer])
        t.record(Annotation.client_send(1))

        self.assertEqual(
            json.loads(self.destination.getvalue()),
            [{'trace_id': '0000000000000001',
              'span_id': '0000000000000002',
              'name': 'test',
              'annotations': [
                  {'type': 'timestamp', 'value': 1, 'key': 'cs'}]}])
Пример #13
0
    def test_record_sets_annotation_endpoint(self):
        tracer = mock.Mock()
        web_endpoint = Endpoint('127.0.0.1', 8080, 'web')

        t = Trace('test_trace', trace_id=1, span_id=1, tracers=[tracer])
        t.set_endpoint(web_endpoint)
        annotation = Annotation.client_send(timestamp=1)
        t.record(annotation)

        tracer.record.assert_called_with([(t, (annotation,))])

        self.assertEqual(annotation.endpoint, web_endpoint)
Пример #14
0
    def test_delegates_on_end_annotation(self):
        tracer = EndAnnotationTracer(self.tracer)

        t = Trace('test_delegation', tracers=[tracer])

        cs = Annotation.client_send()
        ce = Annotation.client_recv()

        t.record(cs)
        t.record(ce)

        self.tracer.record.assert_called_once_with([(t, [cs, ce])])
Пример #15
0
    def test_delegates_on_end_annotation(self):
        tracer = EndAnnotationTracer(self.tracer)

        t = Trace('test_delegation', tracers=[tracer])

        cs = Annotation.client_send()
        ce = Annotation.client_recv()

        t.record(cs)
        t.record(ce)

        self.tracer.record.assert_called_once_with([(t, [cs, ce])])
Пример #16
0
    def test_handles_batched_traces(self):
        t1 = Trace('test', 1, 2)
        t2 = Trace('test2', 3, 4)

        cs1 = Annotation.client_send(1)
        cs2 = Annotation.client_send(2)
        cr1 = Annotation.client_recv(3)
        cr2 = Annotation.client_recv(4)

        self.tracer.record([(t1, [cs1, cr1]), (t2, [cs2, cr2])])

        self.assertEqual(self.scribe.log.call_count, 1)

        args = self.scribe.log.mock_calls[0][1]

        self.assertEqual('restkin', args[0])
        entries = args[1]
        self.assertEqual(len(entries), 1)

        self.assertEqual(json.loads(entries[0]), [{
            'trace_id':
            '0000000000000001',
            'span_id':
            '0000000000000002',
            'name':
            'test',
            'annotations': [{
                'type': 'timestamp',
                'value': 1,
                'key': 'cs'
            }, {
                'type': 'timestamp',
                'value': 3,
                'key': 'cr'
            }]
        }, {
            'trace_id':
            '0000000000000003',
            'span_id':
            '0000000000000004',
            'name':
            'test2',
            'annotations': [{
                'type': 'timestamp',
                'value': 2,
                'key': 'cs'
            }, {
                'type': 'timestamp',
                'value': 4,
                'key': 'cr'
            }]
        }])
Пример #17
0
    def test_non_default_end(self):
        tracer = EndAnnotationTracer(self.tracer, end_annotations=['timeout'])

        t = Trace('test_non-default', tracers=[tracer])

        cs = Annotation.client_send()

        t.record(cs)

        timeout = Annotation.timestamp('timeout')

        t.record(timeout)

        self.tracer.record.assert_called_once_with([(t, [cs, timeout])])
Пример #18
0
    def test_inequality(self):
        trace = Trace('test_trace', trace_id=1, span_id=1, parent_span_id=1)

        self.assertNotEqual(trace, None)

        self.assertNotEqual(
            trace, Trace('test_trace', trace_id=2, span_id=1,
                         parent_span_id=1))
        self.assertNotEqual(
            trace, Trace('test_trace', trace_id=1, span_id=2,
                         parent_span_id=1))
        self.assertNotEqual(
            trace, Trace('test_trace', trace_id=1, span_id=1,
                         parent_span_id=2))
Пример #19
0
    def test_writes_trace(self):
        t = Trace('test', 1, 2, tracers=[self.tracer])
        t.record(Annotation.client_send(1))

        self.assertEqual(
            json.loads(self.destination.getvalue()),
            [{
                'trace_id': '0000000000000001',
                'span_id': '0000000000000002',
                'name': 'test',
                'annotations': [{
                    'type': 'timestamp',
                    'value': 1,
                    'key': 'cs'
                }]
            }])
Пример #20
0
    def __init__(self, method, uri, version="HTTP/1.0", headers=None,
                 body=None, remote_ip=None, protocol=None, host=None,
                 files=None, connection=None):
        self.method = method
        self.uri = uri
        self.version = version
        self.headers = headers or httputil.HTTPHeaders()
        self.body = body or ""
        self.trace = None

        # set remote IP and protocol
        self.remote_ip = remote_ip
        if protocol:
            self.protocol = protocol
        elif connection and isinstance(connection.stream,
                                       iostream.SSLIOStream):
            self.protocol = "https"
        else:
            self.protocol = "http"

        # xheaders can override the defaults
        if connection and connection.xheaders:
            # Squid uses X-Forwarded-For, others use X-Real-Ip
            ip = self.headers.get("X-Forwarded-For", self.remote_ip)
            ip = ip.split(',')[-1].strip()
            ip = self.headers.get(
                "X-Real-Ip", ip)
            if netutil.is_valid_ip(ip):
                self.remote_ip = ip
            # AWS uses X-Forwarded-Proto
            proto = self.headers.get(
                "X-Scheme", self.headers.get("X-Forwarded-Proto", self.protocol))
            if proto in ("http", "https"):
                self.protocol = proto
            # Zipkin users 
            if options.server_trace:
                parent_span_id = self.headers.get("X-B3-ParentSpanId", None)
                trace_id = self.headers.get("X-B3-TraceId", None)
                span_id = self.headers.get("X-B3-SpanId", None)
                name =  method
                endpoint = Endpoint(ipv4=socket.gethostbyname(socket.gethostname()), port=port, service_name=service_name)
                
                self.trace = Trace(name=name, trace_id=trace_id, span_id=span_id, parent_span_id=parent_span_id)
                self.trace.set_endpoint(endpoint)

        self.host = host or self.headers.get("Host") or "127.0.0.1"
        self.files = files or {}
        self.connection = connection
        self._start_time = time.time()
        self._finish_time = None

        self.path, sep, self.query = uri.partition('?')
        self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)
        self.query_arguments = copy.deepcopy(self.arguments)
        self.body_arguments = {}
        
        if options.server_trace:
            self.trace.record(Annotation.string('Url', uri))
            self.trace.record(Annotation.string('Header', self.headers))
            self.trace.record(Annotation.server_recv())
Пример #21
0
    def test_handles_batched_traces(self):
        tracer = EndAnnotationTracer(self.tracer)

        t1 = Trace('test1', tracers=[tracer])
        t2 = Trace('test2', tracers=[tracer])

        cs1 = Annotation.client_send()
        cs2 = Annotation.client_send()

        cr1 = Annotation.client_recv()
        cr2 = Annotation.client_recv()

        tracer.record([(t1, [cs1, cr1]), (t2, [cs2, cr2])])

        self.assertEqual(self.tracer.record.call_count, 2)

        self.tracer.record.assert_any_call([(t1, [cs1, cr1])])
        self.tracer.record.assert_any_call([(t2, [cs2, cr2])])
Пример #22
0
    def test_traces_immediately(self):
        t = Trace('test', 1, 2, tracers=[self.tracer])
        t.record(Annotation.client_send(1))

        self.assertEqual(self.scribe.log.call_count, 1)

        args = self.scribe.log.mock_calls[0][1]

        self.assertEqual('restkin', args[0])
        entries = args[1]
        self.assertEqual(len(entries), 1)

        self.assertEqual(
            json.loads(entries[0]),
            [{'trace_id': '0000000000000001',
             'span_id': '0000000000000002',
             'name': 'test',
             'annotations': [
                 {'type': 'timestamp', 'value': 1, 'key': 'cs'}]}])
Пример #23
0
    def test_handles_batched_traces(self):
        tracer = RawZipkinTracer(self.scribe)
        t1 = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])

        cs1 = Annotation.client_send(1)

        t2 = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])

        cs2 = Annotation.client_send(1)
        cr2 = Annotation.client_recv(2)

        tracer.record([(t1, [cs1]), (t2, [cs2, cr2])])

        self.scribe.log.assert_called_once_with('zipkin', [
            'CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
            'MAAAAAQoAAQAA\nAAAAAAABCwACAAAAAmNzAA8ACAwAAAAAAA==',
            'CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
            'MAAAAAgoAAQAA\nAAAAAAABCwACAAAAAmNzAAoAAQAAAAAAAAACCwACAAAAAmNy'
            'AA8ACAwAAAAAAA=='
        ])
Пример #24
0
    def test_new_Trace(self):
        t = Trace('test_trace')
        self.assertNotEqual(t.trace_id, None)
        self.assertIsInstance(t.trace_id, (int, long))
        self.failIf(t.trace_id >= MAX_ID)

        self.assertNotEqual(t.span_id, None)
        self.assertIsInstance(t.span_id, (int, long))
        self.failIf(t.span_id >= MAX_ID)

        self.assertEqual(t.parent_span_id, None)
Пример #25
0
    def request(self, method, uri, headers=None, bodyProducer=None):
        """
        Send a client request following HTTP redirects.

        @see: L{Agent.request}.
        """
        if self._parent_trace is None:
            trace = Trace(method)
        else:
            trace = self._parent_trace.child(method)

        if self._endpoint is not None:
            trace.set_endpoint(self._endpoint)

        if headers is None:
            headers = Headers({})

        # These headers are based on the headers used by finagle's tracing
        # http Codec.
        #
        # https://github.com/twitter/finagle/blob/master/finagle-http/
        #
        # Currently not implemented are X-B3-Sampled and X-B3-Flags
        # Tryfer's underlying Trace implementation has no notion of a Sampled
        # trace and I haven't figured out what flags are for.
        headers.setRawHeaders('X-B3-TraceId', [hex_str(trace.trace_id)])
        headers.setRawHeaders('X-B3-SpanId', [hex_str(trace.span_id)])

        if trace.parent_span_id is not None:
            headers.setRawHeaders('X-B3-ParentSpanId',
                                  [hex_str(trace.parent_span_id)])

        # Similar to the headers above we use the annotation 'http.uri' for
        # because that is the standard set forth in the finagle http Codec.
        trace.record(Annotation.string('http.uri', uri))
        trace.record(Annotation.client_send())

        def _finished(resp):
            # TODO: It may be advantageous here to return a wrapped response
            # whose deliverBody can wrap it's protocol and record when the
            # application has finished reading the contents.
            trace.record(
                Annotation.string('http.responsecode',
                                  '{0} {1}'.format(resp.code, resp.phrase)))
            trace.record(Annotation.client_recv())
            return resp

        d = self._agent.request(method, uri, headers, bodyProducer)
        d.addBoth(_finished)

        return d
Пример #26
0
    def test_traces_buffered_until_max_idle_time(self):
        completed_trace = (Trace('completed'), [
            Annotation.client_send(1),
            Annotation.client_recv(2)
        ])

        self.tracer.record([completed_trace])
        self.assertEqual(self.record_function.call_count, 0)

        self.clock.advance(10)

        self.assertEqual(self.record_function.call_count, 1)
Пример #27
0
    def getChildWithDefault(self, path, request):
        headers = request.requestHeaders

        # Construct and endpoint from the requested host and port and the
        # passed service name.
        host = request.getHost()

        endpoint = Endpoint(host.host, host.port, self._service_name)

        # Construct the trace using the headers X-B3-* headers that the
        # TracingAgent will send.
        trace = Trace(
            request.method,
            int_or_none(headers.getRawHeaders('X-B3-TraceId', [None])[0]),
            int_or_none(headers.getRawHeaders('X-B3-SpanId', [None])[0]),
            int_or_none(headers.getRawHeaders('X-B3-ParentSpanId', [None])[0]))

        trace.set_endpoint(endpoint)

        # twisted.web.server.Request is a subclass of Componentized this
        # allows us to use ITrace(request) to get the trace and object (and
        # create children of this trace) in the wrapped resource.
        request.setComponent(ITrace, trace)

        trace.record(Annotation.server_recv())

        def _record_finish(_ignore):
            trace.record(Annotation.server_send())

        # notifyFinish returns a deferred that fires when the request is
        # finished this will allow us to record server_send regardless of if
        # the wrapped resource us using deferred rendering.
        request.notifyFinish().addCallback(_record_finish)

        return self._wrapped.getChildWithDefault(path, request)
Пример #28
0
    def test_handles_batched_traces(self):
        t1 = self.trace
        t2 = Trace('test2', 3, 4)

        cs1 = Annotation.client_send(1)
        cs2 = Annotation.client_send(2)
        cr1 = Annotation.client_recv(3)
        cr2 = Annotation.client_recv(4)

        self.tracer.record([(t1, [cs1, cr1]), (t2, [cs2, cr2])])

        self.assertEqual(self.agent.request.call_count, 1)

        args = self.agent.request.mock_calls[0][1]
        self.assertEqual(('POST', 'http://trace.it', Headers({})), args[:3])

        bodyProducer = args[3]

        return self.assertBodyEquals(bodyProducer, [{
            'trace_id':
            '0000000000000001',
            'span_id':
            '0000000000000002',
            'name':
            'test',
            'annotations': [{
                'type': 'timestamp',
                'value': 1,
                'key': 'cs'
            }, {
                'type': 'timestamp',
                'value': 3,
                'key': 'cr'
            }]
        }, {
            'trace_id':
            '0000000000000003',
            'span_id':
            '0000000000000004',
            'name':
            'test2',
            'annotations': [{
                'type': 'timestamp',
                'value': 2,
                'key': 'cs'
            }, {
                'type': 'timestamp',
                'value': 4,
                'key': 'cr'
            }]
        }])
Пример #29
0
    def request(self, method, uri, headers=None, bodyProducer=None):
        """
        Send a client request following HTTP redirects.

        @see: L{Agent.request}.
        """
        if self._parent_trace is None:
            trace = Trace(method)
        else:
            trace = self._parent_trace.child(method)

        if self._endpoint is not None:
            trace.set_endpoint(self._endpoint)

        if headers is None:
            headers = Headers({})

        # These headers are based on the headers used by finagle's tracing
        # http Codec.
        #
        # https://github.com/twitter/finagle/blob/master/finagle-http/
        #
        # Currently not implemented are X-B3-Sampled and X-B3-Flags
        # Tryfer's underlying Trace implementation has no notion of a Sampled
        # trace and I haven't figured out what flags are for.
        headers.setRawHeaders('X-B3-TraceId', [hex_str(trace.trace_id)])
        headers.setRawHeaders('X-B3-SpanId', [hex_str(trace.span_id)])

        if trace.parent_span_id is not None:
            headers.setRawHeaders('X-B3-ParentSpanId',
                                  [hex_str(trace.parent_span_id)])

        # Similar to the headers above we use the annotation 'http.uri' for
        # because that is the standard set forth in the finagle http Codec.
        trace.record(Annotation.string('http.uri', uri))
        trace.record(Annotation.client_send())

        def _finished(resp):
            # TODO: It may be advantageous here to return a wrapped response
            # whose deliverBody can wrap it's protocol and record when the
            # application has finished reading the contents.
            trace.record(Annotation.string(
                'http.responsecode',
                '{0} {1}'.format(resp.code, resp.phrase)))
            trace.record(Annotation.client_recv())
            return resp

        d = self._agent.request(method, uri, headers, bodyProducer)
        d.addBoth(_finished)

        return d
Пример #30
0
    def test_traces_immediately(self):
        t = Trace('test', 1, 2, tracers=[self.tracer])
        t.record(Annotation.client_send(1))

        self.assertEqual(self.scribe.log.call_count, 1)

        args = self.scribe.log.mock_calls[0][1]

        self.assertEqual('restkin', args[0])
        entries = args[1]
        self.assertEqual(len(entries), 1)

        self.assertEqual(
            json.loads(entries[0]),
            [{
                'trace_id': '0000000000000001',
                'span_id': '0000000000000002',
                'name': 'test',
                'annotations': [{
                    'type': 'timestamp',
                    'value': 1,
                    'key': 'cs'
                }]
            }])
Пример #31
0
    def test_traces_bufferred_until_max_traces(self):
        completed_traces = [
            (Trace('completed'),
             [Annotation.client_send(1),
              Annotation.client_recv(2)]) for x in xrange(50)
        ]

        self.tracer.record(completed_traces[:10])
        self.clock.advance(1)

        self.assertEqual(self.record_function.call_count, 0)
        self.tracer.record(completed_traces[10:])

        self.clock.advance(1)

        self.assertEqual(self.record_function.call_count, 1)
Пример #32
0
    def test_record_sets_annotation_endpoint(self):
        tracer = mock.Mock()
        web_endpoint = Endpoint('127.0.0.1', 8080, 'web')

        t = Trace('test_trace', trace_id=1, span_id=1, tracers=[tracer])
        t.set_endpoint(web_endpoint)
        annotation = Annotation.client_send(timestamp=1)
        t.record(annotation)

        tracer.record.assert_called_with([(t, (annotation, ))])

        self.assertEqual(annotation.endpoint, web_endpoint)
Пример #33
0
    def test_non_default_end(self):
        tracer = EndAnnotationTracer(self.tracer, end_annotations=['timeout'])

        t = Trace('test_non-default', tracers=[tracer])

        cs = Annotation.client_send()

        t.record(cs)

        timeout = Annotation.timestamp('timeout')

        t.record(timeout)

        self.tracer.record.assert_called_once_with([(t, [cs, timeout])])
Пример #34
0
class RawRESTkinHTTPTracerTests(TestCase):
    def assertBodyEquals(self, bodyProducer, expectedOutput):
        output = StringIO()
        consumer = FileConsumer(output)

        def _check_body(_):
            self.assertEqual(
                json.loads(output.getvalue()),
                expectedOutput
            )

        d = bodyProducer.startProducing(consumer)
        d.addCallback(_check_body)

        return d

    def setUp(self):
        self.agent = mock.Mock()

        self.tracer = RawRESTkinHTTPTracer(self.agent, 'http://trace.it')
        self.trace = Trace('test', 1, 2, tracers=[self.tracer])

    def test_verifyObject(self):
        verifyObject(ITracer, self.tracer)

    def test_posts_immediately(self):
        self.trace.record(Annotation.client_send(1))

        self.assertEqual(self.agent.request.call_count, 1)

        args = self.agent.request.mock_calls[0][1]
        self.assertEqual(('POST', 'http://trace.it', Headers({})), args[:3])

        bodyProducer = args[3]

        return self.assertBodyEquals(
            bodyProducer,
            [{'trace_id': '0000000000000001',
              'span_id': '0000000000000002',
              'name': 'test',
              'annotations': [
                  {'type': 'timestamp', 'value': 1, 'key': 'cs'}
              ]}])

    def test_handles_batched_traces(self):
        t1 = self.trace
        t2 = Trace('test2', 3, 4)

        cs1 = Annotation.client_send(1)
        cs2 = Annotation.client_send(2)
        cr1 = Annotation.client_recv(3)
        cr2 = Annotation.client_recv(4)

        self.tracer.record([(t1, [cs1, cr1]), (t2, [cs2, cr2])])

        self.assertEqual(self.agent.request.call_count, 1)

        args = self.agent.request.mock_calls[0][1]
        self.assertEqual(('POST', 'http://trace.it', Headers({})), args[:3])

        bodyProducer = args[3]

        return self.assertBodyEquals(
            bodyProducer,
            [{'trace_id': '0000000000000001',
              'span_id': '0000000000000002',
              'name': 'test',
              'annotations': [
                  {'type': 'timestamp', 'value': 1, 'key': 'cs'},
                  {'type': 'timestamp', 'value': 3, 'key': 'cr'}
              ]},
             {'trace_id': '0000000000000003',
              'span_id': '0000000000000004',
              'name': 'test2',
              'annotations': [
                  {'type': 'timestamp', 'value': 2, 'key': 'cs'},
                  {'type': 'timestamp', 'value': 4, 'key': 'cr'}
              ]}])
Пример #35
0
class HTTPRequest(object):
    """A single HTTP request.

    All attributes are type `str` unless otherwise noted.

    .. attribute:: method

       HTTP request method, e.g. "GET" or "POST"

    .. attribute:: uri

       The requested uri.

    .. attribute:: path

       The path portion of `uri`

    .. attribute:: query

       The query portion of `uri`

    .. attribute:: version

       HTTP version specified in request, e.g. "HTTP/1.1"

    .. attribute:: headers

       `.HTTPHeaders` dictionary-like object for request headers.  Acts like
       a case-insensitive dictionary with additional methods for repeated
       headers.

    .. attribute:: body

       Request body, if present, as a byte string.

    .. attribute:: remote_ip

       Client's IP address as a string.  If ``HTTPServer.xheaders`` is set,
       will pass along the real IP address provided by a load balancer
       in the ``X-Real-Ip`` or ``X-Forwarded-For`` header.

    .. versionchanged:: 3.1
       The list format of ``X-Forwarded-For`` is now supported.

    .. attribute:: protocol

       The protocol used, either "http" or "https".  If ``HTTPServer.xheaders``
       is set, will pass along the protocol used by a load balancer if
       reported via an ``X-Scheme`` header.

    .. attribute:: host

       The requested hostname, usually taken from the ``Host`` header.

    .. attribute:: arguments

       GET/POST arguments are available in the arguments property, which
       maps arguments names to lists of values (to support multiple values
       for individual names). Names are of type `str`, while arguments
       are byte strings.  Note that this is different from
       `.RequestHandler.get_argument`, which returns argument values as
       unicode strings.

    .. attribute:: query_arguments

       Same format as ``arguments``, but contains only arguments extracted
       from the query string.

       .. versionadded:: 3.2

    .. attribute:: body_arguments

       Same format as ``arguments``, but contains only arguments extracted
       from the request body.

       .. versionadded:: 3.2

    .. attribute:: files

       File uploads are available in the files property, which maps file
       names to lists of `.HTTPFile`.

    .. attribute:: connection

       An HTTP request is attached to a single HTTP connection, which can
       be accessed through the "connection" attribute. Since connections
       are typically kept open in HTTP/1.1, multiple requests can be handled
       sequentially on a single connection.
    """
    def __init__(self, method, uri, version="HTTP/1.0", headers=None,
                 body=None, remote_ip=None, protocol=None, host=None,
                 files=None, connection=None):
        self.method = method
        self.uri = uri
        self.version = version
        self.headers = headers or httputil.HTTPHeaders()
        self.body = body or ""
        self.trace = None

        # set remote IP and protocol
        self.remote_ip = remote_ip
        if protocol:
            self.protocol = protocol
        elif connection and isinstance(connection.stream,
                                       iostream.SSLIOStream):
            self.protocol = "https"
        else:
            self.protocol = "http"

        # xheaders can override the defaults
        if connection and connection.xheaders:
            # Squid uses X-Forwarded-For, others use X-Real-Ip
            ip = self.headers.get("X-Forwarded-For", self.remote_ip)
            ip = ip.split(',')[-1].strip()
            ip = self.headers.get(
                "X-Real-Ip", ip)
            if netutil.is_valid_ip(ip):
                self.remote_ip = ip
            # AWS uses X-Forwarded-Proto
            proto = self.headers.get(
                "X-Scheme", self.headers.get("X-Forwarded-Proto", self.protocol))
            if proto in ("http", "https"):
                self.protocol = proto
            # Zipkin users 
            if options.server_trace:
                parent_span_id = self.headers.get("X-B3-ParentSpanId", None)
                trace_id = self.headers.get("X-B3-TraceId", None)
                span_id = self.headers.get("X-B3-SpanId", None)
                name =  method
                endpoint = Endpoint(ipv4=socket.gethostbyname(socket.gethostname()), port=port, service_name=service_name)
                
                self.trace = Trace(name=name, trace_id=trace_id, span_id=span_id, parent_span_id=parent_span_id)
                self.trace.set_endpoint(endpoint)

        self.host = host or self.headers.get("Host") or "127.0.0.1"
        self.files = files or {}
        self.connection = connection
        self._start_time = time.time()
        self._finish_time = None

        self.path, sep, self.query = uri.partition('?')
        self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)
        self.query_arguments = copy.deepcopy(self.arguments)
        self.body_arguments = {}
        
        if options.server_trace:
            self.trace.record(Annotation.string('Url', uri))
            self.trace.record(Annotation.string('Header', self.headers))
            self.trace.record(Annotation.server_recv())

    def supports_http_1_1(self):
        """Returns True if this request supports HTTP/1.1 semantics"""
        return self.version == "HTTP/1.1"

    @property
    def cookies(self):
        """A dictionary of Cookie.Morsel objects."""
        if not hasattr(self, "_cookies"):
            self._cookies = Cookie.SimpleCookie()
            if "Cookie" in self.headers:
                try:
                    self._cookies.load(
                        native_str(self.headers["Cookie"]))
                except Exception:
                    self._cookies = {}
        return self._cookies

    def write(self, chunk, callback=None):
        """Writes the given chunk to the response stream."""
        assert isinstance(chunk, bytes_type)
        self.connection.write(chunk, callback=callback)

    def finish(self):
        """Finishes this HTTP request on the open connection."""       
        if options.server_trace:
            self.headers.set("X-B3-TraceId", [hex_str(self.trace.trace_id)])
            self.headers.set("X-B3-SpanId", [hex_str(self.trace.span_id)])
            
            if self.trace.parent_span_id is not None:
                self.headers.set("X-B3-ParentSpanId", [hex_str(self.trace.parent_span_id)])

            self.trace.record(Annotation.server_send())
            
        self.connection.finish()
        self._finish_time = time.time()
        
    def full_url(self):
        """Reconstructs the full URL for this request."""
        return self.protocol + "://" + self.host + self.uri

    def request_time(self):
        """Returns the amount of time it took for this request to execute."""
        if self._finish_time is None:
            return time.time() - self._start_time
        else:
            return self._finish_time - self._start_time

    def get_ssl_certificate(self, binary_form=False):
        """Returns the client's SSL certificate, if any.

        To use client certificates, the HTTPServer must have been constructed
        with cert_reqs set in ssl_options, e.g.::

            server = HTTPServer(app,
                ssl_options=dict(
                    certfile="foo.crt",
                    keyfile="foo.key",
                    cert_reqs=ssl.CERT_REQUIRED,
                    ca_certs="cacert.crt"))

        By default, the return value is a dictionary (or None, if no
        client certificate is present).  If ``binary_form`` is true, a
        DER-encoded form of the certificate is returned instead.  See
        SSLSocket.getpeercert() in the standard library for more
        details.
        http://docs.python.org/library/ssl.html#sslsocket-objects
        """
        try:
            return self.connection.stream.socket.getpeercert(
                binary_form=binary_form)
        except ssl.SSLError:
            return None

    def __repr__(self):
        attrs = ("protocol", "host", "method", "uri", "version", "remote_ip")
        args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
        return "%s(%s, headers=%s)" % (
            self.__class__.__name__, args, dict(self.headers))
Пример #36
0
class RawRESTkinHTTPTracerTests(TestCase):
    def assertBodyEquals(self, bodyProducer, expectedOutput):
        output = StringIO()

        def _check_body(_):
            self.assertEqual(json.loads(output.getvalue()), expectedOutput)

        d = bodyProducer.startProducing(output)
        d.addCallback(_check_body)

        return d

    def setUp(self):
        self.agent = mock.Mock()

        self.tracer = RawRESTkinHTTPTracer(self.agent, 'http://trace.it')
        self.trace = Trace('test', 1, 2, tracers=[self.tracer])

    def test_verifyObject(self):
        verifyObject(ITracer, self.tracer)

    def test_posts_immediately(self):
        self.trace.record(Annotation.client_send(1))

        self.assertEqual(self.agent.request.call_count, 1)

        args = self.agent.request.mock_calls[0][1]
        self.assertEqual(('POST', 'http://trace.it', Headers({})), args[:3])

        bodyProducer = args[3]

        return self.assertBodyEquals(
            bodyProducer,
            [{
                'trace_id': '0000000000000001',
                'span_id': '0000000000000002',
                'name': 'test',
                'annotations': [{
                    'type': 'timestamp',
                    'value': 1,
                    'key': 'cs'
                }]
            }])

    def test_handles_batched_traces(self):
        t1 = self.trace
        t2 = Trace('test2', 3, 4)

        cs1 = Annotation.client_send(1)
        cs2 = Annotation.client_send(2)
        cr1 = Annotation.client_recv(3)
        cr2 = Annotation.client_recv(4)

        self.tracer.record([(t1, [cs1, cr1]), (t2, [cs2, cr2])])

        self.assertEqual(self.agent.request.call_count, 1)

        args = self.agent.request.mock_calls[0][1]
        self.assertEqual(('POST', 'http://trace.it', Headers({})), args[:3])

        bodyProducer = args[3]

        return self.assertBodyEquals(bodyProducer, [{
            'trace_id':
            '0000000000000001',
            'span_id':
            '0000000000000002',
            'name':
            'test',
            'annotations': [{
                'type': 'timestamp',
                'value': 1,
                'key': 'cs'
            }, {
                'type': 'timestamp',
                'value': 3,
                'key': 'cr'
            }]
        }, {
            'trace_id':
            '0000000000000003',
            'span_id':
            '0000000000000004',
            'name':
            'test2',
            'annotations': [{
                'type': 'timestamp',
                'value': 2,
                'key': 'cs'
            }, {
                'type': 'timestamp',
                'value': 4,
                'key': 'cr'
            }]
        }])
Пример #37
0
    def setUp(self):
        self.agent = mock.Mock()

        self.tracer = RawRESTkinHTTPTracer(self.agent, 'http://trace.it')
        self.trace = Trace('test', 1, 2, tracers=[self.tracer])
Пример #38
0
    def __init__(self,
                 url,
                 method="GET",
                 headers=None,
                 body=None,
                 auth_username=None,
                 auth_password=None,
                 auth_mode=None,
                 connect_timeout=None,
                 request_timeout=None,
                 if_modified_since=None,
                 follow_redirects=None,
                 max_redirects=None,
                 user_agent=None,
                 use_gzip=None,
                 network_interface=None,
                 streaming_callback=None,
                 header_callback=None,
                 prepare_curl_callback=None,
                 proxy_host=None,
                 proxy_port=None,
                 proxy_username=None,
                 proxy_password=None,
                 allow_nonstandard_methods=None,
                 validate_cert=None,
                 ca_certs=None,
                 allow_ipv6=None,
                 client_key=None,
                 client_cert=None,
                 parent_trace=None,
                 endpoint=None):
        r"""All parameters except ``url`` are optional.

        :arg string url: URL to fetch
        :arg string method: HTTP method, e.g. "GET" or "POST"
        :arg headers: Additional HTTP headers to pass on the request
        :arg body: HTTP body to pass on the request
        :type headers: `~tornado.httputil.HTTPHeaders` or `dict`
        :arg string auth_username: Username for HTTP authentication
        :arg string auth_password: Password for HTTP authentication
        :arg string auth_mode: Authentication mode; default is "basic".
           Allowed values are implementation-defined; ``curl_httpclient``
           supports "basic" and "digest"; ``simple_httpclient`` only supports
           "basic"
        :arg float connect_timeout: Timeout for initial connection in seconds
        :arg float request_timeout: Timeout for entire request in seconds
        :arg if_modified_since: Timestamp for ``If-Modified-Since`` header
        :type if_modified_since: `datetime` or `float`
        :arg bool follow_redirects: Should redirects be followed automatically
           or return the 3xx response?
        :arg int max_redirects: Limit for ``follow_redirects``
        :arg string user_agent: String to send as ``User-Agent`` header
        :arg bool use_gzip: Request gzip encoding from the server
        :arg string network_interface: Network interface to use for request.
           ``curl_httpclient`` only; see note below.
        :arg callable streaming_callback: If set, ``streaming_callback`` will
           be run with each chunk of data as it is received, and
           ``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
           the final response.
        :arg callable header_callback: If set, ``header_callback`` will
           be run with each header line as it is received (including the
           first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
           containing only ``\r\n``.  All lines include the trailing newline
           characters).  ``HTTPResponse.headers`` will be empty in the final
           response.  This is most useful in conjunction with
           ``streaming_callback``, because it's the only way to get access to
           header data while the request is in progress.
        :arg callable prepare_curl_callback: If set, will be called with
           a ``pycurl.Curl`` object to allow the application to make additional
           ``setopt`` calls.
        :arg string proxy_host: HTTP proxy hostname.  To use proxies,
           ``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and
           ``proxy_pass`` are optional.  Proxies are currently only supported
           with ``curl_httpclient``.
        :arg int proxy_port: HTTP proxy port
        :arg string proxy_username: HTTP proxy username
        :arg string proxy_password: HTTP proxy password
        :arg bool allow_nonstandard_methods: Allow unknown values for ``method``
           argument?
        :arg bool validate_cert: For HTTPS requests, validate the server's
           certificate?
        :arg string ca_certs: filename of CA certificates in PEM format,
           or None to use defaults.  See note below when used with
           ``curl_httpclient``.
        :arg bool allow_ipv6: Use IPv6 when available?  Default is false in
           ``simple_httpclient`` and true in ``curl_httpclient``
        :arg string client_key: Filename for client SSL key, if any.  See
           note below when used with ``curl_httpclient``.
        :arg string client_cert: Filename for client SSL certificate, if any.
           See note below when used with ``curl_httpclient``.
        :arg string parent_trace: parent trace id.
        :arg string endpoint: request endpoint.

        .. note::

            When using ``curl_httpclient`` certain options may be
            inherited by subsequent fetches because ``pycurl`` does
            not allow them to be cleanly reset.  This applies to the
            ``ca_certs``, ``client_key``, ``client_cert``, and
            ``network_interface`` arguments.  If you use these
            options, you should pass them on every request (you don't
            have to always use the same values, but it's not possible
            to mix requests that specify these options with ones that
            use the defaults).

        .. versionadded:: 3.1
           The ``auth_mode`` argument.
        """
        # Note that some of these attributes go through property setters
        # defined below.
        self.headers = headers
        if if_modified_since:
            self.headers["If-Modified-Since"] = httputil.format_timestamp(
                if_modified_since)
        self.proxy_host = proxy_host
        self.proxy_port = proxy_port
        self.proxy_username = proxy_username
        self.proxy_password = proxy_password
        self.url = url
        self.method = method
        self.body = body
        self.auth_username = auth_username
        self.auth_password = auth_password
        self.auth_mode = auth_mode
        self.connect_timeout = connect_timeout
        self.request_timeout = request_timeout
        self.follow_redirects = follow_redirects
        self.max_redirects = max_redirects
        self.user_agent = user_agent
        self.use_gzip = use_gzip
        self.network_interface = network_interface
        self.streaming_callback = streaming_callback
        self.header_callback = header_callback
        self.prepare_curl_callback = prepare_curl_callback
        self.allow_nonstandard_methods = allow_nonstandard_methods
        self.validate_cert = validate_cert
        self.ca_certs = ca_certs
        self.allow_ipv6 = allow_ipv6
        self.client_key = client_key
        self.client_cert = client_cert
        self.start_time = time.time()
        self._parent_trace = parent_trace
        self._endpoint = endpoint
        self.trace = None

        if options.client_trace:
            if self._parent_trace is None:
                self.trace = Trace(method)
            else:
                self.trace = self._parent_trace.child(method)
            if self._endpoint is not None:
                self.trace.set_endpoint(self._endpoint)
            else:
                self._endpoint = Endpoint(ipv4=socket.gethostbyname(
                    socket.gethostname()),
                                          port=0,
                                          service_name=service_name)
                self.trace.set_endpoint(self._endpoint)

            self.headers.set('X-B3-TraceId', [hex_str(self.trace.trace_id)])
            self.headers.set('X-B3-SpanId', [hex_str(self.trace.span_id)])
            if trace.parent_span_id is not None:
                self.headers.set('X-B3-ParentSpanId',
                                 [hex_str(self.trace.parent_span_id)])

            self.trace.record(Annotation.string('HTTPClient REQ', self.url))
Пример #39
0
    def __init__(self, url, method="GET", headers=None, body=None,
                 auth_username=None, auth_password=None, auth_mode=None,
                 connect_timeout=None, request_timeout=None,
                 if_modified_since=None, follow_redirects=None,
                 max_redirects=None, user_agent=None, use_gzip=None,
                 network_interface=None, streaming_callback=None,
                 header_callback=None, prepare_curl_callback=None,
                 proxy_host=None, proxy_port=None, proxy_username=None,
                 proxy_password=None, allow_nonstandard_methods=None,
                 validate_cert=None, ca_certs=None,
                 allow_ipv6=None,
                 client_key=None, client_cert=None,
                 parent_trace=None, endpoint=None):
        r"""All parameters except ``url`` are optional.

        :arg string url: URL to fetch
        :arg string method: HTTP method, e.g. "GET" or "POST"
        :arg headers: Additional HTTP headers to pass on the request
        :arg body: HTTP body to pass on the request
        :type headers: `~tornado.httputil.HTTPHeaders` or `dict`
        :arg string auth_username: Username for HTTP authentication
        :arg string auth_password: Password for HTTP authentication
        :arg string auth_mode: Authentication mode; default is "basic".
           Allowed values are implementation-defined; ``curl_httpclient``
           supports "basic" and "digest"; ``simple_httpclient`` only supports
           "basic"
        :arg float connect_timeout: Timeout for initial connection in seconds
        :arg float request_timeout: Timeout for entire request in seconds
        :arg if_modified_since: Timestamp for ``If-Modified-Since`` header
        :type if_modified_since: `datetime` or `float`
        :arg bool follow_redirects: Should redirects be followed automatically
           or return the 3xx response?
        :arg int max_redirects: Limit for ``follow_redirects``
        :arg string user_agent: String to send as ``User-Agent`` header
        :arg bool use_gzip: Request gzip encoding from the server
        :arg string network_interface: Network interface to use for request.
           ``curl_httpclient`` only; see note below.
        :arg callable streaming_callback: If set, ``streaming_callback`` will
           be run with each chunk of data as it is received, and
           ``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
           the final response.
        :arg callable header_callback: If set, ``header_callback`` will
           be run with each header line as it is received (including the
           first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
           containing only ``\r\n``.  All lines include the trailing newline
           characters).  ``HTTPResponse.headers`` will be empty in the final
           response.  This is most useful in conjunction with
           ``streaming_callback``, because it's the only way to get access to
           header data while the request is in progress.
        :arg callable prepare_curl_callback: If set, will be called with
           a ``pycurl.Curl`` object to allow the application to make additional
           ``setopt`` calls.
        :arg string proxy_host: HTTP proxy hostname.  To use proxies,
           ``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and
           ``proxy_pass`` are optional.  Proxies are currently only supported
           with ``curl_httpclient``.
        :arg int proxy_port: HTTP proxy port
        :arg string proxy_username: HTTP proxy username
        :arg string proxy_password: HTTP proxy password
        :arg bool allow_nonstandard_methods: Allow unknown values for ``method``
           argument?
        :arg bool validate_cert: For HTTPS requests, validate the server's
           certificate?
        :arg string ca_certs: filename of CA certificates in PEM format,
           or None to use defaults.  See note below when used with
           ``curl_httpclient``.
        :arg bool allow_ipv6: Use IPv6 when available?  Default is false in
           ``simple_httpclient`` and true in ``curl_httpclient``
        :arg string client_key: Filename for client SSL key, if any.  See
           note below when used with ``curl_httpclient``.
        :arg string client_cert: Filename for client SSL certificate, if any.
           See note below when used with ``curl_httpclient``.
        :arg string parent_trace: parent trace id.
        :arg string endpoint: request endpoint.

        .. note::

            When using ``curl_httpclient`` certain options may be
            inherited by subsequent fetches because ``pycurl`` does
            not allow them to be cleanly reset.  This applies to the
            ``ca_certs``, ``client_key``, ``client_cert``, and
            ``network_interface`` arguments.  If you use these
            options, you should pass them on every request (you don't
            have to always use the same values, but it's not possible
            to mix requests that specify these options with ones that
            use the defaults).

        .. versionadded:: 3.1
           The ``auth_mode`` argument.
        """
        # Note that some of these attributes go through property setters
        # defined below.
        self.headers = headers
        if if_modified_since:
            self.headers["If-Modified-Since"] = httputil.format_timestamp(
                if_modified_since)
        self.proxy_host = proxy_host
        self.proxy_port = proxy_port
        self.proxy_username = proxy_username
        self.proxy_password = proxy_password
        self.url = url
        self.method = method
        self.body = body
        self.auth_username = auth_username
        self.auth_password = auth_password
        self.auth_mode = auth_mode
        self.connect_timeout = connect_timeout
        self.request_timeout = request_timeout
        self.follow_redirects = follow_redirects
        self.max_redirects = max_redirects
        self.user_agent = user_agent
        self.use_gzip = use_gzip
        self.network_interface = network_interface
        self.streaming_callback = streaming_callback
        self.header_callback = header_callback
        self.prepare_curl_callback = prepare_curl_callback
        self.allow_nonstandard_methods = allow_nonstandard_methods
        self.validate_cert = validate_cert
        self.ca_certs = ca_certs
        self.allow_ipv6 = allow_ipv6
        self.client_key = client_key
        self.client_cert = client_cert
        self.start_time = time.time()
        self._parent_trace = parent_trace
        self._endpoint = endpoint
        self.trace = None
        
        if options.client_trace:
            if self._parent_trace is None:
                self.trace = Trace(method)
            else:
                self.trace = self._parent_trace.child(method)
            if self._endpoint is not None:
                self.trace.set_endpoint(self._endpoint)
            else:
                self._endpoint = Endpoint(ipv4=socket.gethostbyname(socket.gethostname()), port=0, service_name=service_name)
                self.trace.set_endpoint(self._endpoint)
            
            self.headers.set('X-B3-TraceId', [hex_str(self.trace.trace_id)])
            self.headers.set('X-B3-SpanId', [hex_str(self.trace.span_id)])
            if trace.parent_span_id is not None:
                self.headers.set('X-B3-ParentSpanId', [hex_str(self.trace.parent_span_id)])
            
            self.trace.record(Annotation.string('HTTPClient REQ', self.url))
Пример #40
0
    def __init__(self,
                 method,
                 uri,
                 version="HTTP/1.0",
                 headers=None,
                 body=None,
                 remote_ip=None,
                 protocol=None,
                 host=None,
                 files=None,
                 connection=None):
        self.method = method
        self.uri = uri
        self.version = version
        self.headers = headers or httputil.HTTPHeaders()
        self.body = body or ""
        self.trace = None

        # set remote IP and protocol
        self.remote_ip = remote_ip
        if protocol:
            self.protocol = protocol
        elif connection and isinstance(connection.stream,
                                       iostream.SSLIOStream):
            self.protocol = "https"
        else:
            self.protocol = "http"

        # xheaders can override the defaults
        if connection and connection.xheaders:
            # Squid uses X-Forwarded-For, others use X-Real-Ip
            ip = self.headers.get("X-Forwarded-For", self.remote_ip)
            ip = ip.split(',')[-1].strip()
            ip = self.headers.get("X-Real-Ip", ip)
            if netutil.is_valid_ip(ip):
                self.remote_ip = ip
            # AWS uses X-Forwarded-Proto
            proto = self.headers.get(
                "X-Scheme", self.headers.get("X-Forwarded-Proto",
                                             self.protocol))
            if proto in ("http", "https"):
                self.protocol = proto
            # Zipkin users
            if options.server_trace:
                parent_span_id = self.headers.get("X-B3-ParentSpanId", None)
                trace_id = self.headers.get("X-B3-TraceId", None)
                span_id = self.headers.get("X-B3-SpanId", None)
                name = method
                endpoint = Endpoint(ipv4=socket.gethostbyname(
                    socket.gethostname()),
                                    port=port,
                                    service_name=service_name)

                self.trace = Trace(name=name,
                                   trace_id=trace_id,
                                   span_id=span_id,
                                   parent_span_id=parent_span_id)
                self.trace.set_endpoint(endpoint)

        self.host = host or self.headers.get("Host") or "127.0.0.1"
        self.files = files or {}
        self.connection = connection
        self._start_time = time.time()
        self._finish_time = None

        self.path, sep, self.query = uri.partition('?')
        self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)
        self.query_arguments = copy.deepcopy(self.arguments)
        self.body_arguments = {}

        if options.server_trace:
            self.trace.record(Annotation.string('Url', uri))
            self.trace.record(Annotation.string('Header', self.headers))
            self.trace.record(Annotation.server_recv())
Пример #41
0
 def test_no_unfinished_traces(self):
     unfinished_trace = (Trace('unfinished'), [Annotation.client_send(1)])
     self.tracer.record([unfinished_trace])
     self.assertEqual(self.record_function.call_count, 0)
Пример #42
0
    def test_repr(self):
        trace = Trace('test_trace', trace_id=1, span_id=1, parent_span_id=1)

        self.assertEqual(
            repr(trace),
            "Trace('test_trace', trace_id=1, span_id=1, parent_span_id=1)")
Пример #43
0
    def setUp(self):
        self.agent = mock.Mock()

        self.tracer = RawRESTkinHTTPTracer(self.agent, 'http://trace.it')
        self.trace = Trace('test', 1, 2, tracers=[self.tracer])
Пример #44
0
    def process(self, data):
        msg = json.loads(data)
        if msg["frontend_name"] == "haproxy_monitoring":
            return

        frontend = FE_OR_BE.search(msg["frontend_name"]).group(2)
        backend = FE_OR_BE.search(msg["backend_name"]).group(2)
        server = SERVER_SUB.sub("_", msg["server_name"])

        # Consider server received to be the accept_date plus the time to
        # receive request headers.
        accept = parse_date_micro(msg["accept_date"])
        request = int(msg["time_request"]) * 1000
        if request < 0:
            request = 0
        sr = accept + request

        ctrace = Trace(msg["http_verb"],
                       trace_id=int_or_none(msg.get("zipkin_trace_id")),
                       span_id=int_or_none(msg.get("zipkin_next_span_id")),
                       parent_span_id=int_or_none(msg.get("zipkin_span_id")),
                       tracers=self.tracers)

        # There's an assumption here and in the server endpoint that the 'host'
        # parsed by Logstash from the syslog message is the same host as the
        # service is running on.
        endpoint = Endpoint(msg["host"], 0, backend)
        ctrace.set_endpoint(endpoint)

        # For client sent we consider the time in queue plus backend connect
        # time (which is the time to get the SYN/ACK from the backend).
        queue = int(msg["time_queue"]) * 1000
        if queue < 0:
            queue = 0
        connect = int(msg["time_backend_connect"]) * 1000
        if connect < 0:
            connect = 0
        cs = sr + queue + connect
        ctrace.record(Annotation.client_send(cs))
        ctrace.record(Annotation.string('haproxy.backend.server_name', server))

        # Record response time in ms into an annotation since it doesn't fit
        # anywhere in zipkin.
        response = int(msg["time_backend_response"]) * 1000
        if response < 0:
            response = 0
        ctrace.record(Annotation.string("haproxy.backend.time",
                                        str(response)))
        ctrace.record(Annotation.string("haproxy.backend.queue",
                                        msg["backend_queue"]))

        # Apache sends the duration in microseconds already. For haproxy we
        # have to convert from ms.
        duration = int(msg["time_duration"])
        if msg["program"] == "haproxy":
            duration = duration * 1000

        if duration < 0:
            duration = 0

        # Assume client response time to be the same as the request duration
        # minus a microsecond, just to keep ordering.
        ss = sr + duration
        cr = ss - 1
        ctrace.record(Annotation.client_recv(cr))

        # The top-level span is generally Apache. We record the parent span id
        # as '-' there if the parent span id is missing so convert to None.
        parent_span_id = msg.get("zipkin_parent_span_id")
        if parent_span_id is not None and parent_span_id == "-":
            parent_span_id = None

        strace = Trace(msg["http_verb"],
                       trace_id=int_or_none(msg.get("zipkin_trace_id")),
                       span_id=int_or_none(msg.get("zipkin_span_id")),
                       parent_span_id=int_or_none(parent_span_id),
                       tracers=self.tracers)
        endpoint = Endpoint(msg["host"], 0, frontend)
        strace.set_endpoint(endpoint)
        strace.record(Annotation.server_recv(sr))
        strace.record(Annotation.string('http.uri', msg["http_request"]))
        strace.record(Annotation.string(
            'http.responsecode', msg["http_status_code"]))
        strace.record(Annotation.server_send(ss))
Пример #45
0
 def test_equality(self):
     self.assertEqual(
         Trace('test_trace', trace_id=1, span_id=1, parent_span_id=1),
         Trace('test_trace2', trace_id=1, span_id=1, parent_span_id=1))
Пример #46
0
 def test_verifyObject(self):
     verifyObject(ITrace, Trace('test'))
Пример #47
0
class HTTPRequest(object):
    """A single HTTP request.

    All attributes are type `str` unless otherwise noted.

    .. attribute:: method

       HTTP request method, e.g. "GET" or "POST"

    .. attribute:: uri

       The requested uri.

    .. attribute:: path

       The path portion of `uri`

    .. attribute:: query

       The query portion of `uri`

    .. attribute:: version

       HTTP version specified in request, e.g. "HTTP/1.1"

    .. attribute:: headers

       `.HTTPHeaders` dictionary-like object for request headers.  Acts like
       a case-insensitive dictionary with additional methods for repeated
       headers.

    .. attribute:: body

       Request body, if present, as a byte string.

    .. attribute:: remote_ip

       Client's IP address as a string.  If ``HTTPServer.xheaders`` is set,
       will pass along the real IP address provided by a load balancer
       in the ``X-Real-Ip`` or ``X-Forwarded-For`` header.

    .. versionchanged:: 3.1
       The list format of ``X-Forwarded-For`` is now supported.

    .. attribute:: protocol

       The protocol used, either "http" or "https".  If ``HTTPServer.xheaders``
       is set, will pass along the protocol used by a load balancer if
       reported via an ``X-Scheme`` header.

    .. attribute:: host

       The requested hostname, usually taken from the ``Host`` header.

    .. attribute:: arguments

       GET/POST arguments are available in the arguments property, which
       maps arguments names to lists of values (to support multiple values
       for individual names). Names are of type `str`, while arguments
       are byte strings.  Note that this is different from
       `.RequestHandler.get_argument`, which returns argument values as
       unicode strings.

    .. attribute:: query_arguments

       Same format as ``arguments``, but contains only arguments extracted
       from the query string.

       .. versionadded:: 3.2

    .. attribute:: body_arguments

       Same format as ``arguments``, but contains only arguments extracted
       from the request body.

       .. versionadded:: 3.2

    .. attribute:: files

       File uploads are available in the files property, which maps file
       names to lists of `.HTTPFile`.

    .. attribute:: connection

       An HTTP request is attached to a single HTTP connection, which can
       be accessed through the "connection" attribute. Since connections
       are typically kept open in HTTP/1.1, multiple requests can be handled
       sequentially on a single connection.
    """
    def __init__(self,
                 method,
                 uri,
                 version="HTTP/1.0",
                 headers=None,
                 body=None,
                 remote_ip=None,
                 protocol=None,
                 host=None,
                 files=None,
                 connection=None):
        self.method = method
        self.uri = uri
        self.version = version
        self.headers = headers or httputil.HTTPHeaders()
        self.body = body or ""
        self.trace = None

        # set remote IP and protocol
        self.remote_ip = remote_ip
        if protocol:
            self.protocol = protocol
        elif connection and isinstance(connection.stream,
                                       iostream.SSLIOStream):
            self.protocol = "https"
        else:
            self.protocol = "http"

        # xheaders can override the defaults
        if connection and connection.xheaders:
            # Squid uses X-Forwarded-For, others use X-Real-Ip
            ip = self.headers.get("X-Forwarded-For", self.remote_ip)
            ip = ip.split(',')[-1].strip()
            ip = self.headers.get("X-Real-Ip", ip)
            if netutil.is_valid_ip(ip):
                self.remote_ip = ip
            # AWS uses X-Forwarded-Proto
            proto = self.headers.get(
                "X-Scheme", self.headers.get("X-Forwarded-Proto",
                                             self.protocol))
            if proto in ("http", "https"):
                self.protocol = proto
            # Zipkin users
            if options.server_trace:
                parent_span_id = self.headers.get("X-B3-ParentSpanId", None)
                trace_id = self.headers.get("X-B3-TraceId", None)
                span_id = self.headers.get("X-B3-SpanId", None)
                name = method
                endpoint = Endpoint(ipv4=socket.gethostbyname(
                    socket.gethostname()),
                                    port=port,
                                    service_name=service_name)

                self.trace = Trace(name=name,
                                   trace_id=trace_id,
                                   span_id=span_id,
                                   parent_span_id=parent_span_id)
                self.trace.set_endpoint(endpoint)

        self.host = host or self.headers.get("Host") or "127.0.0.1"
        self.files = files or {}
        self.connection = connection
        self._start_time = time.time()
        self._finish_time = None

        self.path, sep, self.query = uri.partition('?')
        self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)
        self.query_arguments = copy.deepcopy(self.arguments)
        self.body_arguments = {}

        if options.server_trace:
            self.trace.record(Annotation.string('Url', uri))
            self.trace.record(Annotation.string('Header', self.headers))
            self.trace.record(Annotation.server_recv())

    def supports_http_1_1(self):
        """Returns True if this request supports HTTP/1.1 semantics"""
        return self.version == "HTTP/1.1"

    @property
    def cookies(self):
        """A dictionary of Cookie.Morsel objects."""
        if not hasattr(self, "_cookies"):
            self._cookies = Cookie.SimpleCookie()
            if "Cookie" in self.headers:
                try:
                    self._cookies.load(native_str(self.headers["Cookie"]))
                except Exception:
                    self._cookies = {}
        return self._cookies

    def write(self, chunk, callback=None):
        """Writes the given chunk to the response stream."""
        assert isinstance(chunk, bytes_type)
        self.connection.write(chunk, callback=callback)

    def finish(self):
        """Finishes this HTTP request on the open connection."""
        if options.server_trace:
            self.headers.set("X-B3-TraceId", [hex_str(self.trace.trace_id)])
            self.headers.set("X-B3-SpanId", [hex_str(self.trace.span_id)])

            if self.trace.parent_span_id is not None:
                self.headers.set("X-B3-ParentSpanId",
                                 [hex_str(self.trace.parent_span_id)])

            self.trace.record(Annotation.server_send())

        self.connection.finish()
        self._finish_time = time.time()

    def full_url(self):
        """Reconstructs the full URL for this request."""
        return self.protocol + "://" + self.host + self.uri

    def request_time(self):
        """Returns the amount of time it took for this request to execute."""
        if self._finish_time is None:
            return time.time() - self._start_time
        else:
            return self._finish_time - self._start_time

    def get_ssl_certificate(self, binary_form=False):
        """Returns the client's SSL certificate, if any.

        To use client certificates, the HTTPServer must have been constructed
        with cert_reqs set in ssl_options, e.g.::

            server = HTTPServer(app,
                ssl_options=dict(
                    certfile="foo.crt",
                    keyfile="foo.key",
                    cert_reqs=ssl.CERT_REQUIRED,
                    ca_certs="cacert.crt"))

        By default, the return value is a dictionary (or None, if no
        client certificate is present).  If ``binary_form`` is true, a
        DER-encoded form of the certificate is returned instead.  See
        SSLSocket.getpeercert() in the standard library for more
        details.
        http://docs.python.org/library/ssl.html#sslsocket-objects
        """
        try:
            return self.connection.stream.socket.getpeercert(
                binary_form=binary_form)
        except ssl.SSLError:
            return None

    def __repr__(self):
        attrs = ("protocol", "host", "method", "uri", "version", "remote_ip")
        args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
        return "%s(%s, headers=%s)" % (self.__class__.__name__, args,
                                       dict(self.headers))