def getChildWithDefault(self, path, request):
headers = request.requestHeaders
# Construct and endpoint from the requested host and port and the
# passed service name.
host = request.getHost()
endpoint = Endpoint(host.host, host.port, self._service_name)
# Construct the trace using the headers X-B3-* headers that the
# TracingAgent will send.
trace = Trace(
request.method,
int_or_none(headers.getRawHeaders('X-B3-TraceId', [None])[0]),
int_or_none(headers.getRawHeaders('X-B3-SpanId', [None])[0]),
int_or_none(headers.getRawHeaders('X-B3-ParentSpanId', [None])[0]))
trace.set_endpoint(endpoint)
# twisted.web.server.Request is a subclass of Componentized this
# allows us to use ITrace(request) to get the trace and object (and
# create children of this trace) in the wrapped resource.
request.setComponent(ITrace, trace)
trace.record(Annotation.server_recv())
def _record_finish(_ignore):
trace.record(Annotation.server_send())
# notifyFinish returns a deferred that fires when the request is
# finished this will allow us to record server_send regardless of if
# the wrapped resource us using deferred rendering.
request.notifyFinish().addCallback(_record_finish)
return self._wrapped.getChildWithDefault(path, request)
def render_POST(self, request):
print "For CORS by john"
request.setHeader('Access-Control-Allow-Origin', '*')
request.setHeader('Access-Control-Allow-Methods', 'POST')
request.setHeader('Access-Control-Allow-Headers', 'authorization, content-type')
request.responseHeaders.setRawHeaders(
'content-type', ['application/json'])
body = request.content.read()
try:
spans = json.loads(body)
except ValueError:
log.err(None, 'Failed to decode request body')
msg = 'Could not decode request body (invalid JSON)'
return json.dumps({'error': msg})
succeeded, failed = 0, 0
for json_span in spans:
trace_id = None
span_id = None
try:
trace_id = decode_hex_number('trace_id', json_span['trace_id'])
span_id = decode_hex_number('span_id', json_span['span_id'])
parent_span_id = json_span.get('parent_span_id', None)
if parent_span_id is not None:
parent_span_id = decode_hex_number('parent_span_id',
parent_span_id)
t = Trace(json_span['name'], trace_id, span_id, parent_span_id)
for json_annotation in json_span['annotations']:
annotation = Annotation(
json_annotation['key'],
json_annotation['value'],
json_annotation['type'])
host = json_annotation.get('host', None)
if host:
annotation.endpoint = Endpoint(
host['ipv4'], host['port'], host['service_name'])
t.record(annotation)
succeeded = succeeded + 1
except Exception:
log.err(None,
'Failed to insert a trace: trace_id=%r,span_id=%r' %
(trace_id, span_id))
failed = failed + 1
continue
return json.dumps({'succeeded': succeeded, 'failed': failed})
def test_Trace_child(self):
t = Trace('test_trace', trace_id=1, span_id=1)
c = t.child('child_test_trace')
self.assertEqual(c.trace_id, 1)
self.assertEqual(c.parent_span_id, 1)
self.assertNotEqual(c.span_id, 1)
def test_record_invokes_tracer(self):
tracer = mock.Mock()
t = Trace('test_trace', trace_id=1, span_id=1, tracers=[tracer])
annotation = Annotation.client_send(timestamp=0)
t.record(annotation)
tracer.record.assert_called_with([(t, (annotation,))])
def test_Trace_child(self):
t = Trace('test_trace', trace_id=1, span_id=1)
c = t.child('child_test_trace')
self.assertEqual(c.trace_id, 1)
self.assertEqual(c.parent_span_id, 1)
self.assertNotEqual(c.span_id, 1)
def test_record_invokes_tracer(self):
tracer = mock.Mock()
t = Trace('test_trace', trace_id=1, span_id=1, tracers=[tracer])
annotation = Annotation.client_send(timestamp=0)
t.record(annotation)
tracer.record.assert_called_with([(t, (annotation, ))])
def render_POST(self, request):
print "For CORS by john"
request.setHeader('Access-Control-Allow-Origin', '*')
request.setHeader('Access-Control-Allow-Methods', 'POST')
request.responseHeaders.setRawHeaders('content-type',
['application/json'])
body = request.content.read()
try:
spans = json.loads(body)
except ValueError:
log.err(None, 'Failed to decode request body')
msg = 'Could not decode request body (invalid JSON)'
return json.dumps({'error': msg})
succeeded, failed = 0, 0
for json_span in spans:
trace_id = None
span_id = None
try:
trace_id = decode_hex_number('trace_id', json_span['trace_id'])
span_id = decode_hex_number('span_id', json_span['span_id'])
parent_span_id = json_span.get('parent_span_id', None)
if parent_span_id is not None:
parent_span_id = decode_hex_number('parent_span_id',
parent_span_id)
t = Trace(json_span['name'], trace_id, span_id, parent_span_id)
for json_annotation in json_span['annotations']:
annotation = Annotation(json_annotation['key'],
json_annotation['value'],
json_annotation['type'])
host = json_annotation.get('host', None)
if host:
annotation.endpoint = Endpoint(host['ipv4'],
host['port'],
host['service_name'])
t.record(annotation)
succeeded = succeeded + 1
except Exception:
log.err(
None, 'Failed to insert a trace: trace_id=%r,span_id=%r' %
(trace_id, span_id))
failed = failed + 1
continue
return json.dumps({'succeeded': succeeded, 'failed': failed})
def test_logs_to_scribe_immediately(self):
tracer = RawZipkinTracer(self.scribe)
t = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])
t.record(Annotation.client_send(1))
self.scribe.log.assert_called_once_with(
'zipkin',
['CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
'MAAAAAQoAAQAA\nAAAAAAABCwACAAAAAmNzAA8ACAwAAAAAAA=='])
def test_logs_to_scribe_immediately(self):
tracer = RawZipkinTracer(self.scribe)
t = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])
t.record(Annotation.client_send(1))
self.scribe.log.assert_called_once_with('zipkin', [
'CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
'MAAAAAQoAAQAA\nAAAAAAABCwACAAAAAmNzAA8ACAwAAAAAAA=='
])
def test_logs_to_scribe_with_non_default_category(self):
tracer = RawZipkinTracer(self.scribe, 'not-zipkin')
t = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])
t.record(Annotation.client_send(1), Annotation.client_recv(2))
self.scribe.log.assert_called_once_with('not-zipkin', [
'CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
'MAAAAAgoAAQAA\nAAAAAAABCwACAAAAAmNzAAoAAQAAAAAAAAACCwACAAAAAmNy'
'AA8ACAwAAAAAAA=='
])
def test_logs_to_scribe_with_non_default_category(self):
tracer = RawZipkinTracer(self.scribe, 'not-zipkin')
t = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])
t.record(Annotation.client_send(1), Annotation.client_recv(2))
self.scribe.log.assert_called_once_with(
'not-zipkin',
['CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
'MAAAAAgoAAQAA\nAAAAAAABCwACAAAAAmNzAAoAAQAAAAAAAAACCwACAAAAAmNy'
'AA8ACAwAAAAAAA=='])
def test_writes_trace(self):
t = Trace('test', 1, 2, tracers=[self.tracer])
t.record(Annotation.client_send(1))
self.assertEqual(
json.loads(self.destination.getvalue()),
[{'trace_id': '0000000000000001',
'span_id': '0000000000000002',
'name': 'test',
'annotations': [
{'type': 'timestamp', 'value': 1, 'key': 'cs'}]}])
def test_record_sets_annotation_endpoint(self):
tracer = mock.Mock()
web_endpoint = Endpoint('127.0.0.1', 8080, 'web')
t = Trace('test_trace', trace_id=1, span_id=1, tracers=[tracer])
t.set_endpoint(web_endpoint)
annotation = Annotation.client_send(timestamp=1)
t.record(annotation)
tracer.record.assert_called_with([(t, (annotation,))])
self.assertEqual(annotation.endpoint, web_endpoint)
def test_delegates_on_end_annotation(self):
tracer = EndAnnotationTracer(self.tracer)
t = Trace('test_delegation', tracers=[tracer])
cs = Annotation.client_send()
ce = Annotation.client_recv()
t.record(cs)
t.record(ce)
self.tracer.record.assert_called_once_with([(t, [cs, ce])])
def test_delegates_on_end_annotation(self):
tracer = EndAnnotationTracer(self.tracer)
t = Trace('test_delegation', tracers=[tracer])
cs = Annotation.client_send()
ce = Annotation.client_recv()
t.record(cs)
t.record(ce)
self.tracer.record.assert_called_once_with([(t, [cs, ce])])
def test_handles_batched_traces(self):
t1 = Trace('test', 1, 2)
t2 = Trace('test2', 3, 4)
cs1 = Annotation.client_send(1)
cs2 = Annotation.client_send(2)
cr1 = Annotation.client_recv(3)
cr2 = Annotation.client_recv(4)
self.tracer.record([(t1, [cs1, cr1]), (t2, [cs2, cr2])])
self.assertEqual(self.scribe.log.call_count, 1)
args = self.scribe.log.mock_calls[0][1]
self.assertEqual('restkin', args[0])
entries = args[1]
self.assertEqual(len(entries), 1)
self.assertEqual(json.loads(entries[0]), [{
'trace_id':
'0000000000000001',
'span_id':
'0000000000000002',
'name':
'test',
'annotations': [{
'type': 'timestamp',
'value': 1,
'key': 'cs'
}, {
'type': 'timestamp',
'value': 3,
'key': 'cr'
}]
}, {
'trace_id':
'0000000000000003',
'span_id':
'0000000000000004',
'name':
'test2',
'annotations': [{
'type': 'timestamp',
'value': 2,
'key': 'cs'
}, {
'type': 'timestamp',
'value': 4,
'key': 'cr'
}]
}])
def test_non_default_end(self):
tracer = EndAnnotationTracer(self.tracer, end_annotations=['timeout'])
t = Trace('test_non-default', tracers=[tracer])
cs = Annotation.client_send()
t.record(cs)
timeout = Annotation.timestamp('timeout')
t.record(timeout)
self.tracer.record.assert_called_once_with([(t, [cs, timeout])])
def test_inequality(self):
trace = Trace('test_trace', trace_id=1, span_id=1, parent_span_id=1)
self.assertNotEqual(trace, None)
self.assertNotEqual(
trace, Trace('test_trace', trace_id=2, span_id=1,
parent_span_id=1))
self.assertNotEqual(
trace, Trace('test_trace', trace_id=1, span_id=2,
parent_span_id=1))
self.assertNotEqual(
trace, Trace('test_trace', trace_id=1, span_id=1,
parent_span_id=2))
def test_writes_trace(self):
t = Trace('test', 1, 2, tracers=[self.tracer])
t.record(Annotation.client_send(1))
self.assertEqual(
json.loads(self.destination.getvalue()),
[{
'trace_id': '0000000000000001',
'span_id': '0000000000000002',
'name': 'test',
'annotations': [{
'type': 'timestamp',
'value': 1,
'key': 'cs'
}]
}])
def __init__(self, method, uri, version="HTTP/1.0", headers=None,
body=None, remote_ip=None, protocol=None, host=None,
files=None, connection=None):
self.method = method
self.uri = uri
self.version = version
self.headers = headers or httputil.HTTPHeaders()
self.body = body or ""
self.trace = None
# set remote IP and protocol
self.remote_ip = remote_ip
if protocol:
self.protocol = protocol
elif connection and isinstance(connection.stream,
iostream.SSLIOStream):
self.protocol = "https"
else:
self.protocol = "http"
# xheaders can override the defaults
if connection and connection.xheaders:
# Squid uses X-Forwarded-For, others use X-Real-Ip
ip = self.headers.get("X-Forwarded-For", self.remote_ip)
ip = ip.split(',')[-1].strip()
ip = self.headers.get(
"X-Real-Ip", ip)
if netutil.is_valid_ip(ip):
self.remote_ip = ip
# AWS uses X-Forwarded-Proto
proto = self.headers.get(
"X-Scheme", self.headers.get("X-Forwarded-Proto", self.protocol))
if proto in ("http", "https"):
self.protocol = proto
# Zipkin users
if options.server_trace:
parent_span_id = self.headers.get("X-B3-ParentSpanId", None)
trace_id = self.headers.get("X-B3-TraceId", None)
span_id = self.headers.get("X-B3-SpanId", None)
name = method
endpoint = Endpoint(ipv4=socket.gethostbyname(socket.gethostname()), port=port, service_name=service_name)
self.trace = Trace(name=name, trace_id=trace_id, span_id=span_id, parent_span_id=parent_span_id)
self.trace.set_endpoint(endpoint)
self.host = host or self.headers.get("Host") or "127.0.0.1"
self.files = files or {}
self.connection = connection
self._start_time = time.time()
self._finish_time = None
self.path, sep, self.query = uri.partition('?')
self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)
self.query_arguments = copy.deepcopy(self.arguments)
self.body_arguments = {}
if options.server_trace:
self.trace.record(Annotation.string('Url', uri))
self.trace.record(Annotation.string('Header', self.headers))
self.trace.record(Annotation.server_recv())
def test_handles_batched_traces(self):
tracer = EndAnnotationTracer(self.tracer)
t1 = Trace('test1', tracers=[tracer])
t2 = Trace('test2', tracers=[tracer])
cs1 = Annotation.client_send()
cs2 = Annotation.client_send()
cr1 = Annotation.client_recv()
cr2 = Annotation.client_recv()
tracer.record([(t1, [cs1, cr1]), (t2, [cs2, cr2])])
self.assertEqual(self.tracer.record.call_count, 2)
self.tracer.record.assert_any_call([(t1, [cs1, cr1])])
self.tracer.record.assert_any_call([(t2, [cs2, cr2])])
def test_traces_immediately(self):
t = Trace('test', 1, 2, tracers=[self.tracer])
t.record(Annotation.client_send(1))
self.assertEqual(self.scribe.log.call_count, 1)
args = self.scribe.log.mock_calls[0][1]
self.assertEqual('restkin', args[0])
entries = args[1]
self.assertEqual(len(entries), 1)
self.assertEqual(
json.loads(entries[0]),
[{'trace_id': '0000000000000001',
'span_id': '0000000000000002',
'name': 'test',
'annotations': [
{'type': 'timestamp', 'value': 1, 'key': 'cs'}]}])
def test_handles_batched_traces(self):
tracer = RawZipkinTracer(self.scribe)
t1 = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])
cs1 = Annotation.client_send(1)
t2 = Trace('test_raw_zipkin', 1, 2, tracers=[tracer])
cs2 = Annotation.client_send(1)
cr2 = Annotation.client_recv(2)
tracer.record([(t1, [cs1]), (t2, [cs2, cr2])])
self.scribe.log.assert_called_once_with('zipkin', [
'CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
'MAAAAAQoAAQAA\nAAAAAAABCwACAAAAAmNzAA8ACAwAAAAAAA==',
'CgABAAAAAAAAAAELAAMAAAAPdGVzdF9yYXdfemlwa2luCgAEAAAAAAAAAAIPAAY'
'MAAAAAgoAAQAA\nAAAAAAABCwACAAAAAmNzAAoAAQAAAAAAAAACCwACAAAAAmNy'
'AA8ACAwAAAAAAA=='
])
def test_new_Trace(self):
t = Trace('test_trace')
self.assertNotEqual(t.trace_id, None)
self.assertIsInstance(t.trace_id, (int, long))
self.failIf(t.trace_id >= MAX_ID)
self.assertNotEqual(t.span_id, None)
self.assertIsInstance(t.span_id, (int, long))
self.failIf(t.span_id >= MAX_ID)
self.assertEqual(t.parent_span_id, None)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Send a client request following HTTP redirects.
@see: L{Agent.request}.
"""
if self._parent_trace is None:
trace = Trace(method)
else:
trace = self._parent_trace.child(method)
if self._endpoint is not None:
trace.set_endpoint(self._endpoint)
if headers is None:
headers = Headers({})
# These headers are based on the headers used by finagle's tracing
# http Codec.
#
# https://github.com/twitter/finagle/blob/master/finagle-http/
#
# Currently not implemented are X-B3-Sampled and X-B3-Flags
# Tryfer's underlying Trace implementation has no notion of a Sampled
# trace and I haven't figured out what flags are for.
headers.setRawHeaders('X-B3-TraceId', [hex_str(trace.trace_id)])
headers.setRawHeaders('X-B3-SpanId', [hex_str(trace.span_id)])
if trace.parent_span_id is not None:
headers.setRawHeaders('X-B3-ParentSpanId',
[hex_str(trace.parent_span_id)])
# Similar to the headers above we use the annotation 'http.uri' for
# because that is the standard set forth in the finagle http Codec.
trace.record(Annotation.string('http.uri', uri))
trace.record(Annotation.client_send())
def _finished(resp):
# TODO: It may be advantageous here to return a wrapped response
# whose deliverBody can wrap it's protocol and record when the
# application has finished reading the contents.
trace.record(
Annotation.string('http.responsecode',
'{0} {1}'.format(resp.code, resp.phrase)))
trace.record(Annotation.client_recv())
return resp
d = self._agent.request(method, uri, headers, bodyProducer)
d.addBoth(_finished)
return d
def test_traces_buffered_until_max_idle_time(self):
completed_trace = (Trace('completed'), [
Annotation.client_send(1),
Annotation.client_recv(2)
])
self.tracer.record([completed_trace])
self.assertEqual(self.record_function.call_count, 0)
self.clock.advance(10)
self.assertEqual(self.record_function.call_count, 1)
def getChildWithDefault(self, path, request):
headers = request.requestHeaders
# Construct and endpoint from the requested host and port and the
# passed service name.
host = request.getHost()
endpoint = Endpoint(host.host, host.port, self._service_name)
# Construct the trace using the headers X-B3-* headers that the
# TracingAgent will send.
trace = Trace(
request.method,
int_or_none(headers.getRawHeaders('X-B3-TraceId', [None])[0]),
int_or_none(headers.getRawHeaders('X-B3-SpanId', [None])[0]),
int_or_none(headers.getRawHeaders('X-B3-ParentSpanId', [None])[0]))
trace.set_endpoint(endpoint)
# twisted.web.server.Request is a subclass of Componentized this
# allows us to use ITrace(request) to get the trace and object (and
# create children of this trace) in the wrapped resource.
request.setComponent(ITrace, trace)
trace.record(Annotation.server_recv())
def _record_finish(_ignore):
trace.record(Annotation.server_send())
# notifyFinish returns a deferred that fires when the request is
# finished this will allow us to record server_send regardless of if
# the wrapped resource us using deferred rendering.
request.notifyFinish().addCallback(_record_finish)
return self._wrapped.getChildWithDefault(path, request)
def test_handles_batched_traces(self):
t1 = self.trace
t2 = Trace('test2', 3, 4)
cs1 = Annotation.client_send(1)
cs2 = Annotation.client_send(2)
cr1 = Annotation.client_recv(3)
cr2 = Annotation.client_recv(4)
self.tracer.record([(t1, [cs1, cr1]), (t2, [cs2, cr2])])
self.assertEqual(self.agent.request.call_count, 1)
args = self.agent.request.mock_calls[0][1]
self.assertEqual(('POST', 'http://trace.it', Headers({})), args[:3])
bodyProducer = args[3]
return self.assertBodyEquals(bodyProducer, [{
'trace_id':
'0000000000000001',
'span_id':
'0000000000000002',
'name':
'test',
'annotations': [{
'type': 'timestamp',
'value': 1,
'key': 'cs'
}, {
'type': 'timestamp',
'value': 3,
'key': 'cr'
}]
}, {
'trace_id':
'0000000000000003',
'span_id':
'0000000000000004',
'name':
'test2',
'annotations': [{
'type': 'timestamp',
'value': 2,
'key': 'cs'
}, {
'type': 'timestamp',
'value': 4,
'key': 'cr'
}]
}])
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Send a client request following HTTP redirects.
@see: L{Agent.request}.
"""
if self._parent_trace is None:
trace = Trace(method)
else:
trace = self._parent_trace.child(method)
if self._endpoint is not None:
trace.set_endpoint(self._endpoint)
if headers is None:
headers = Headers({})
# These headers are based on the headers used by finagle's tracing
# http Codec.
#
# https://github.com/twitter/finagle/blob/master/finagle-http/
#
# Currently not implemented are X-B3-Sampled and X-B3-Flags
# Tryfer's underlying Trace implementation has no notion of a Sampled
# trace and I haven't figured out what flags are for.
headers.setRawHeaders('X-B3-TraceId', [hex_str(trace.trace_id)])
headers.setRawHeaders('X-B3-SpanId', [hex_str(trace.span_id)])
if trace.parent_span_id is not None:
headers.setRawHeaders('X-B3-ParentSpanId',
[hex_str(trace.parent_span_id)])
# Similar to the headers above we use the annotation 'http.uri' for
# because that is the standard set forth in the finagle http Codec.
trace.record(Annotation.string('http.uri', uri))
trace.record(Annotation.client_send())
def _finished(resp):
# TODO: It may be advantageous here to return a wrapped response
# whose deliverBody can wrap it's protocol and record when the
# application has finished reading the contents.
trace.record(Annotation.string(
'http.responsecode',
'{0} {1}'.format(resp.code, resp.phrase)))
trace.record(Annotation.client_recv())
return resp
d = self._agent.request(method, uri, headers, bodyProducer)
d.addBoth(_finished)
return d
def test_traces_immediately(self):
t = Trace('test', 1, 2, tracers=[self.tracer])
t.record(Annotation.client_send(1))
self.assertEqual(self.scribe.log.call_count, 1)
args = self.scribe.log.mock_calls[0][1]
self.assertEqual('restkin', args[0])
entries = args[1]
self.assertEqual(len(entries), 1)
self.assertEqual(
json.loads(entries[0]),
[{
'trace_id': '0000000000000001',
'span_id': '0000000000000002',
'name': 'test',
'annotations': [{
'type': 'timestamp',
'value': 1,
'key': 'cs'
}]
}])
def test_traces_bufferred_until_max_traces(self):
completed_traces = [
(Trace('completed'),
[Annotation.client_send(1),
Annotation.client_recv(2)]) for x in xrange(50)
]
self.tracer.record(completed_traces[:10])
self.clock.advance(1)
self.assertEqual(self.record_function.call_count, 0)
self.tracer.record(completed_traces[10:])
self.clock.advance(1)
self.assertEqual(self.record_function.call_count, 1)
def test_record_sets_annotation_endpoint(self):
tracer = mock.Mock()
web_endpoint = Endpoint('127.0.0.1', 8080, 'web')
t = Trace('test_trace', trace_id=1, span_id=1, tracers=[tracer])
t.set_endpoint(web_endpoint)
annotation = Annotation.client_send(timestamp=1)
t.record(annotation)
tracer.record.assert_called_with([(t, (annotation, ))])
self.assertEqual(annotation.endpoint, web_endpoint)
def test_non_default_end(self):
tracer = EndAnnotationTracer(self.tracer, end_annotations=['timeout'])
t = Trace('test_non-default', tracers=[tracer])
cs = Annotation.client_send()
t.record(cs)
timeout = Annotation.timestamp('timeout')
t.record(timeout)
self.tracer.record.assert_called_once_with([(t, [cs, timeout])])
class RawRESTkinHTTPTracerTests(TestCase):
def assertBodyEquals(self, bodyProducer, expectedOutput):
output = StringIO()
consumer = FileConsumer(output)
def _check_body(_):
self.assertEqual(
json.loads(output.getvalue()),
expectedOutput
)
d = bodyProducer.startProducing(consumer)
d.addCallback(_check_body)
return d
def setUp(self):
self.agent = mock.Mock()
self.tracer = RawRESTkinHTTPTracer(self.agent, 'http://trace.it')
self.trace = Trace('test', 1, 2, tracers=[self.tracer])
def test_verifyObject(self):
verifyObject(ITracer, self.tracer)
def test_posts_immediately(self):
self.trace.record(Annotation.client_send(1))
self.assertEqual(self.agent.request.call_count, 1)
args = self.agent.request.mock_calls[0][1]
self.assertEqual(('POST', 'http://trace.it', Headers({})), args[:3])
bodyProducer = args[3]
return self.assertBodyEquals(
bodyProducer,
[{'trace_id': '0000000000000001',
'span_id': '0000000000000002',
'name': 'test',
'annotations': [
{'type': 'timestamp', 'value': 1, 'key': 'cs'}
]}])
def test_handles_batched_traces(self):
t1 = self.trace
t2 = Trace('test2', 3, 4)
cs1 = Annotation.client_send(1)
cs2 = Annotation.client_send(2)
cr1 = Annotation.client_recv(3)
cr2 = Annotation.client_recv(4)
self.tracer.record([(t1, [cs1, cr1]), (t2, [cs2, cr2])])
self.assertEqual(self.agent.request.call_count, 1)
args = self.agent.request.mock_calls[0][1]
self.assertEqual(('POST', 'http://trace.it', Headers({})), args[:3])
bodyProducer = args[3]
return self.assertBodyEquals(
bodyProducer,
[{'trace_id': '0000000000000001',
'span_id': '0000000000000002',
'name': 'test',
'annotations': [
{'type': 'timestamp', 'value': 1, 'key': 'cs'},
{'type': 'timestamp', 'value': 3, 'key': 'cr'}
]},
{'trace_id': '0000000000000003',
'span_id': '0000000000000004',
'name': 'test2',
'annotations': [
{'type': 'timestamp', 'value': 2, 'key': 'cs'},
{'type': 'timestamp', 'value': 4, 'key': 'cr'}
]}])
class HTTPRequest(object):
"""A single HTTP request.
All attributes are type `str` unless otherwise noted.
.. attribute:: method
HTTP request method, e.g. "GET" or "POST"
.. attribute:: uri
The requested uri.
.. attribute:: path
The path portion of `uri`
.. attribute:: query
The query portion of `uri`
.. attribute:: version
HTTP version specified in request, e.g. "HTTP/1.1"
.. attribute:: headers
`.HTTPHeaders` dictionary-like object for request headers. Acts like
a case-insensitive dictionary with additional methods for repeated
headers.
.. attribute:: body
Request body, if present, as a byte string.
.. attribute:: remote_ip
Client's IP address as a string. If ``HTTPServer.xheaders`` is set,
will pass along the real IP address provided by a load balancer
in the ``X-Real-Ip`` or ``X-Forwarded-For`` header.
.. versionchanged:: 3.1
The list format of ``X-Forwarded-For`` is now supported.
.. attribute:: protocol
The protocol used, either "http" or "https". If ``HTTPServer.xheaders``
is set, will pass along the protocol used by a load balancer if
reported via an ``X-Scheme`` header.
.. attribute:: host
The requested hostname, usually taken from the ``Host`` header.
.. attribute:: arguments
GET/POST arguments are available in the arguments property, which
maps arguments names to lists of values (to support multiple values
for individual names). Names are of type `str`, while arguments
are byte strings. Note that this is different from
`.RequestHandler.get_argument`, which returns argument values as
unicode strings.
.. attribute:: query_arguments
Same format as ``arguments``, but contains only arguments extracted
from the query string.
.. versionadded:: 3.2
.. attribute:: body_arguments
Same format as ``arguments``, but contains only arguments extracted
from the request body.
.. versionadded:: 3.2
.. attribute:: files
File uploads are available in the files property, which maps file
names to lists of `.HTTPFile`.
.. attribute:: connection
An HTTP request is attached to a single HTTP connection, which can
be accessed through the "connection" attribute. Since connections
are typically kept open in HTTP/1.1, multiple requests can be handled
sequentially on a single connection.
"""
def __init__(self, method, uri, version="HTTP/1.0", headers=None,
body=None, remote_ip=None, protocol=None, host=None,
files=None, connection=None):
self.method = method
self.uri = uri
self.version = version
self.headers = headers or httputil.HTTPHeaders()
self.body = body or ""
self.trace = None
# set remote IP and protocol
self.remote_ip = remote_ip
if protocol:
self.protocol = protocol
elif connection and isinstance(connection.stream,
iostream.SSLIOStream):
self.protocol = "https"
else:
self.protocol = "http"
# xheaders can override the defaults
if connection and connection.xheaders:
# Squid uses X-Forwarded-For, others use X-Real-Ip
ip = self.headers.get("X-Forwarded-For", self.remote_ip)
ip = ip.split(',')[-1].strip()
ip = self.headers.get(
"X-Real-Ip", ip)
if netutil.is_valid_ip(ip):
self.remote_ip = ip
# AWS uses X-Forwarded-Proto
proto = self.headers.get(
"X-Scheme", self.headers.get("X-Forwarded-Proto", self.protocol))
if proto in ("http", "https"):
self.protocol = proto
# Zipkin users
if options.server_trace:
parent_span_id = self.headers.get("X-B3-ParentSpanId", None)
trace_id = self.headers.get("X-B3-TraceId", None)
span_id = self.headers.get("X-B3-SpanId", None)
name = method
endpoint = Endpoint(ipv4=socket.gethostbyname(socket.gethostname()), port=port, service_name=service_name)
self.trace = Trace(name=name, trace_id=trace_id, span_id=span_id, parent_span_id=parent_span_id)
self.trace.set_endpoint(endpoint)
self.host = host or self.headers.get("Host") or "127.0.0.1"
self.files = files or {}
self.connection = connection
self._start_time = time.time()
self._finish_time = None
self.path, sep, self.query = uri.partition('?')
self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)
self.query_arguments = copy.deepcopy(self.arguments)
self.body_arguments = {}
if options.server_trace:
self.trace.record(Annotation.string('Url', uri))
self.trace.record(Annotation.string('Header', self.headers))
self.trace.record(Annotation.server_recv())
def supports_http_1_1(self):
"""Returns True if this request supports HTTP/1.1 semantics"""
return self.version == "HTTP/1.1"
@property
def cookies(self):
"""A dictionary of Cookie.Morsel objects."""
if not hasattr(self, "_cookies"):
self._cookies = Cookie.SimpleCookie()
if "Cookie" in self.headers:
try:
self._cookies.load(
native_str(self.headers["Cookie"]))
except Exception:
self._cookies = {}
return self._cookies
def write(self, chunk, callback=None):
"""Writes the given chunk to the response stream."""
assert isinstance(chunk, bytes_type)
self.connection.write(chunk, callback=callback)
def finish(self):
"""Finishes this HTTP request on the open connection."""
if options.server_trace:
self.headers.set("X-B3-TraceId", [hex_str(self.trace.trace_id)])
self.headers.set("X-B3-SpanId", [hex_str(self.trace.span_id)])
if self.trace.parent_span_id is not None:
self.headers.set("X-B3-ParentSpanId", [hex_str(self.trace.parent_span_id)])
self.trace.record(Annotation.server_send())
self.connection.finish()
self._finish_time = time.time()
def full_url(self):
"""Reconstructs the full URL for this request."""
return self.protocol + "://" + self.host + self.uri
def request_time(self):
"""Returns the amount of time it took for this request to execute."""
if self._finish_time is None:
return time.time() - self._start_time
else:
return self._finish_time - self._start_time
def get_ssl_certificate(self, binary_form=False):
"""Returns the client's SSL certificate, if any.
To use client certificates, the HTTPServer must have been constructed
with cert_reqs set in ssl_options, e.g.::
server = HTTPServer(app,
ssl_options=dict(
certfile="foo.crt",
keyfile="foo.key",
cert_reqs=ssl.CERT_REQUIRED,
ca_certs="cacert.crt"))
By default, the return value is a dictionary (or None, if no
client certificate is present). If ``binary_form`` is true, a
DER-encoded form of the certificate is returned instead. See
SSLSocket.getpeercert() in the standard library for more
details.
http://docs.python.org/library/ssl.html#sslsocket-objects
"""
try:
return self.connection.stream.socket.getpeercert(
binary_form=binary_form)
except ssl.SSLError:
return None
def __repr__(self):
attrs = ("protocol", "host", "method", "uri", "version", "remote_ip")
args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
return "%s(%s, headers=%s)" % (
self.__class__.__name__, args, dict(self.headers))
class RawRESTkinHTTPTracerTests(TestCase):
def assertBodyEquals(self, bodyProducer, expectedOutput):
output = StringIO()
def _check_body(_):
self.assertEqual(json.loads(output.getvalue()), expectedOutput)
d = bodyProducer.startProducing(output)
d.addCallback(_check_body)
return d
def setUp(self):
self.agent = mock.Mock()
self.tracer = RawRESTkinHTTPTracer(self.agent, 'http://trace.it')
self.trace = Trace('test', 1, 2, tracers=[self.tracer])
def test_verifyObject(self):
verifyObject(ITracer, self.tracer)
def test_posts_immediately(self):
self.trace.record(Annotation.client_send(1))
self.assertEqual(self.agent.request.call_count, 1)
args = self.agent.request.mock_calls[0][1]
self.assertEqual(('POST', 'http://trace.it', Headers({})), args[:3])
bodyProducer = args[3]
return self.assertBodyEquals(
bodyProducer,
[{
'trace_id': '0000000000000001',
'span_id': '0000000000000002',
'name': 'test',
'annotations': [{
'type': 'timestamp',
'value': 1,
'key': 'cs'
}]
}])
def test_handles_batched_traces(self):
t1 = self.trace
t2 = Trace('test2', 3, 4)
cs1 = Annotation.client_send(1)
cs2 = Annotation.client_send(2)
cr1 = Annotation.client_recv(3)
cr2 = Annotation.client_recv(4)
self.tracer.record([(t1, [cs1, cr1]), (t2, [cs2, cr2])])
self.assertEqual(self.agent.request.call_count, 1)
args = self.agent.request.mock_calls[0][1]
self.assertEqual(('POST', 'http://trace.it', Headers({})), args[:3])
bodyProducer = args[3]
return self.assertBodyEquals(bodyProducer, [{
'trace_id':
'0000000000000001',
'span_id':
'0000000000000002',
'name':
'test',
'annotations': [{
'type': 'timestamp',
'value': 1,
'key': 'cs'
}, {
'type': 'timestamp',
'value': 3,
'key': 'cr'
}]
}, {
'trace_id':
'0000000000000003',
'span_id':
'0000000000000004',
'name':
'test2',
'annotations': [{
'type': 'timestamp',
'value': 2,
'key': 'cs'
}, {
'type': 'timestamp',
'value': 4,
'key': 'cr'
}]
}])
def setUp(self):
self.agent = mock.Mock()
self.tracer = RawRESTkinHTTPTracer(self.agent, 'http://trace.it')
self.trace = Trace('test', 1, 2, tracers=[self.tracer])
def __init__(self,
url,
method="GET",
headers=None,
body=None,
auth_username=None,
auth_password=None,
auth_mode=None,
connect_timeout=None,
request_timeout=None,
if_modified_since=None,
follow_redirects=None,
max_redirects=None,
user_agent=None,
use_gzip=None,
network_interface=None,
streaming_callback=None,
header_callback=None,
prepare_curl_callback=None,
proxy_host=None,
proxy_port=None,
proxy_username=None,
proxy_password=None,
allow_nonstandard_methods=None,
validate_cert=None,
ca_certs=None,
allow_ipv6=None,
client_key=None,
client_cert=None,
parent_trace=None,
endpoint=None):
r"""All parameters except ``url`` are optional.
:arg string url: URL to fetch
:arg string method: HTTP method, e.g. "GET" or "POST"
:arg headers: Additional HTTP headers to pass on the request
:arg body: HTTP body to pass on the request
:type headers: `~tornado.httputil.HTTPHeaders` or `dict`
:arg string auth_username: Username for HTTP authentication
:arg string auth_password: Password for HTTP authentication
:arg string auth_mode: Authentication mode; default is "basic".
Allowed values are implementation-defined; ``curl_httpclient``
supports "basic" and "digest"; ``simple_httpclient`` only supports
"basic"
:arg float connect_timeout: Timeout for initial connection in seconds
:arg float request_timeout: Timeout for entire request in seconds
:arg if_modified_since: Timestamp for ``If-Modified-Since`` header
:type if_modified_since: `datetime` or `float`
:arg bool follow_redirects: Should redirects be followed automatically
or return the 3xx response?
:arg int max_redirects: Limit for ``follow_redirects``
:arg string user_agent: String to send as ``User-Agent`` header
:arg bool use_gzip: Request gzip encoding from the server
:arg string network_interface: Network interface to use for request.
``curl_httpclient`` only; see note below.
:arg callable streaming_callback: If set, ``streaming_callback`` will
be run with each chunk of data as it is received, and
``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
the final response.
:arg callable header_callback: If set, ``header_callback`` will
be run with each header line as it is received (including the
first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
containing only ``\r\n``. All lines include the trailing newline
characters). ``HTTPResponse.headers`` will be empty in the final
response. This is most useful in conjunction with
``streaming_callback``, because it's the only way to get access to
header data while the request is in progress.
:arg callable prepare_curl_callback: If set, will be called with
a ``pycurl.Curl`` object to allow the application to make additional
``setopt`` calls.
:arg string proxy_host: HTTP proxy hostname. To use proxies,
``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and
``proxy_pass`` are optional. Proxies are currently only supported
with ``curl_httpclient``.
:arg int proxy_port: HTTP proxy port
:arg string proxy_username: HTTP proxy username
:arg string proxy_password: HTTP proxy password
:arg bool allow_nonstandard_methods: Allow unknown values for ``method``
argument?
:arg bool validate_cert: For HTTPS requests, validate the server's
certificate?
:arg string ca_certs: filename of CA certificates in PEM format,
or None to use defaults. See note below when used with
``curl_httpclient``.
:arg bool allow_ipv6: Use IPv6 when available? Default is false in
``simple_httpclient`` and true in ``curl_httpclient``
:arg string client_key: Filename for client SSL key, if any. See
note below when used with ``curl_httpclient``.
:arg string client_cert: Filename for client SSL certificate, if any.
See note below when used with ``curl_httpclient``.
:arg string parent_trace: parent trace id.
:arg string endpoint: request endpoint.
.. note::
When using ``curl_httpclient`` certain options may be
inherited by subsequent fetches because ``pycurl`` does
not allow them to be cleanly reset. This applies to the
``ca_certs``, ``client_key``, ``client_cert``, and
``network_interface`` arguments. If you use these
options, you should pass them on every request (you don't
have to always use the same values, but it's not possible
to mix requests that specify these options with ones that
use the defaults).
.. versionadded:: 3.1
The ``auth_mode`` argument.
"""
# Note that some of these attributes go through property setters
# defined below.
self.headers = headers
if if_modified_since:
self.headers["If-Modified-Since"] = httputil.format_timestamp(
if_modified_since)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.url = url
self.method = method
self.body = body
self.auth_username = auth_username
self.auth_password = auth_password
self.auth_mode = auth_mode
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
self.use_gzip = use_gzip
self.network_interface = network_interface
self.streaming_callback = streaming_callback
self.header_callback = header_callback
self.prepare_curl_callback = prepare_curl_callback
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.start_time = time.time()
self._parent_trace = parent_trace
self._endpoint = endpoint
self.trace = None
if options.client_trace:
if self._parent_trace is None:
self.trace = Trace(method)
else:
self.trace = self._parent_trace.child(method)
if self._endpoint is not None:
self.trace.set_endpoint(self._endpoint)
else:
self._endpoint = Endpoint(ipv4=socket.gethostbyname(
socket.gethostname()),
port=0,
service_name=service_name)
self.trace.set_endpoint(self._endpoint)
self.headers.set('X-B3-TraceId', [hex_str(self.trace.trace_id)])
self.headers.set('X-B3-SpanId', [hex_str(self.trace.span_id)])
if trace.parent_span_id is not None:
self.headers.set('X-B3-ParentSpanId',
[hex_str(self.trace.parent_span_id)])
self.trace.record(Annotation.string('HTTPClient REQ', self.url))
def __init__(self, url, method="GET", headers=None, body=None,
auth_username=None, auth_password=None, auth_mode=None,
connect_timeout=None, request_timeout=None,
if_modified_since=None, follow_redirects=None,
max_redirects=None, user_agent=None, use_gzip=None,
network_interface=None, streaming_callback=None,
header_callback=None, prepare_curl_callback=None,
proxy_host=None, proxy_port=None, proxy_username=None,
proxy_password=None, allow_nonstandard_methods=None,
validate_cert=None, ca_certs=None,
allow_ipv6=None,
client_key=None, client_cert=None,
parent_trace=None, endpoint=None):
r"""All parameters except ``url`` are optional.
:arg string url: URL to fetch
:arg string method: HTTP method, e.g. "GET" or "POST"
:arg headers: Additional HTTP headers to pass on the request
:arg body: HTTP body to pass on the request
:type headers: `~tornado.httputil.HTTPHeaders` or `dict`
:arg string auth_username: Username for HTTP authentication
:arg string auth_password: Password for HTTP authentication
:arg string auth_mode: Authentication mode; default is "basic".
Allowed values are implementation-defined; ``curl_httpclient``
supports "basic" and "digest"; ``simple_httpclient`` only supports
"basic"
:arg float connect_timeout: Timeout for initial connection in seconds
:arg float request_timeout: Timeout for entire request in seconds
:arg if_modified_since: Timestamp for ``If-Modified-Since`` header
:type if_modified_since: `datetime` or `float`
:arg bool follow_redirects: Should redirects be followed automatically
or return the 3xx response?
:arg int max_redirects: Limit for ``follow_redirects``
:arg string user_agent: String to send as ``User-Agent`` header
:arg bool use_gzip: Request gzip encoding from the server
:arg string network_interface: Network interface to use for request.
``curl_httpclient`` only; see note below.
:arg callable streaming_callback: If set, ``streaming_callback`` will
be run with each chunk of data as it is received, and
``HTTPResponse.body`` and ``HTTPResponse.buffer`` will be empty in
the final response.
:arg callable header_callback: If set, ``header_callback`` will
be run with each header line as it is received (including the
first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
containing only ``\r\n``. All lines include the trailing newline
characters). ``HTTPResponse.headers`` will be empty in the final
response. This is most useful in conjunction with
``streaming_callback``, because it's the only way to get access to
header data while the request is in progress.
:arg callable prepare_curl_callback: If set, will be called with
a ``pycurl.Curl`` object to allow the application to make additional
``setopt`` calls.
:arg string proxy_host: HTTP proxy hostname. To use proxies,
``proxy_host`` and ``proxy_port`` must be set; ``proxy_username`` and
``proxy_pass`` are optional. Proxies are currently only supported
with ``curl_httpclient``.
:arg int proxy_port: HTTP proxy port
:arg string proxy_username: HTTP proxy username
:arg string proxy_password: HTTP proxy password
:arg bool allow_nonstandard_methods: Allow unknown values for ``method``
argument?
:arg bool validate_cert: For HTTPS requests, validate the server's
certificate?
:arg string ca_certs: filename of CA certificates in PEM format,
or None to use defaults. See note below when used with
``curl_httpclient``.
:arg bool allow_ipv6: Use IPv6 when available? Default is false in
``simple_httpclient`` and true in ``curl_httpclient``
:arg string client_key: Filename for client SSL key, if any. See
note below when used with ``curl_httpclient``.
:arg string client_cert: Filename for client SSL certificate, if any.
See note below when used with ``curl_httpclient``.
:arg string parent_trace: parent trace id.
:arg string endpoint: request endpoint.
.. note::
When using ``curl_httpclient`` certain options may be
inherited by subsequent fetches because ``pycurl`` does
not allow them to be cleanly reset. This applies to the
``ca_certs``, ``client_key``, ``client_cert``, and
``network_interface`` arguments. If you use these
options, you should pass them on every request (you don't
have to always use the same values, but it's not possible
to mix requests that specify these options with ones that
use the defaults).
.. versionadded:: 3.1
The ``auth_mode`` argument.
"""
# Note that some of these attributes go through property setters
# defined below.
self.headers = headers
if if_modified_since:
self.headers["If-Modified-Since"] = httputil.format_timestamp(
if_modified_since)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_username = proxy_username
self.proxy_password = proxy_password
self.url = url
self.method = method
self.body = body
self.auth_username = auth_username
self.auth_password = auth_password
self.auth_mode = auth_mode
self.connect_timeout = connect_timeout
self.request_timeout = request_timeout
self.follow_redirects = follow_redirects
self.max_redirects = max_redirects
self.user_agent = user_agent
self.use_gzip = use_gzip
self.network_interface = network_interface
self.streaming_callback = streaming_callback
self.header_callback = header_callback
self.prepare_curl_callback = prepare_curl_callback
self.allow_nonstandard_methods = allow_nonstandard_methods
self.validate_cert = validate_cert
self.ca_certs = ca_certs
self.allow_ipv6 = allow_ipv6
self.client_key = client_key
self.client_cert = client_cert
self.start_time = time.time()
self._parent_trace = parent_trace
self._endpoint = endpoint
self.trace = None
if options.client_trace:
if self._parent_trace is None:
self.trace = Trace(method)
else:
self.trace = self._parent_trace.child(method)
if self._endpoint is not None:
self.trace.set_endpoint(self._endpoint)
else:
self._endpoint = Endpoint(ipv4=socket.gethostbyname(socket.gethostname()), port=0, service_name=service_name)
self.trace.set_endpoint(self._endpoint)
self.headers.set('X-B3-TraceId', [hex_str(self.trace.trace_id)])
self.headers.set('X-B3-SpanId', [hex_str(self.trace.span_id)])
if trace.parent_span_id is not None:
self.headers.set('X-B3-ParentSpanId', [hex_str(self.trace.parent_span_id)])
self.trace.record(Annotation.string('HTTPClient REQ', self.url))
def __init__(self,
method,
uri,
version="HTTP/1.0",
headers=None,
body=None,
remote_ip=None,
protocol=None,
host=None,
files=None,
connection=None):
self.method = method
self.uri = uri
self.version = version
self.headers = headers or httputil.HTTPHeaders()
self.body = body or ""
self.trace = None
# set remote IP and protocol
self.remote_ip = remote_ip
if protocol:
self.protocol = protocol
elif connection and isinstance(connection.stream,
iostream.SSLIOStream):
self.protocol = "https"
else:
self.protocol = "http"
# xheaders can override the defaults
if connection and connection.xheaders:
# Squid uses X-Forwarded-For, others use X-Real-Ip
ip = self.headers.get("X-Forwarded-For", self.remote_ip)
ip = ip.split(',')[-1].strip()
ip = self.headers.get("X-Real-Ip", ip)
if netutil.is_valid_ip(ip):
self.remote_ip = ip
# AWS uses X-Forwarded-Proto
proto = self.headers.get(
"X-Scheme", self.headers.get("X-Forwarded-Proto",
self.protocol))
if proto in ("http", "https"):
self.protocol = proto
# Zipkin users
if options.server_trace:
parent_span_id = self.headers.get("X-B3-ParentSpanId", None)
trace_id = self.headers.get("X-B3-TraceId", None)
span_id = self.headers.get("X-B3-SpanId", None)
name = method
endpoint = Endpoint(ipv4=socket.gethostbyname(
socket.gethostname()),
port=port,
service_name=service_name)
self.trace = Trace(name=name,
trace_id=trace_id,
span_id=span_id,
parent_span_id=parent_span_id)
self.trace.set_endpoint(endpoint)
self.host = host or self.headers.get("Host") or "127.0.0.1"
self.files = files or {}
self.connection = connection
self._start_time = time.time()
self._finish_time = None
self.path, sep, self.query = uri.partition('?')
self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)
self.query_arguments = copy.deepcopy(self.arguments)
self.body_arguments = {}
if options.server_trace:
self.trace.record(Annotation.string('Url', uri))
self.trace.record(Annotation.string('Header', self.headers))
self.trace.record(Annotation.server_recv())
def test_no_unfinished_traces(self):
unfinished_trace = (Trace('unfinished'), [Annotation.client_send(1)])
self.tracer.record([unfinished_trace])
self.assertEqual(self.record_function.call_count, 0)
def test_repr(self):
trace = Trace('test_trace', trace_id=1, span_id=1, parent_span_id=1)
self.assertEqual(
repr(trace),
"Trace('test_trace', trace_id=1, span_id=1, parent_span_id=1)")
def setUp(self):
self.agent = mock.Mock()
self.tracer = RawRESTkinHTTPTracer(self.agent, 'http://trace.it')
self.trace = Trace('test', 1, 2, tracers=[self.tracer])
def process(self, data):
msg = json.loads(data)
if msg["frontend_name"] == "haproxy_monitoring":
return
frontend = FE_OR_BE.search(msg["frontend_name"]).group(2)
backend = FE_OR_BE.search(msg["backend_name"]).group(2)
server = SERVER_SUB.sub("_", msg["server_name"])
# Consider server received to be the accept_date plus the time to
# receive request headers.
accept = parse_date_micro(msg["accept_date"])
request = int(msg["time_request"]) * 1000
if request < 0:
request = 0
sr = accept + request
ctrace = Trace(msg["http_verb"],
trace_id=int_or_none(msg.get("zipkin_trace_id")),
span_id=int_or_none(msg.get("zipkin_next_span_id")),
parent_span_id=int_or_none(msg.get("zipkin_span_id")),
tracers=self.tracers)
# There's an assumption here and in the server endpoint that the 'host'
# parsed by Logstash from the syslog message is the same host as the
# service is running on.
endpoint = Endpoint(msg["host"], 0, backend)
ctrace.set_endpoint(endpoint)
# For client sent we consider the time in queue plus backend connect
# time (which is the time to get the SYN/ACK from the backend).
queue = int(msg["time_queue"]) * 1000
if queue < 0:
queue = 0
connect = int(msg["time_backend_connect"]) * 1000
if connect < 0:
connect = 0
cs = sr + queue + connect
ctrace.record(Annotation.client_send(cs))
ctrace.record(Annotation.string('haproxy.backend.server_name', server))
# Record response time in ms into an annotation since it doesn't fit
# anywhere in zipkin.
response = int(msg["time_backend_response"]) * 1000
if response < 0:
response = 0
ctrace.record(Annotation.string("haproxy.backend.time",
str(response)))
ctrace.record(Annotation.string("haproxy.backend.queue",
msg["backend_queue"]))
# Apache sends the duration in microseconds already. For haproxy we
# have to convert from ms.
duration = int(msg["time_duration"])
if msg["program"] == "haproxy":
duration = duration * 1000
if duration < 0:
duration = 0
# Assume client response time to be the same as the request duration
# minus a microsecond, just to keep ordering.
ss = sr + duration
cr = ss - 1
ctrace.record(Annotation.client_recv(cr))
# The top-level span is generally Apache. We record the parent span id
# as '-' there if the parent span id is missing so convert to None.
parent_span_id = msg.get("zipkin_parent_span_id")
if parent_span_id is not None and parent_span_id == "-":
parent_span_id = None
strace = Trace(msg["http_verb"],
trace_id=int_or_none(msg.get("zipkin_trace_id")),
span_id=int_or_none(msg.get("zipkin_span_id")),
parent_span_id=int_or_none(parent_span_id),
tracers=self.tracers)
endpoint = Endpoint(msg["host"], 0, frontend)
strace.set_endpoint(endpoint)
strace.record(Annotation.server_recv(sr))
strace.record(Annotation.string('http.uri', msg["http_request"]))
strace.record(Annotation.string(
'http.responsecode', msg["http_status_code"]))
strace.record(Annotation.server_send(ss))
def test_equality(self):
self.assertEqual(
Trace('test_trace', trace_id=1, span_id=1, parent_span_id=1),
Trace('test_trace2', trace_id=1, span_id=1, parent_span_id=1))
def test_verifyObject(self):
verifyObject(ITrace, Trace('test'))
class HTTPRequest(object):
"""A single HTTP request.
All attributes are type `str` unless otherwise noted.
.. attribute:: method
HTTP request method, e.g. "GET" or "POST"
.. attribute:: uri
The requested uri.
.. attribute:: path
The path portion of `uri`
.. attribute:: query
The query portion of `uri`
.. attribute:: version
HTTP version specified in request, e.g. "HTTP/1.1"
.. attribute:: headers
`.HTTPHeaders` dictionary-like object for request headers. Acts like
a case-insensitive dictionary with additional methods for repeated
headers.
.. attribute:: body
Request body, if present, as a byte string.
.. attribute:: remote_ip
Client's IP address as a string. If ``HTTPServer.xheaders`` is set,
will pass along the real IP address provided by a load balancer
in the ``X-Real-Ip`` or ``X-Forwarded-For`` header.
.. versionchanged:: 3.1
The list format of ``X-Forwarded-For`` is now supported.
.. attribute:: protocol
The protocol used, either "http" or "https". If ``HTTPServer.xheaders``
is set, will pass along the protocol used by a load balancer if
reported via an ``X-Scheme`` header.
.. attribute:: host
The requested hostname, usually taken from the ``Host`` header.
.. attribute:: arguments
GET/POST arguments are available in the arguments property, which
maps arguments names to lists of values (to support multiple values
for individual names). Names are of type `str`, while arguments
are byte strings. Note that this is different from
`.RequestHandler.get_argument`, which returns argument values as
unicode strings.
.. attribute:: query_arguments
Same format as ``arguments``, but contains only arguments extracted
from the query string.
.. versionadded:: 3.2
.. attribute:: body_arguments
Same format as ``arguments``, but contains only arguments extracted
from the request body.
.. versionadded:: 3.2
.. attribute:: files
File uploads are available in the files property, which maps file
names to lists of `.HTTPFile`.
.. attribute:: connection
An HTTP request is attached to a single HTTP connection, which can
be accessed through the "connection" attribute. Since connections
are typically kept open in HTTP/1.1, multiple requests can be handled
sequentially on a single connection.
"""
def __init__(self,
method,
uri,
version="HTTP/1.0",
headers=None,
body=None,
remote_ip=None,
protocol=None,
host=None,
files=None,
connection=None):
self.method = method
self.uri = uri
self.version = version
self.headers = headers or httputil.HTTPHeaders()
self.body = body or ""
self.trace = None
# set remote IP and protocol
self.remote_ip = remote_ip
if protocol:
self.protocol = protocol
elif connection and isinstance(connection.stream,
iostream.SSLIOStream):
self.protocol = "https"
else:
self.protocol = "http"
# xheaders can override the defaults
if connection and connection.xheaders:
# Squid uses X-Forwarded-For, others use X-Real-Ip
ip = self.headers.get("X-Forwarded-For", self.remote_ip)
ip = ip.split(',')[-1].strip()
ip = self.headers.get("X-Real-Ip", ip)
if netutil.is_valid_ip(ip):
self.remote_ip = ip
# AWS uses X-Forwarded-Proto
proto = self.headers.get(
"X-Scheme", self.headers.get("X-Forwarded-Proto",
self.protocol))
if proto in ("http", "https"):
self.protocol = proto
# Zipkin users
if options.server_trace:
parent_span_id = self.headers.get("X-B3-ParentSpanId", None)
trace_id = self.headers.get("X-B3-TraceId", None)
span_id = self.headers.get("X-B3-SpanId", None)
name = method
endpoint = Endpoint(ipv4=socket.gethostbyname(
socket.gethostname()),
port=port,
service_name=service_name)
self.trace = Trace(name=name,
trace_id=trace_id,
span_id=span_id,
parent_span_id=parent_span_id)
self.trace.set_endpoint(endpoint)
self.host = host or self.headers.get("Host") or "127.0.0.1"
self.files = files or {}
self.connection = connection
self._start_time = time.time()
self._finish_time = None
self.path, sep, self.query = uri.partition('?')
self.arguments = parse_qs_bytes(self.query, keep_blank_values=True)
self.query_arguments = copy.deepcopy(self.arguments)
self.body_arguments = {}
if options.server_trace:
self.trace.record(Annotation.string('Url', uri))
self.trace.record(Annotation.string('Header', self.headers))
self.trace.record(Annotation.server_recv())
def supports_http_1_1(self):
"""Returns True if this request supports HTTP/1.1 semantics"""
return self.version == "HTTP/1.1"
@property
def cookies(self):
"""A dictionary of Cookie.Morsel objects."""
if not hasattr(self, "_cookies"):
self._cookies = Cookie.SimpleCookie()
if "Cookie" in self.headers:
try:
self._cookies.load(native_str(self.headers["Cookie"]))
except Exception:
self._cookies = {}
return self._cookies
def write(self, chunk, callback=None):
"""Writes the given chunk to the response stream."""
assert isinstance(chunk, bytes_type)
self.connection.write(chunk, callback=callback)
def finish(self):
"""Finishes this HTTP request on the open connection."""
if options.server_trace:
self.headers.set("X-B3-TraceId", [hex_str(self.trace.trace_id)])
self.headers.set("X-B3-SpanId", [hex_str(self.trace.span_id)])
if self.trace.parent_span_id is not None:
self.headers.set("X-B3-ParentSpanId",
[hex_str(self.trace.parent_span_id)])
self.trace.record(Annotation.server_send())
self.connection.finish()
self._finish_time = time.time()
def full_url(self):
"""Reconstructs the full URL for this request."""
return self.protocol + "://" + self.host + self.uri
def request_time(self):
"""Returns the amount of time it took for this request to execute."""
if self._finish_time is None:
return time.time() - self._start_time
else:
return self._finish_time - self._start_time
def get_ssl_certificate(self, binary_form=False):
"""Returns the client's SSL certificate, if any.
To use client certificates, the HTTPServer must have been constructed
with cert_reqs set in ssl_options, e.g.::
server = HTTPServer(app,
ssl_options=dict(
certfile="foo.crt",
keyfile="foo.key",
cert_reqs=ssl.CERT_REQUIRED,
ca_certs="cacert.crt"))
By default, the return value is a dictionary (or None, if no
client certificate is present). If ``binary_form`` is true, a
DER-encoded form of the certificate is returned instead. See
SSLSocket.getpeercert() in the standard library for more
details.
http://docs.python.org/library/ssl.html#sslsocket-objects
"""
try:
return self.connection.stream.socket.getpeercert(
binary_form=binary_form)
except ssl.SSLError:
return None
def __repr__(self):
attrs = ("protocol", "host", "method", "uri", "version", "remote_ip")
args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
return "%s(%s, headers=%s)" % (self.__class__.__name__, args,
dict(self.headers))