def fetch():
headers = make_cross_agent_headers(inbound_payload, ENCODING_KEY,
cat_id)
resp = yield from aiohttp_app.client.request(method,
uri,
headers=headers)
if _raw_headers:
raw_headers = _raw_headers
else:
raw_headers = {
k.decode('utf-8'): v.decode('utf-8')
for k, v in resp.raw_headers
}
if expected_intrinsics:
# test valid CAT response header
assert 'X-NewRelic-App-Data' in raw_headers
app_data = json.loads(
deobfuscate(raw_headers['X-NewRelic-App-Data'], ENCODING_KEY))
assert app_data[0] == cat_id
assert app_data[1] == ('WebTransaction/Function/%s' % metric_name)
else:
assert 'X-NewRelic-App-Data' not in resp.headers
def process_response_headers(self, response_headers):
"""
Decode the response headers and create appropriate metics based on the
header values. The response_headers are passed in as a list of tuples.
[(HEADER_NAME0, HEADER_VALUE0), (HEADER_NAME1, HEADER_VALUE1)]
"""
settings = self.settings
if not settings:
return
if not settings.cross_application_tracer.enabled:
return
appdata = None
try:
for k, v in response_headers:
if k.upper() == self.cat_appdata_key.upper():
appdata = json_decode(deobfuscate(v,
settings.encoding_key))
break
if appdata:
self.params['cross_process_id'] = appdata[0]
self.params['external_txn_name'] = appdata[1]
self.params['transaction_guid'] = appdata[5]
except Exception:
pass
def decode_newrelic_header(self, environ, header_name):
encoded_header = environ.get(header_name)
if encoded_header:
try:
decoded_header = json_decode(deobfuscate(
encoded_header, self._settings.encoding_key))
except Exception:
decoded_header = None
return decoded_header
def test_capture_attributes_enabled():
settings = application_settings()
assert settings.browser_monitoring.enabled
assert settings.browser_monitoring.attributes.enabled
assert settings.js_agent_loader
response = fully_featured_application.get('/')
header = response.html.html.head.script.string
content = response.html.html.body.p.string
footer = response.html.html.body.script.string
# Validate actual body content.
assert content == 'RESPONSE'
# We no longer are in control of the JS contents of the header so
# just check to make sure it contains at least the magic string
# 'NREUM'.
assert header.find('NREUM') != -1
# Now validate the various fields of the footer related to analytics.
# The fields are held by a JSON dictionary.
data = json.loads(footer.split('NREUM.info=')[1])
obfuscation_key = settings.license_key[:13]
attributes = json.loads(deobfuscate(data['atts'], obfuscation_key))
user_attrs = attributes['u']
# When you round-trip through json encoding and json decoding, you
# always end up with unicode (unicode in Python 2, str in Python 3.)
#
# Previously, we would drop attribute values of type 'bytes' in Python 3.
# Now, we accept them and `json_encode` uses an encoding of 'latin-1',
# just like it does for Python 2. This only applies to attributes in browser
# monitoring
browser_attributes = _user_attributes.copy()
browser_attributes['bytes'] = u'bytes-value'
browser_attributes['invalid-utf8'] = _user_attributes[
'invalid-utf8'].decode('latin-1')
browser_attributes['multibyte-utf8'] = _user_attributes[
'multibyte-utf8'].decode('latin-1')
for attr, value in browser_attributes.items():
assert user_attrs[attr] == value, (
"attribute %r expected %r, found %r" %
(attr, value, user_attrs[attr]))
def test_footer_attributes():
settings = application_settings()
assert settings.browser_monitoring.enabled
assert settings.browser_key
assert settings.browser_monitoring.loader_version
assert settings.js_agent_loader
assert isinstance(settings.js_agent_file, six.string_types)
assert settings.beacon
assert settings.error_beacon
token = '0123456789ABCDEF'
headers = {'Cookie': 'NRAGENT=tk=%s' % token}
response = target_application_manual_rum.get('/', headers=headers)
html = BeautifulSoup(response.body, 'html.parser')
header = html.html.head.script.string
content = html.html.body.p.string
footer = html.html.body.script.string
# Validate actual body content.
assert content == 'RESPONSE'
# Validate the insertion of RUM header.
assert header.find('NREUM HEADER') != -1
# Now validate the various fields of the footer. The fields are
# held by a JSON dictionary.
data = json.loads(footer.split('NREUM.info=')[1])
assert data['licenseKey'] == settings.browser_key
assert data['applicationID'] == settings.application_id
assert data['agent'] == settings.js_agent_file
assert data['beacon'] == settings.beacon
assert data['errorBeacon'] == settings.error_beacon
assert data['applicationTime'] >= 0
assert data['queueTime'] >= 0
obfuscation_key = settings.license_key[:13]
assert type(data['transactionName']) == type(u'')
txn_name = deobfuscate(data['transactionName'], obfuscation_key)
assert txn_name == u'WebTransaction/Uri/'
assert 'atts' not in data
def validate_outbound_headers(header_id='X-NewRelic-ID',
header_transaction='X-NewRelic-Transaction'):
transaction = current_transaction()
headers = transaction._test_request_headers
settings = transaction.settings
encoding_key = settings.encoding_key
assert header_id in headers
values = headers[header_id]
if isinstance(values, list):
assert len(values) == 1, headers
assert isinstance(values[0], type(''))
value = values[0]
else:
value = values
cross_process_id = deobfuscate(value, encoding_key)
assert cross_process_id == settings.cross_process_id
assert header_transaction in headers
values = headers[header_transaction]
if isinstance(values, list):
assert len(values) == 1, headers
assert isinstance(values[0], type(''))
value = values[0]
else:
value = values
(guid, record_tt, trip_id, path_hash) = \
json_decode(deobfuscate(value, encoding_key))
assert guid == transaction.guid
assert record_tt == transaction.record_tt
assert trip_id == transaction.trip_id
assert path_hash == transaction.path_hash
def _test():
cat_headers = make_cross_agent_headers(inbound_payload, ENCODING_KEY,
cat_id)
response = app.fetch('get', url, headers=dict(cat_headers))
if expected_intrinsics:
# test valid CAT response header
assert b'X-NewRelic-App-Data' in raw_headers(response)
cat_response_header = response.headers.get("X-NewRelic-App-Data",
None)
app_data = json.loads(
deobfuscate(cat_response_header, ENCODING_KEY))
assert app_data[0] == cat_id
assert app_data[1] == ('WebTransaction/Function/%s' % metric_name)
else:
assert b'X-NewRelic-App-Data' not in raw_headers(response)
def __init__(self, application, environ):
# The web transaction can be enabled/disabled by
# the value of the variable "newrelic.enabled"
# in the WSGI environ dictionary. We need to check
# this before initialising the transaction as needs
# to be passed in base class constructor. The
# default is None, which would then result in the
# base class making the decision based on whether
# application or agent as a whole are enabled.
enabled = _lookup_environ_setting(environ,
'newrelic.enabled', None)
# Initialise the common transaction base class.
super(WebTransaction, self).__init__(application, enabled)
# Disable transactions for websocket connections.
# Also disable autorum if this is a websocket. This is a good idea for
# two reasons. First, RUM is unnecessary for websocket transactions
# anyway. Secondly, due to a bug in the gevent-websocket (0.9.5)
# package, if our _WSGIApplicationMiddleware is applied a websocket
# connection cannot be made.
if _is_websocket(environ):
self.autorum_disabled = True
self.enabled = False
# Bail out if the transaction is running in a
# disabled state.
if not self.enabled:
return
# Will need to check the settings a number of times.
settings = self._settings
# Check for override settings from WSGI environ.
self.background_task = _lookup_environ_setting(environ,
'newrelic.set_background_task', False)
self.ignore_transaction = _lookup_environ_setting(environ,
'newrelic.ignore_transaction', False)
self.suppress_apdex = _lookup_environ_setting(environ,
'newrelic.suppress_apdex_metric', False)
self.suppress_transaction_trace = _lookup_environ_setting(environ,
'newrelic.suppress_transaction_trace', False)
self.capture_params = _lookup_environ_setting(environ,
'newrelic.capture_request_params',
settings.capture_params)
self.autorum_disabled = _lookup_environ_setting(environ,
'newrelic.disable_browser_autorum',
not settings.browser_monitoring.auto_instrument)
# Make sure that if high security mode is enabled that
# capture of request params is still being disabled.
# No warning is issued for this in the logs because it
# is a per request configuration and would create a lot
# of noise.
if settings.high_security:
self.capture_params = False
# WSGI spec says SERVER_PORT "can never be empty string",
# but I'm going to set a default value anyway...
port = environ.get('SERVER_PORT', None)
if port:
try:
self._port = int(port)
except Exception:
pass
# Extract from the WSGI environ dictionary
# details of the URL path. This will be set as
# default path for the web transaction. This can
# be overridden by framework to be more specific
# to avoid metrics explosion problem resulting
# from too many distinct URLs for same resource
# due to use of REST style URL concepts or
# otherwise.
request_uri = environ.get('REQUEST_URI', None)
if request_uri is None:
# The gunicorn WSGI server uses RAW_URI instead
# of the more typical REQUEST_URI used by Apache
# and other web servers.
request_uri = environ.get('RAW_URI', None)
script_name = environ.get('SCRIPT_NAME', None)
path_info = environ.get('PATH_INFO', None)
self._request_uri = request_uri
if self._request_uri is not None:
# Need to make sure we drop off any query string
# arguments on the path if we have to fallback
# to using the original REQUEST_URI. Can't use
# attribute access on result as only support for
# Python 2.5+.
self._request_uri = urlparse.urlparse(self._request_uri)[2]
if script_name is not None or path_info is not None:
if path_info is None:
path = script_name
elif script_name is None:
path = path_info
else:
path = script_name + path_info
self.set_transaction_name(path, 'Uri', priority=1)
if self._request_uri is None:
self._request_uri = path
else:
if self._request_uri is not None:
self.set_transaction_name(self._request_uri, 'Uri', priority=1)
# See if the WSGI environ dictionary includes the
# special 'X-Request-Start' or 'X-Queue-Start' HTTP
# headers. These header are optional headers that can be
# set within the underlying web server or WSGI server to
# indicate when the current request was first received
# and ready to be processed. The difference between this
# time and when application starts processing the
# request is the queue time and represents how long
# spent in any explicit request queuing system, or how
# long waiting in connecting state against listener
# sockets where request needs to be proxied between any
# processes within the application server.
#
# Note that mod_wsgi sets its own distinct variables
# automatically. Initially it set mod_wsgi.queue_start,
# which equated to when Apache first accepted the
# request. This got changed to mod_wsgi.request_start
# however, and mod_wsgi.queue_start was instead used
# just for when requests are to be queued up for the
# daemon process and corresponded to the point at which
# they are being proxied, after Apache does any
# authentication etc. We check for both so older
# versions of mod_wsgi will still work, although we
# don't try and use the fact that it is possible to
# distinguish the two points and just pick up the
# earlier of the two.
#
# Checking for the mod_wsgi values means it is not
# necessary to enable and use mod_headers to add X
# -Request-Start or X-Queue-Start. But we still check
# for the headers and give priority to the explicitly
# added header in case that header was added in front
# end server to Apache instead.
#
# Which ever header is used, we accommodate the value
# being in seconds, milliseconds or microseconds. Also
# handle it being prefixed with 't='.
now = time.time()
def _parse_time_stamp(time_stamp):
"""
Converts time_stamp to seconds. Input can be microseconds,
milliseconds or seconds
Divide the timestamp by the highest resolution divisor. If
the result is older than Jan 1 2000, then pick a lower
resolution divisor and repeat. It is safe to assume no
requests were queued for more than 10 years.
"""
for divisor in (1000000.0, 1000.0, 1.0):
converted_time = time_stamp/divisor
# If queue_start is in the future, return 0.0.
if converted_time > now:
return 0.0
if converted_time > JAN_1_2000:
return converted_time
return 0.0
queue_time_headers = ('HTTP_X_REQUEST_START', 'HTTP_X_QUEUE_START',
'mod_wsgi.request_start', 'mod_wsgi.queue_start')
for queue_time_header in queue_time_headers:
value = environ.get(queue_time_header, None)
try:
if value.startswith('t='):
try:
self.queue_start = _parse_time_stamp(float(value[2:]))
except Exception:
pass
else:
try:
self.queue_start = _parse_time_stamp(float(value))
except Exception:
pass
except Exception:
pass
if self.queue_start > 0.0:
break
# Capture query request string parameters, unless we're in
# High Security Mode.
if not settings.high_security:
value = environ.get('QUERY_STRING', None)
if value:
try:
params = urlparse.parse_qs(value, keep_blank_values=True)
except Exception:
params = cgi.parse_qs(value, keep_blank_values=True)
self._request_params.update(params)
# Check for Synthetics header
if settings.synthetics.enabled and \
settings.trusted_account_ids and settings.encoding_key:
try:
header_name = 'HTTP_X_NEWRELIC_SYNTHETICS'
header = self.decode_newrelic_header(environ, header_name)
synthetics = _parse_synthetics_header(header)
if synthetics['account_id'] in settings.trusted_account_ids:
# Save obfuscated header, because we will pass it along
# unchanged in all external requests.
self.synthetics_header = environ.get(header_name)
if synthetics['version'] == 1:
self.synthetics_resource_id = synthetics['resource_id']
self.synthetics_job_id = synthetics['job_id']
self.synthetics_monitor_id = synthetics['monitor_id']
except Exception:
pass
# Check for the New Relic cross process ID header and extract
# the relevant details.
if settings.cross_application_tracer.enabled and \
settings.cross_process_id and settings.trusted_account_ids and \
settings.encoding_key:
client_cross_process_id = environ.get('HTTP_X_NEWRELIC_ID')
if client_cross_process_id:
try:
client_cross_process_id = deobfuscate(
client_cross_process_id, settings.encoding_key)
# The cross process ID consists of the client
# account ID and the ID of the specific application
# the client is recording requests against. We need
# to validate that the client account ID is in the
# list of trusted account IDs and ignore it if it
# isn't. The trusted account IDs list has the
# account IDs as integers, so save the client ones
# away as integers here so easier to compare later.
client_account_id, client_application_id = \
map(int, client_cross_process_id.split('#'))
if client_account_id in settings.trusted_account_ids:
self.client_cross_process_id = client_cross_process_id
self.client_account_id = client_account_id
self.client_application_id = client_application_id
header_name = 'HTTP_X_NEWRELIC_TRANSACTION'
txn_header = self.decode_newrelic_header(
environ, header_name)
if txn_header:
self.is_part_of_cat = True
self.referring_transaction_guid = txn_header[0]
# Incoming record_tt is OR'd with existing
# record_tt. In the scenario where we make multiple
# ext request, this will ensure we don't set the
# record_tt to False by a later request if it was
# set to True by an earlier request.
self.record_tt = self.record_tt or txn_header[1]
if isinstance(txn_header[2], six.string_types):
self._trip_id = txn_header[2]
if isinstance(txn_header[3], six.string_types):
self._referring_path_hash = txn_header[3]
except Exception:
pass
# Capture WSGI request environ dictionary values. We capture
# content length explicitly as will need it for cross process
# metrics.
self._read_length = int(environ.get('CONTENT_LENGTH') or -1)
if settings.capture_environ:
for name in settings.include_environ:
if name in environ:
self._request_environment[name] = environ[name]
# Strip out the query params from the HTTP_REFERER if capture_params
# is disabled in the settings.
if (self._request_environment.get('HTTP_REFERER') and
not self.capture_params):
self._request_environment['HTTP_REFERER'] = \
_remove_query_string(self._request_environment['HTTP_REFERER'])
try:
if 'CONTENT_LENGTH' in self._request_environment:
self._request_environment['CONTENT_LENGTH'] = int(
self._request_environment['CONTENT_LENGTH'])
except Exception:
del self._request_environment['CONTENT_LENGTH']
# Flags for tracking whether RUM header and footer have been
# generated.
self.rum_header_generated = False
self.rum_footer_generated = False
def decode_header(header, encoding_key=ENCODING_KEY):
result = deobfuscate(header, encoding_key)
return json_decode(result)