def fetch(): headers = make_cross_agent_headers(inbound_payload, ENCODING_KEY, cat_id) resp = yield from aiohttp_app.client.request(method, uri, headers=headers) if _raw_headers: raw_headers = _raw_headers else: raw_headers = { k.decode('utf-8'): v.decode('utf-8') for k, v in resp.raw_headers } if expected_intrinsics: # test valid CAT response header assert 'X-NewRelic-App-Data' in raw_headers app_data = json.loads( deobfuscate(raw_headers['X-NewRelic-App-Data'], ENCODING_KEY)) assert app_data[0] == cat_id assert app_data[1] == ('WebTransaction/Function/%s' % metric_name) else: assert 'X-NewRelic-App-Data' not in resp.headers
def process_response_headers(self, response_headers): """ Decode the response headers and create appropriate metics based on the header values. The response_headers are passed in as a list of tuples. [(HEADER_NAME0, HEADER_VALUE0), (HEADER_NAME1, HEADER_VALUE1)] """ settings = self.settings if not settings: return if not settings.cross_application_tracer.enabled: return appdata = None try: for k, v in response_headers: if k.upper() == self.cat_appdata_key.upper(): appdata = json_decode(deobfuscate(v, settings.encoding_key)) break if appdata: self.params['cross_process_id'] = appdata[0] self.params['external_txn_name'] = appdata[1] self.params['transaction_guid'] = appdata[5] except Exception: pass
def decode_newrelic_header(self, environ, header_name): encoded_header = environ.get(header_name) if encoded_header: try: decoded_header = json_decode(deobfuscate( encoded_header, self._settings.encoding_key)) except Exception: decoded_header = None return decoded_header
def test_capture_attributes_enabled(): settings = application_settings() assert settings.browser_monitoring.enabled assert settings.browser_monitoring.attributes.enabled assert settings.js_agent_loader response = fully_featured_application.get('/') header = response.html.html.head.script.string content = response.html.html.body.p.string footer = response.html.html.body.script.string # Validate actual body content. assert content == 'RESPONSE' # We no longer are in control of the JS contents of the header so # just check to make sure it contains at least the magic string # 'NREUM'. assert header.find('NREUM') != -1 # Now validate the various fields of the footer related to analytics. # The fields are held by a JSON dictionary. data = json.loads(footer.split('NREUM.info=')[1]) obfuscation_key = settings.license_key[:13] attributes = json.loads(deobfuscate(data['atts'], obfuscation_key)) user_attrs = attributes['u'] # When you round-trip through json encoding and json decoding, you # always end up with unicode (unicode in Python 2, str in Python 3.) # # Previously, we would drop attribute values of type 'bytes' in Python 3. # Now, we accept them and `json_encode` uses an encoding of 'latin-1', # just like it does for Python 2. This only applies to attributes in browser # monitoring browser_attributes = _user_attributes.copy() browser_attributes['bytes'] = u'bytes-value' browser_attributes['invalid-utf8'] = _user_attributes[ 'invalid-utf8'].decode('latin-1') browser_attributes['multibyte-utf8'] = _user_attributes[ 'multibyte-utf8'].decode('latin-1') for attr, value in browser_attributes.items(): assert user_attrs[attr] == value, ( "attribute %r expected %r, found %r" % (attr, value, user_attrs[attr]))
def test_footer_attributes(): settings = application_settings() assert settings.browser_monitoring.enabled assert settings.browser_key assert settings.browser_monitoring.loader_version assert settings.js_agent_loader assert isinstance(settings.js_agent_file, six.string_types) assert settings.beacon assert settings.error_beacon token = '0123456789ABCDEF' headers = {'Cookie': 'NRAGENT=tk=%s' % token} response = target_application_manual_rum.get('/', headers=headers) html = BeautifulSoup(response.body, 'html.parser') header = html.html.head.script.string content = html.html.body.p.string footer = html.html.body.script.string # Validate actual body content. assert content == 'RESPONSE' # Validate the insertion of RUM header. assert header.find('NREUM HEADER') != -1 # Now validate the various fields of the footer. The fields are # held by a JSON dictionary. data = json.loads(footer.split('NREUM.info=')[1]) assert data['licenseKey'] == settings.browser_key assert data['applicationID'] == settings.application_id assert data['agent'] == settings.js_agent_file assert data['beacon'] == settings.beacon assert data['errorBeacon'] == settings.error_beacon assert data['applicationTime'] >= 0 assert data['queueTime'] >= 0 obfuscation_key = settings.license_key[:13] assert type(data['transactionName']) == type(u'') txn_name = deobfuscate(data['transactionName'], obfuscation_key) assert txn_name == u'WebTransaction/Uri/' assert 'atts' not in data
def validate_outbound_headers(header_id='X-NewRelic-ID', header_transaction='X-NewRelic-Transaction'): transaction = current_transaction() headers = transaction._test_request_headers settings = transaction.settings encoding_key = settings.encoding_key assert header_id in headers values = headers[header_id] if isinstance(values, list): assert len(values) == 1, headers assert isinstance(values[0], type('')) value = values[0] else: value = values cross_process_id = deobfuscate(value, encoding_key) assert cross_process_id == settings.cross_process_id assert header_transaction in headers values = headers[header_transaction] if isinstance(values, list): assert len(values) == 1, headers assert isinstance(values[0], type('')) value = values[0] else: value = values (guid, record_tt, trip_id, path_hash) = \ json_decode(deobfuscate(value, encoding_key)) assert guid == transaction.guid assert record_tt == transaction.record_tt assert trip_id == transaction.trip_id assert path_hash == transaction.path_hash
def _test(): cat_headers = make_cross_agent_headers(inbound_payload, ENCODING_KEY, cat_id) response = app.fetch('get', url, headers=dict(cat_headers)) if expected_intrinsics: # test valid CAT response header assert b'X-NewRelic-App-Data' in raw_headers(response) cat_response_header = response.headers.get("X-NewRelic-App-Data", None) app_data = json.loads( deobfuscate(cat_response_header, ENCODING_KEY)) assert app_data[0] == cat_id assert app_data[1] == ('WebTransaction/Function/%s' % metric_name) else: assert b'X-NewRelic-App-Data' not in raw_headers(response)
def __init__(self, application, environ): # The web transaction can be enabled/disabled by # the value of the variable "newrelic.enabled" # in the WSGI environ dictionary. We need to check # this before initialising the transaction as needs # to be passed in base class constructor. The # default is None, which would then result in the # base class making the decision based on whether # application or agent as a whole are enabled. enabled = _lookup_environ_setting(environ, 'newrelic.enabled', None) # Initialise the common transaction base class. super(WebTransaction, self).__init__(application, enabled) # Disable transactions for websocket connections. # Also disable autorum if this is a websocket. This is a good idea for # two reasons. First, RUM is unnecessary for websocket transactions # anyway. Secondly, due to a bug in the gevent-websocket (0.9.5) # package, if our _WSGIApplicationMiddleware is applied a websocket # connection cannot be made. if _is_websocket(environ): self.autorum_disabled = True self.enabled = False # Bail out if the transaction is running in a # disabled state. if not self.enabled: return # Will need to check the settings a number of times. settings = self._settings # Check for override settings from WSGI environ. self.background_task = _lookup_environ_setting(environ, 'newrelic.set_background_task', False) self.ignore_transaction = _lookup_environ_setting(environ, 'newrelic.ignore_transaction', False) self.suppress_apdex = _lookup_environ_setting(environ, 'newrelic.suppress_apdex_metric', False) self.suppress_transaction_trace = _lookup_environ_setting(environ, 'newrelic.suppress_transaction_trace', False) self.capture_params = _lookup_environ_setting(environ, 'newrelic.capture_request_params', settings.capture_params) self.autorum_disabled = _lookup_environ_setting(environ, 'newrelic.disable_browser_autorum', not settings.browser_monitoring.auto_instrument) # Make sure that if high security mode is enabled that # capture of request params is still being disabled. # No warning is issued for this in the logs because it # is a per request configuration and would create a lot # of noise. if settings.high_security: self.capture_params = False # WSGI spec says SERVER_PORT "can never be empty string", # but I'm going to set a default value anyway... port = environ.get('SERVER_PORT', None) if port: try: self._port = int(port) except Exception: pass # Extract from the WSGI environ dictionary # details of the URL path. This will be set as # default path for the web transaction. This can # be overridden by framework to be more specific # to avoid metrics explosion problem resulting # from too many distinct URLs for same resource # due to use of REST style URL concepts or # otherwise. request_uri = environ.get('REQUEST_URI', None) if request_uri is None: # The gunicorn WSGI server uses RAW_URI instead # of the more typical REQUEST_URI used by Apache # and other web servers. request_uri = environ.get('RAW_URI', None) script_name = environ.get('SCRIPT_NAME', None) path_info = environ.get('PATH_INFO', None) self._request_uri = request_uri if self._request_uri is not None: # Need to make sure we drop off any query string # arguments on the path if we have to fallback # to using the original REQUEST_URI. Can't use # attribute access on result as only support for # Python 2.5+. self._request_uri = urlparse.urlparse(self._request_uri)[2] if script_name is not None or path_info is not None: if path_info is None: path = script_name elif script_name is None: path = path_info else: path = script_name + path_info self.set_transaction_name(path, 'Uri', priority=1) if self._request_uri is None: self._request_uri = path else: if self._request_uri is not None: self.set_transaction_name(self._request_uri, 'Uri', priority=1) # See if the WSGI environ dictionary includes the # special 'X-Request-Start' or 'X-Queue-Start' HTTP # headers. These header are optional headers that can be # set within the underlying web server or WSGI server to # indicate when the current request was first received # and ready to be processed. The difference between this # time and when application starts processing the # request is the queue time and represents how long # spent in any explicit request queuing system, or how # long waiting in connecting state against listener # sockets where request needs to be proxied between any # processes within the application server. # # Note that mod_wsgi sets its own distinct variables # automatically. Initially it set mod_wsgi.queue_start, # which equated to when Apache first accepted the # request. This got changed to mod_wsgi.request_start # however, and mod_wsgi.queue_start was instead used # just for when requests are to be queued up for the # daemon process and corresponded to the point at which # they are being proxied, after Apache does any # authentication etc. We check for both so older # versions of mod_wsgi will still work, although we # don't try and use the fact that it is possible to # distinguish the two points and just pick up the # earlier of the two. # # Checking for the mod_wsgi values means it is not # necessary to enable and use mod_headers to add X # -Request-Start or X-Queue-Start. But we still check # for the headers and give priority to the explicitly # added header in case that header was added in front # end server to Apache instead. # # Which ever header is used, we accommodate the value # being in seconds, milliseconds or microseconds. Also # handle it being prefixed with 't='. now = time.time() def _parse_time_stamp(time_stamp): """ Converts time_stamp to seconds. Input can be microseconds, milliseconds or seconds Divide the timestamp by the highest resolution divisor. If the result is older than Jan 1 2000, then pick a lower resolution divisor and repeat. It is safe to assume no requests were queued for more than 10 years. """ for divisor in (1000000.0, 1000.0, 1.0): converted_time = time_stamp/divisor # If queue_start is in the future, return 0.0. if converted_time > now: return 0.0 if converted_time > JAN_1_2000: return converted_time return 0.0 queue_time_headers = ('HTTP_X_REQUEST_START', 'HTTP_X_QUEUE_START', 'mod_wsgi.request_start', 'mod_wsgi.queue_start') for queue_time_header in queue_time_headers: value = environ.get(queue_time_header, None) try: if value.startswith('t='): try: self.queue_start = _parse_time_stamp(float(value[2:])) except Exception: pass else: try: self.queue_start = _parse_time_stamp(float(value)) except Exception: pass except Exception: pass if self.queue_start > 0.0: break # Capture query request string parameters, unless we're in # High Security Mode. if not settings.high_security: value = environ.get('QUERY_STRING', None) if value: try: params = urlparse.parse_qs(value, keep_blank_values=True) except Exception: params = cgi.parse_qs(value, keep_blank_values=True) self._request_params.update(params) # Check for Synthetics header if settings.synthetics.enabled and \ settings.trusted_account_ids and settings.encoding_key: try: header_name = 'HTTP_X_NEWRELIC_SYNTHETICS' header = self.decode_newrelic_header(environ, header_name) synthetics = _parse_synthetics_header(header) if synthetics['account_id'] in settings.trusted_account_ids: # Save obfuscated header, because we will pass it along # unchanged in all external requests. self.synthetics_header = environ.get(header_name) if synthetics['version'] == 1: self.synthetics_resource_id = synthetics['resource_id'] self.synthetics_job_id = synthetics['job_id'] self.synthetics_monitor_id = synthetics['monitor_id'] except Exception: pass # Check for the New Relic cross process ID header and extract # the relevant details. if settings.cross_application_tracer.enabled and \ settings.cross_process_id and settings.trusted_account_ids and \ settings.encoding_key: client_cross_process_id = environ.get('HTTP_X_NEWRELIC_ID') if client_cross_process_id: try: client_cross_process_id = deobfuscate( client_cross_process_id, settings.encoding_key) # The cross process ID consists of the client # account ID and the ID of the specific application # the client is recording requests against. We need # to validate that the client account ID is in the # list of trusted account IDs and ignore it if it # isn't. The trusted account IDs list has the # account IDs as integers, so save the client ones # away as integers here so easier to compare later. client_account_id, client_application_id = \ map(int, client_cross_process_id.split('#')) if client_account_id in settings.trusted_account_ids: self.client_cross_process_id = client_cross_process_id self.client_account_id = client_account_id self.client_application_id = client_application_id header_name = 'HTTP_X_NEWRELIC_TRANSACTION' txn_header = self.decode_newrelic_header( environ, header_name) if txn_header: self.is_part_of_cat = True self.referring_transaction_guid = txn_header[0] # Incoming record_tt is OR'd with existing # record_tt. In the scenario where we make multiple # ext request, this will ensure we don't set the # record_tt to False by a later request if it was # set to True by an earlier request. self.record_tt = self.record_tt or txn_header[1] if isinstance(txn_header[2], six.string_types): self._trip_id = txn_header[2] if isinstance(txn_header[3], six.string_types): self._referring_path_hash = txn_header[3] except Exception: pass # Capture WSGI request environ dictionary values. We capture # content length explicitly as will need it for cross process # metrics. self._read_length = int(environ.get('CONTENT_LENGTH') or -1) if settings.capture_environ: for name in settings.include_environ: if name in environ: self._request_environment[name] = environ[name] # Strip out the query params from the HTTP_REFERER if capture_params # is disabled in the settings. if (self._request_environment.get('HTTP_REFERER') and not self.capture_params): self._request_environment['HTTP_REFERER'] = \ _remove_query_string(self._request_environment['HTTP_REFERER']) try: if 'CONTENT_LENGTH' in self._request_environment: self._request_environment['CONTENT_LENGTH'] = int( self._request_environment['CONTENT_LENGTH']) except Exception: del self._request_environment['CONTENT_LENGTH'] # Flags for tracking whether RUM header and footer have been # generated. self.rum_header_generated = False self.rum_footer_generated = False
def decode_header(header, encoding_key=ENCODING_KEY): result = deobfuscate(header, encoding_key) return json_decode(result)