def test_blob(method, args): connect_response = data_collector._developer_mode_responses['connect'] connect_response[u'request_headers_map'] = {u'X-Foo': u'Bar'} session = data_collector.create_session('license_key', 'app_name', LINKED_APPLICATIONS, ENVIRONMENT, global_settings_dump()) check_request_called = [] def check_request(url, params, headers, data): check_request_called.append(True) # Check that headers besides the one specified in the request map are # present assert len(headers) == 3 assert headers['X-Foo'] == 'Bar' assert 'User-Agent' in headers assert headers['Content-Encoding'] == 'identity' data_collector._log_request = check_request sender = getattr(session, method) sender(*args) assert check_request_called
def test_no_blob_behavior(headers_map_present, method, args): if headers_map_present: connect_response = data_collector._developer_mode_responses['connect'] connect_response[u'request_headers_map'] = None session = data_collector.create_session('license_key', 'app_name', LINKED_APPLICATIONS, ENVIRONMENT, global_settings_dump()) check_request_called = [] def check_request(url, params, headers, data): check_request_called.append(True) # Check that no other headers besides the default headers are present assert len(headers) == 2 assert 'User-Agent' in headers assert headers['Content-Encoding'] == 'identity' data_collector._log_request = check_request sender = getattr(session, method) sender(*args) assert check_request_called
def test_server_side_config_precedence(): connect_response = data_collector._developer_mode_responses['connect'] connect_response[u'agent_config'] = {u'span_events.enabled': True} connect_response[u'span_events.enabled'] = False session = data_collector.create_session('license_key', 'app_name', LINKED_APPLICATIONS, ENVIRONMENT, global_settings_dump()) assert session.configuration.span_events.enabled is False
def connect( cls, app_name, linked_applications, environment, settings, client_cls=ApplicationModeClient, ): with cls(settings, client_cls=client_cls) as preconnect: redirect_host = preconnect.send("preconnect")["redirect_host"] with cls(settings, host=redirect_host, client_cls=client_cls) as protocol: configuration = protocol.send( "connect", cls._connect_payload(app_name, linked_applications, environment, settings), ) # Apply High Security Mode to server_config, so the local # security settings won't get overwritten when we overlay # the server settings on top of them. configuration = cls._apply_high_security_mode_fixups( configuration, settings) # The agent configuration for the application in constructed # by taking a snapshot of the locally constructed # configuration and overlaying it with that from the server, # as well as creating the attribute filter. settings = finalize_application_settings(configuration, settings) with cls(settings, host=redirect_host, client_cls=client_cls) as protocol: protocol.send( "agent_settings", (global_settings_dump(settings, serializable=True), )) if "messages" in configuration: for item in configuration["messages"]: message = item["message"] level = item["level"] logger_func = cls.LOGGER_FUNC_MAPPING.get(level, None) if logger_func: logger_func("%s", message) return protocol
def test_strip_proxy_details(settings): assert "license_key" in settings assert "api_key" in settings assert "proxy_scheme" in settings assert "proxy_host" in settings assert "proxy_port" in settings assert "proxy_user" in settings assert "proxy_port" in settings stripped = global_settings_dump(settings) # These should always be deleted. assert "license_key" not in stripped assert "api_key" not in stripped # These should be obfuscated or None. obfuscated = "****" assert stripped["proxy_user"] in (None, obfuscated) assert stripped["proxy_pass"] in (None, obfuscated) # The proxy_host and proxy_port will be preserved but proxy_host # needs to be checked to make sure it doesn't contain a username and # password. Do this through parsing to make sure not breaking the # structure in some way and also direct comparison of what is # expected as backup. assert "proxy_scheme" in settings assert "proxy_host" in stripped assert "proxy_port" in stripped proxy_host = stripped["proxy_host"] expected_proxy_host = stripped["expected_proxy_host"] if proxy_host is not None: components = urlparse.urlparse(proxy_host) if components.username: assert components.username == obfuscated if components.password: assert components.password == obfuscated assert proxy_host == expected_proxy_host
def test_full_uri_protocol_lifecycle(): """Exercises the following endpoints: * preconnect * connect * shutdown """ initialize_agent(app_name='Python Agent Test (test_full_uri_payloads)', default_settings=_default_settings) environment = () linked_apps = [] session = FullURIApplicationSession.create_session( None, global_settings().app_name, linked_apps, environment, global_settings_dump()) session.shutdown_session()
def create_session(license_key, app_name, linked_applications, environment, settings): _global_settings = global_settings() if _global_settings.developer_mode: session = DeveloperModeSession.create_session(license_key, app_name, linked_applications, environment, settings) else: session = ApplicationSession.create_session(license_key, app_name, linked_applications, environment, settings) # When session creation is unsucessful None is returned. We need to catch # that and return None. Session creation can fail if data-collector is down # or if the configuration is wrong, such as having the capture_params true # in high security mode. if session is None: return None # We now need to send up the final merged configuration using the # agent_settings() method. We must make sure we pass the # configuration through global_settings_dump() to strip/mask any # sensitive settings. We also convert values which are strings or # numerics to strings before sending to avoid problems with UI # interpreting the values strangely if sent as native types. application_settings = global_settings_dump(session.configuration) for key, value in list(six.iteritems(application_settings)): if not isinstance(key, six.string_types): del application_settings[key] if (not isinstance(value, six.string_types) and not isinstance(value, float) and not isinstance(value, six.integer_types)): application_settings[key] = repr(value) session.agent_settings(application_settings) return session
def create_session(license_key, app_name, linked_applications, environment, settings): _global_settings = global_settings() if _global_settings.developer_mode: session = DeveloperModeSession.create_session(license_key, app_name, linked_applications, environment, settings) else: session = ApplicationSession.create_session(license_key, app_name, linked_applications, environment, settings) # When session creation is unsucessful None is returned. We need to catch # that and return None. Session creation can fail if data-collector is down # or if the configuration is wrong, such as having the capture_params true # in high security mode. if session is None: return None # We now need to send up the final merged configuration using the # agent_settings() method. We must make sure we pass the # configuration through global_settings_dump() to strip/mask any # sensitive settings. We also convert values which are strings or # numerics to strings before sending to avoid problems with UI # interpreting the values strangely if sent as native types. application_settings = global_settings_dump(session.configuration) for key, value in list(six.iteritems(application_settings)): if not isinstance(key, six.string_types): del application_settings[key] if (not isinstance(value, six.string_types) and not isinstance(value, float) and not isinstance(value, six.integer_types)): application_settings[key] = repr(value) try: session.agent_settings(application_settings) except NetworkInterfaceException: # The reason for errors of this type have already been logged. # No matter what the error we just pass back None. The upper # layer will deal with not being successful. _logger.warning('Agent registration failed due to error in ' 'uploading agent settings. Registration should retry ' 'automatically.') pass except Exception: # Any other errors are going to be unexpected and likely will # indicate an issue with the implementation of the agent. _logger.exception('Unexpected exception when attempting to ' 'update agent settings with the data collector. Please ' 'report this problem to New Relic support for further ' 'investigation.') _logger.warning('Agent registration failed due to error in ' 'uploading agent settings. Registration should retry ' 'automatically.') pass else: return session
def _connect_payload(app_name, linked_applications, environment, settings): settings = global_settings_dump(settings) app_names = [app_name] + linked_applications hostname = system_info.gethostname( settings["heroku.use_dyno_names"], settings["heroku.dyno_name_prefixes_to_shorten"], ) ip_address = system_info.getips() connect_settings = {} connect_settings["browser_monitoring.loader"] = settings[ "browser_monitoring.loader" ] connect_settings["browser_monitoring.debug"] = settings[ "browser_monitoring.debug" ] security_settings = {} security_settings["capture_params"] = settings["capture_params"] security_settings["transaction_tracer"] = {} security_settings["transaction_tracer"]["record_sql"] = settings[ "transaction_tracer.record_sql" ] utilization_settings = {} # metadata_version corresponds to the utilization spec being used. utilization_settings["metadata_version"] = 5 utilization_settings[ "logical_processors" ] = system_info.logical_processor_count() utilization_settings["total_ram_mib"] = system_info.total_physical_memory() utilization_settings["hostname"] = hostname if ip_address: utilization_settings["ip_address"] = ip_address boot_id = system_info.BootIdUtilization.detect() if boot_id: utilization_settings["boot_id"] = boot_id utilization_conf = {} logical_processor_conf = settings["utilization.logical_processors"] total_ram_conf = settings["utilization.total_ram_mib"] hostname_conf = settings["utilization.billing_hostname"] if logical_processor_conf: utilization_conf["logical_processors"] = logical_processor_conf if total_ram_conf: utilization_conf["total_ram_mib"] = total_ram_conf if hostname_conf: utilization_conf["hostname"] = hostname_conf if utilization_conf: utilization_settings["config"] = utilization_conf vendors = [] if settings["utilization.detect_aws"]: vendors.append(AWSUtilization) if settings["utilization.detect_pcf"]: vendors.append(PCFUtilization) if settings["utilization.detect_gcp"]: vendors.append(GCPUtilization) if settings["utilization.detect_azure"]: vendors.append(AzureUtilization) utilization_vendor_settings = {} for vendor in vendors: metadata = vendor.detect() if metadata: utilization_vendor_settings[vendor.VENDOR_NAME] = metadata break if settings["utilization.detect_docker"]: docker = DockerUtilization.detect() if docker: utilization_vendor_settings["docker"] = docker if settings["utilization.detect_kubernetes"]: kubernetes = KubernetesUtilization.detect() if kubernetes: utilization_vendor_settings["kubernetes"] = kubernetes if utilization_vendor_settings: utilization_settings["vendors"] = utilization_vendor_settings display_host = settings["process_host.display_name"] if display_host is None: display_host = hostname metadata = {} for env_var in os.environ: if env_var.startswith("NEW_RELIC_METADATA_"): metadata[env_var] = os.environ[env_var] return ( { "host": hostname, "pid": os.getpid(), "language": "python", "app_name": app_names, "identifier": ",".join(app_names), "agent_version": version, "environment": environment, "metadata": metadata, "settings": connect_settings, "security_settings": security_settings, "utilization": utilization_settings, "high_security": settings["high_security"], "event_harvest_config": settings["event_harvest_config"], "labels": settings["labels"], "display_host": display_host, }, )
def create_session(license_key, app_name, linked_applications, environment, settings): _global_settings = global_settings() if _global_settings.developer_mode: session = DeveloperModeSession.create_session(license_key, app_name, linked_applications, environment, settings) else: session = ApplicationSession.create_session(license_key, app_name, linked_applications, environment, settings) # When session creation is unsuccessful None is returned. We need to catch # that and return None. Session creation can fail if data-collector is down # or if the configuration is wrong, such as having the capture_params true # in high security mode. if session is None: return None # We now need to send up the final merged configuration using the # agent_settings() method. We must make sure we pass the # configuration through global_settings_dump() to strip/mask any # sensitive settings. We also convert values which are strings or # numerics to strings before sending to avoid problems with UI # interpreting the values strangely if sent as native types. application_settings = global_settings_dump(session.configuration) for key, value in list(six.iteritems(application_settings)): if not isinstance(key, six.string_types): del application_settings[key] if (not isinstance(value, six.string_types) and not isinstance(value, float) and not isinstance(value, six.integer_types)): application_settings[key] = repr(value) try: session.agent_settings(application_settings) except NetworkInterfaceException: # The reason for errors of this type have already been logged. # No matter what the error we just pass back None. The upper # layer will deal with not being successful. _logger.warning('Agent registration failed due to error in ' 'uploading agent settings. Registration should retry ' 'automatically.') pass except Exception: # Any other errors are going to be unexpected and likely will # indicate an issue with the implementation of the agent. _logger.exception('Unexpected exception when attempting to ' 'update agent settings with the data collector. Please ' 'report this problem to New Relic support for further ' 'investigation.') _logger.warning('Agent registration failed due to error in ' 'uploading agent settings. Registration should retry ' 'automatically.') pass else: return session
def connect_to_data_collector(self): """Performs the actual registration of the application with the data collector if no current active session. """ if self._agent_shutdown: return if self._active_session: return if self._detect_deadlock: imp.acquire_lock() self._deadlock_event.set() imp.release_lock() # Register the application with the data collector. Any errors # that occur will be dealt with by create_session(). The result # will either be a session object or None. In the event of a # failure to register we will try again, gradually backing off # for longer and longer periods as we retry. The retry interval # will be capped at 300 seconds. retries = [(15, False, False), (15, False, False), (30, False, False), (60, True, False), (120, False, False), (300, False, True),] try: while not self._active_session: self._active_session = create_session(None, self._app_name, self.linked_applications, environment_settings(), global_settings_dump()) # We were successful, but first need to make sure we do # not have any problems with the agent normalization # rules provided by the data collector. These could blow # up when being compiled if the patterns are broken or # use text which conflicts with extensions in Python's # regular expression syntax. if self._active_session: configuration = self._active_session.configuration try: settings = global_settings() if settings.debug.log_normalization_rules: _logger.info('The URL normalization rules for ' '%r are %r.', self._app_name, configuration.url_rules) _logger.info('The metric normalization rules ' 'for %r are %r.', self._app_name, configuration.metric_name_rules) _logger.info('The transaction normalization ' 'rules for %r are %r.', self._app_name, configuration.transaction_name_rules) self._rules_engine['url'] = RulesEngine( configuration.url_rules) self._rules_engine['metric'] = RulesEngine( configuration.metric_name_rules) self._rules_engine['transaction'] = RulesEngine( configuration.transaction_name_rules) except Exception: _logger.exception('The agent normalization rules ' 'received from the data collector could not ' 'be compiled properly by the agent due to a ' 'syntactical error or other problem. Please ' 'report this to New Relic support for ' 'investigation.') # For good measure, in this situation we explicitly # shutdown the session as then the data collector # will record this. Ignore any error from this. Then # we discard the session so we go into a retry loop # on presumption that issue with the URL rules will # be fixed. try: self._active_session.shutdown_session() except Exception: pass self._active_session = None # Were we successful. If not go into the retry loop. Log # warnings or errors as per schedule associated with the # retry intervals. if not self._active_session: if retries: timeout, warning, error = retries.pop(0) if warning: _logger.warning('Registration of the application ' '%r with the data collector failed after ' 'multiple attempts. Check the prior log ' 'entries and remedy any issue as ' 'necessary, or if the problem persists, ' 'report this problem to New Relic ' 'support for further investigation.', self._app_name) elif error: _logger.error('Registration of the application ' '%r with the data collector failed after ' 'further additional attempts. Please ' 'report this problem to New Relic support ' 'for further investigation.', self._app_name) else: timeout = 300 _logger.debug('Retrying registration of the application ' '%r with the data collector after a further %d ' 'seconds.', self._app_name, timeout) time.sleep(timeout) continue # Ensure we have cleared out any cached data from a # prior agent run for this application. configuration = self._active_session.configuration with self._stats_lock: self._stats_engine.reset_stats(configuration) with self._stats_custom_lock: self._stats_custom_engine.reset_stats(configuration) # Record an initial start time for the reporting period and # clear record of last transaction processed. self._period_start = time.time() self._transaction_count = 0 self._last_transaction = 0.0 # Clear any prior count of harvest merges due to failures. self._merge_count = 0 # Flag that the session activation has completed to # anyone who has been waiting through calling the # wait_for_session_activation() method. self._connected_event.set() except Exception: # If an exception occurs after agent has been flagged to be # shutdown then we ignore the error. This is because all # sorts of wierd errors could occur when main thread start # destroying objects and this background thread to register # the application is still running. if not self._agent_shutdown: _logger.exception('Unexpected exception when registering ' 'agent with the data collector. If this problem ' 'persists, please report this problem to New Relic ' 'support for further investigation.')