def test_blob(method, args):
    connect_response = data_collector._developer_mode_responses['connect']
    connect_response[u'request_headers_map'] = {u'X-Foo': u'Bar'}

    session = data_collector.create_session('license_key', 'app_name',
                                            LINKED_APPLICATIONS, ENVIRONMENT,
                                            global_settings_dump())

    check_request_called = []

    def check_request(url, params, headers, data):
        check_request_called.append(True)

        # Check that headers besides the one specified in the request map are
        # present
        assert len(headers) == 3

        assert headers['X-Foo'] == 'Bar'

        assert 'User-Agent' in headers
        assert headers['Content-Encoding'] == 'identity'

    data_collector._log_request = check_request
    sender = getattr(session, method)
    sender(*args)

    assert check_request_called
def test_no_blob_behavior(headers_map_present, method, args):
    if headers_map_present:
        connect_response = data_collector._developer_mode_responses['connect']
        connect_response[u'request_headers_map'] = None

    session = data_collector.create_session('license_key', 'app_name',
                                            LINKED_APPLICATIONS, ENVIRONMENT,
                                            global_settings_dump())

    check_request_called = []

    def check_request(url, params, headers, data):
        check_request_called.append(True)

        # Check that no other headers besides the default headers are present
        assert len(headers) == 2

        assert 'User-Agent' in headers
        assert headers['Content-Encoding'] == 'identity'

    data_collector._log_request = check_request
    sender = getattr(session, method)
    sender(*args)

    assert check_request_called
def test_server_side_config_precedence():
    connect_response = data_collector._developer_mode_responses['connect']
    connect_response[u'agent_config'] = {u'span_events.enabled': True}
    connect_response[u'span_events.enabled'] = False

    session = data_collector.create_session('license_key', 'app_name',
                                            LINKED_APPLICATIONS, ENVIRONMENT,
                                            global_settings_dump())

    assert session.configuration.span_events.enabled is False
Exemplo n.º 4
0
    def connect(
        cls,
        app_name,
        linked_applications,
        environment,
        settings,
        client_cls=ApplicationModeClient,
    ):
        with cls(settings, client_cls=client_cls) as preconnect:
            redirect_host = preconnect.send("preconnect")["redirect_host"]

        with cls(settings, host=redirect_host,
                 client_cls=client_cls) as protocol:
            configuration = protocol.send(
                "connect",
                cls._connect_payload(app_name, linked_applications,
                                     environment, settings),
            )

        # Apply High Security Mode to server_config, so the local
        # security settings won't get overwritten when we overlay
        # the server settings on top of them.

        configuration = cls._apply_high_security_mode_fixups(
            configuration, settings)

        # The agent configuration for the application in constructed
        # by taking a snapshot of the locally constructed
        # configuration and overlaying it with that from the server,
        # as well as creating the attribute filter.
        settings = finalize_application_settings(configuration, settings)

        with cls(settings, host=redirect_host,
                 client_cls=client_cls) as protocol:
            protocol.send(
                "agent_settings",
                (global_settings_dump(settings, serializable=True), ))

        if "messages" in configuration:
            for item in configuration["messages"]:
                message = item["message"]
                level = item["level"]
                logger_func = cls.LOGGER_FUNC_MAPPING.get(level, None)
                if logger_func:
                    logger_func("%s", message)

        return protocol
def test_strip_proxy_details(settings):
    assert "license_key" in settings
    assert "api_key" in settings

    assert "proxy_scheme" in settings
    assert "proxy_host" in settings
    assert "proxy_port" in settings
    assert "proxy_user" in settings
    assert "proxy_port" in settings

    stripped = global_settings_dump(settings)

    # These should always be deleted.

    assert "license_key" not in stripped
    assert "api_key" not in stripped

    # These should be obfuscated or None.

    obfuscated = "****"

    assert stripped["proxy_user"] in (None, obfuscated)
    assert stripped["proxy_pass"] in (None, obfuscated)

    # The proxy_host and proxy_port will be preserved but proxy_host
    # needs to be checked to make sure it doesn't contain a username and
    # password. Do this through parsing to make sure not breaking the
    # structure in some way and also direct comparison of what is
    # expected as backup.

    assert "proxy_scheme" in settings
    assert "proxy_host" in stripped
    assert "proxy_port" in stripped

    proxy_host = stripped["proxy_host"]
    expected_proxy_host = stripped["expected_proxy_host"]

    if proxy_host is not None:
        components = urlparse.urlparse(proxy_host)

        if components.username:
            assert components.username == obfuscated
        if components.password:
            assert components.password == obfuscated

    assert proxy_host == expected_proxy_host
Exemplo n.º 6
0
def test_full_uri_protocol_lifecycle():
    """Exercises the following endpoints:

    * preconnect
    * connect
    * shutdown
    """
    initialize_agent(app_name='Python Agent Test (test_full_uri_payloads)',
                     default_settings=_default_settings)

    environment = ()
    linked_apps = []

    session = FullURIApplicationSession.create_session(
        None,
        global_settings().app_name, linked_apps, environment,
        global_settings_dump())
    session.shutdown_session()
Exemplo n.º 7
0
def create_session(license_key, app_name, linked_applications, environment,
                   settings):

    _global_settings = global_settings()

    if _global_settings.developer_mode:
        session = DeveloperModeSession.create_session(license_key, app_name,
                                                      linked_applications,
                                                      environment, settings)
    else:
        session = ApplicationSession.create_session(license_key, app_name,
                                                    linked_applications,
                                                    environment, settings)

    # When session creation is unsucessful None is returned. We need to catch
    # that and return None. Session creation can fail if data-collector is down
    # or if the configuration is wrong, such as having the capture_params true
    # in high security mode.

    if session is None:
        return None

    # We now need to send up the final merged configuration using the
    # agent_settings() method. We must make sure we pass the
    # configuration through global_settings_dump() to strip/mask any
    # sensitive settings. We also convert values which are strings or
    # numerics to strings before sending to avoid problems with UI
    # interpreting the values strangely if sent as native types.

    application_settings = global_settings_dump(session.configuration)

    for key, value in list(six.iteritems(application_settings)):
        if not isinstance(key, six.string_types):
            del application_settings[key]

        if (not isinstance(value, six.string_types)
                and not isinstance(value, float)
                and not isinstance(value, six.integer_types)):
            application_settings[key] = repr(value)

    session.agent_settings(application_settings)

    return session
Exemplo n.º 8
0
def create_session(license_key, app_name, linked_applications,
        environment, settings):

    _global_settings = global_settings()

    if _global_settings.developer_mode:
        session = DeveloperModeSession.create_session(license_key, app_name,
                linked_applications, environment, settings)
    else:
        session = ApplicationSession.create_session(license_key, app_name,
                linked_applications, environment, settings)

    # When session creation is unsucessful None is returned. We need to catch
    # that and return None. Session creation can fail if data-collector is down
    # or if the configuration is wrong, such as having the capture_params true
    # in high security mode.

    if session is None:
        return None

    # We now need to send up the final merged configuration using the
    # agent_settings() method. We must make sure we pass the
    # configuration through global_settings_dump() to strip/mask any
    # sensitive settings. We also convert values which are strings or
    # numerics to strings before sending to avoid problems with UI
    # interpreting the values strangely if sent as native types.

    application_settings = global_settings_dump(session.configuration)

    for key, value in list(six.iteritems(application_settings)):
        if not isinstance(key, six.string_types):
            del application_settings[key]

        if (not isinstance(value, six.string_types) and
                not isinstance(value, float) and
                not isinstance(value, six.integer_types)):
            application_settings[key] = repr(value)

    session.agent_settings(application_settings)

    return session
Exemplo n.º 9
0
def create_session(license_key, app_name, linked_applications,
        environment, settings):

    _global_settings = global_settings()

    if _global_settings.developer_mode:
        session = DeveloperModeSession.create_session(license_key, app_name,
                linked_applications, environment, settings)
    else:
        session = ApplicationSession.create_session(license_key, app_name,
                linked_applications, environment, settings)

    # When session creation is unsucessful None is returned. We need to catch
    # that and return None. Session creation can fail if data-collector is down
    # or if the configuration is wrong, such as having the capture_params true
    # in high security mode.

    if session is None:
        return None

    # We now need to send up the final merged configuration using the
    # agent_settings() method. We must make sure we pass the
    # configuration through global_settings_dump() to strip/mask any
    # sensitive settings. We also convert values which are strings or
    # numerics to strings before sending to avoid problems with UI
    # interpreting the values strangely if sent as native types.

    application_settings = global_settings_dump(session.configuration)

    for key, value in list(six.iteritems(application_settings)):
        if not isinstance(key, six.string_types):
            del application_settings[key]

        if (not isinstance(value, six.string_types) and
                not isinstance(value, float) and
                not isinstance(value, six.integer_types)):
            application_settings[key] = repr(value)

    try:
        session.agent_settings(application_settings)

    except NetworkInterfaceException:
        # The reason for errors of this type have already been logged.
        # No matter what the error we just pass back None. The upper
        # layer will deal with not being successful.

        _logger.warning('Agent registration failed due to error in '
                'uploading agent settings. Registration should retry '
                'automatically.')

        pass

    except Exception:
        # Any other errors are going to be unexpected and likely will
        # indicate an issue with the implementation of the agent.

        _logger.exception('Unexpected exception when attempting to '
                'update agent settings with the data collector. Please '
                'report this problem to New Relic support for further '
                'investigation.')

        _logger.warning('Agent registration failed due to error in '
                'uploading agent settings. Registration should retry '
                'automatically.')

        pass

    else:
        return session
Exemplo n.º 10
0
    def _connect_payload(app_name, linked_applications, environment, settings):
        settings = global_settings_dump(settings)
        app_names = [app_name] + linked_applications

        hostname = system_info.gethostname(
            settings["heroku.use_dyno_names"],
            settings["heroku.dyno_name_prefixes_to_shorten"],
        )

        ip_address = system_info.getips()

        connect_settings = {}
        connect_settings["browser_monitoring.loader"] = settings[
            "browser_monitoring.loader"
        ]
        connect_settings["browser_monitoring.debug"] = settings[
            "browser_monitoring.debug"
        ]

        security_settings = {}
        security_settings["capture_params"] = settings["capture_params"]
        security_settings["transaction_tracer"] = {}
        security_settings["transaction_tracer"]["record_sql"] = settings[
            "transaction_tracer.record_sql"
        ]

        utilization_settings = {}
        # metadata_version corresponds to the utilization spec being used.
        utilization_settings["metadata_version"] = 5
        utilization_settings[
            "logical_processors"
        ] = system_info.logical_processor_count()
        utilization_settings["total_ram_mib"] = system_info.total_physical_memory()
        utilization_settings["hostname"] = hostname
        if ip_address:
            utilization_settings["ip_address"] = ip_address

        boot_id = system_info.BootIdUtilization.detect()
        if boot_id:
            utilization_settings["boot_id"] = boot_id

        utilization_conf = {}
        logical_processor_conf = settings["utilization.logical_processors"]
        total_ram_conf = settings["utilization.total_ram_mib"]
        hostname_conf = settings["utilization.billing_hostname"]
        if logical_processor_conf:
            utilization_conf["logical_processors"] = logical_processor_conf
        if total_ram_conf:
            utilization_conf["total_ram_mib"] = total_ram_conf
        if hostname_conf:
            utilization_conf["hostname"] = hostname_conf
        if utilization_conf:
            utilization_settings["config"] = utilization_conf

        vendors = []
        if settings["utilization.detect_aws"]:
            vendors.append(AWSUtilization)
        if settings["utilization.detect_pcf"]:
            vendors.append(PCFUtilization)
        if settings["utilization.detect_gcp"]:
            vendors.append(GCPUtilization)
        if settings["utilization.detect_azure"]:
            vendors.append(AzureUtilization)

        utilization_vendor_settings = {}
        for vendor in vendors:
            metadata = vendor.detect()
            if metadata:
                utilization_vendor_settings[vendor.VENDOR_NAME] = metadata
                break

        if settings["utilization.detect_docker"]:
            docker = DockerUtilization.detect()
            if docker:
                utilization_vendor_settings["docker"] = docker

        if settings["utilization.detect_kubernetes"]:
            kubernetes = KubernetesUtilization.detect()
            if kubernetes:
                utilization_vendor_settings["kubernetes"] = kubernetes

        if utilization_vendor_settings:
            utilization_settings["vendors"] = utilization_vendor_settings

        display_host = settings["process_host.display_name"]
        if display_host is None:
            display_host = hostname

        metadata = {}
        for env_var in os.environ:
            if env_var.startswith("NEW_RELIC_METADATA_"):
                metadata[env_var] = os.environ[env_var]

        return (
            {
                "host": hostname,
                "pid": os.getpid(),
                "language": "python",
                "app_name": app_names,
                "identifier": ",".join(app_names),
                "agent_version": version,
                "environment": environment,
                "metadata": metadata,
                "settings": connect_settings,
                "security_settings": security_settings,
                "utilization": utilization_settings,
                "high_security": settings["high_security"],
                "event_harvest_config": settings["event_harvest_config"],
                "labels": settings["labels"],
                "display_host": display_host,
            },
        )
Exemplo n.º 11
0
def create_session(license_key, app_name, linked_applications,
        environment, settings):

    _global_settings = global_settings()

    if _global_settings.developer_mode:
        session = DeveloperModeSession.create_session(license_key, app_name,
                linked_applications, environment, settings)
    else:
        session = ApplicationSession.create_session(license_key, app_name,
                linked_applications, environment, settings)

    # When session creation is unsuccessful None is returned. We need to catch
    # that and return None. Session creation can fail if data-collector is down
    # or if the configuration is wrong, such as having the capture_params true
    # in high security mode.

    if session is None:
        return None

    # We now need to send up the final merged configuration using the
    # agent_settings() method. We must make sure we pass the
    # configuration through global_settings_dump() to strip/mask any
    # sensitive settings. We also convert values which are strings or
    # numerics to strings before sending to avoid problems with UI
    # interpreting the values strangely if sent as native types.

    application_settings = global_settings_dump(session.configuration)

    for key, value in list(six.iteritems(application_settings)):
        if not isinstance(key, six.string_types):
            del application_settings[key]

        if (not isinstance(value, six.string_types) and
                not isinstance(value, float) and
                not isinstance(value, six.integer_types)):
            application_settings[key] = repr(value)

    try:
        session.agent_settings(application_settings)

    except NetworkInterfaceException:
        # The reason for errors of this type have already been logged.
        # No matter what the error we just pass back None. The upper
        # layer will deal with not being successful.

        _logger.warning('Agent registration failed due to error in '
                'uploading agent settings. Registration should retry '
                'automatically.')

        pass

    except Exception:
        # Any other errors are going to be unexpected and likely will
        # indicate an issue with the implementation of the agent.

        _logger.exception('Unexpected exception when attempting to '
                'update agent settings with the data collector. Please '
                'report this problem to New Relic support for further '
                'investigation.')

        _logger.warning('Agent registration failed due to error in '
                'uploading agent settings. Registration should retry '
                'automatically.')

        pass

    else:
        return session
Exemplo n.º 12
0
    def connect_to_data_collector(self):
        """Performs the actual registration of the application with the
        data collector if no current active session.

        """

        if self._agent_shutdown:
            return

        if self._active_session:
            return

        if self._detect_deadlock:
            imp.acquire_lock()
            self._deadlock_event.set()
            imp.release_lock()

        # Register the application with the data collector. Any errors
        # that occur will be dealt with by create_session(). The result
        # will either be a session object or None. In the event of a
        # failure to register we will try again, gradually backing off
        # for longer and longer periods as we retry. The retry interval
        # will be capped at 300 seconds.

        retries = [(15, False, False), (15, False, False),
                   (30, False, False), (60, True, False),
                   (120, False, False), (300, False, True),]

        try:
            while not self._active_session:

                self._active_session = create_session(None, self._app_name,
                        self.linked_applications, environment_settings(),
                        global_settings_dump())

                # We were successful, but first need to make sure we do
                # not have any problems with the agent normalization
                # rules provided by the data collector. These could blow
                # up when being compiled if the patterns are broken or
                # use text which conflicts with extensions in Python's
                # regular expression syntax.

                if self._active_session:
                    configuration = self._active_session.configuration

                    try:
                        settings = global_settings()

                        if settings.debug.log_normalization_rules:
                            _logger.info('The URL normalization rules for '
                                    '%r are %r.', self._app_name,
                                     configuration.url_rules)
                            _logger.info('The metric normalization rules '
                                    'for %r are %r.', self._app_name,
                                     configuration.metric_name_rules)
                            _logger.info('The transaction normalization '
                                    'rules for %r are %r.', self._app_name,
                                     configuration.transaction_name_rules)

                        self._rules_engine['url'] = RulesEngine(
                                configuration.url_rules)
                        self._rules_engine['metric'] = RulesEngine(
                                configuration.metric_name_rules)
                        self._rules_engine['transaction'] = RulesEngine(
                                configuration.transaction_name_rules)

                    except Exception:
                        _logger.exception('The agent normalization rules '
                                'received from the data collector could not '
                                'be compiled properly by the agent due to a '
                                'syntactical error or other problem. Please '
                                'report this to New Relic support for '
                                'investigation.')

                        # For good measure, in this situation we explicitly
                        # shutdown the session as then the data collector
                        # will record this. Ignore any error from this. Then
                        # we discard the session so we go into a retry loop
                        # on presumption that issue with the URL rules will
                        # be fixed.

                        try:
                            self._active_session.shutdown_session()
                        except Exception:
                            pass

                        self._active_session = None

                # Were we successful. If not go into the retry loop. Log
                # warnings or errors as per schedule associated with the
                # retry intervals.

                if not self._active_session:
                    if retries:
                        timeout, warning, error = retries.pop(0)

                        if warning:
                            _logger.warning('Registration of the application '
                                    '%r with the data collector failed after '
                                    'multiple attempts. Check the prior log '
                                    'entries and remedy any issue as '
                                    'necessary, or if the problem persists, '
                                    'report this problem to New Relic '
                                    'support for further investigation.',
                                    self._app_name)

                        elif error:
                            _logger.error('Registration of the application '
                                    '%r with the data collector failed after '
                                    'further additional attempts. Please '
                                    'report this problem to New Relic support '
                                    'for further investigation.',
                                    self._app_name)

                    else:
                        timeout = 300

                    _logger.debug('Retrying registration of the application '
                            '%r with the data collector after a further %d '
                            'seconds.', self._app_name, timeout)

                    time.sleep(timeout)

                    continue

                # Ensure we have cleared out any cached data from a
                # prior agent run for this application.

                configuration = self._active_session.configuration

                with self._stats_lock:
                    self._stats_engine.reset_stats(configuration)

                with self._stats_custom_lock:
                    self._stats_custom_engine.reset_stats(configuration)

                # Record an initial start time for the reporting period and
                # clear record of last transaction processed.

                self._period_start = time.time()

                self._transaction_count = 0
                self._last_transaction = 0.0

                # Clear any prior count of harvest merges due to failures.

                self._merge_count = 0

                # Flag that the session activation has completed to
                # anyone who has been waiting through calling the
                # wait_for_session_activation() method.

                self._connected_event.set()

        except Exception:
            # If an exception occurs after agent has been flagged to be
            # shutdown then we ignore the error. This is because all
            # sorts of wierd errors could occur when main thread start
            # destroying objects and this background thread to register
            # the application is still running.

            if not self._agent_shutdown:
                _logger.exception('Unexpected exception when registering '
                        'agent with the data collector. If this problem '
                        'persists, please report this problem to New Relic '
                        'support for further investigation.')