def test_finalize_application_settings(server_settings):
settings = global_settings()
finalize_application_settings(server_side_config=server_settings,
settings=settings)
# hostname set in ini_file and not in env vars
assert settings.utilization.billing_hostname == 'file-hostname'
def test_status_code_exceptions(status_code, expected_exc, log_level, caplog):
caplog.set_level(logging.INFO)
HttpClientRecorder.STATUS_CODE = status_code
settings = finalize_application_settings({
"license_key": "123LICENSEKEY",
})
protocol = AgentProtocol(settings, client_cls=HttpClientRecorder)
internal_metrics = CustomMetrics()
with InternalTraceContext(internal_metrics):
with pytest.raises(expected_exc):
protocol.send("analytic_event_data")
internal_metrics = dict(internal_metrics.metrics())
if status_code == 413:
assert internal_metrics[
"Supportability/Python/Collector/MaxPayloadSizeLimit/analytic_event_data"] == [
1, 0, 0, 0, 0, 0
]
else:
assert (
"Supportability/Python/Collector/MaxPayloadSizeLimit/analytic_event_data"
not in internal_metrics)
assert len(HttpClientRecorder.SENT) == 1
request = HttpClientRecorder.SENT[0]
assert request.params["method"] == "analytic_event_data"
assert len(caplog.records) == 1
assert caplog.records[0].levelname == log_level
message = caplog.records[0].getMessage()
assert "123LICENSEKEY" not in message
def test_protocol_context_manager():
protocol = AgentProtocol(finalize_application_settings(),
client_cls=HttpClientRecorder)
with protocol:
assert HttpClientRecorder.STATE == 1
assert HttpClientRecorder.STATE == 0
def test_max_payload_size_limit():
settings = finalize_application_settings(
{"max_payload_size_in_bytes": 0, "port": -1}
)
protocol = AgentProtocol(settings, host="localhost")
with pytest.raises(DiscardDataForRequest):
protocol.send("metric_data")
def test_audit_logging():
with tempfile.NamedTemporaryFile(delete=False) as f:
f.write(b"*\n")
settings = finalize_application_settings({"audit_log_file": f.name})
protocol = AgentProtocol(settings, client_cls=HttpClientRecorder)
protocol.send("preconnect")
with open(f.name) as f:
audit_log_contents = f.read()
assert audit_log_contents.startswith("*\n")
assert len(audit_log_contents) > 2
def test_connect_metadata(monkeypatch):
monkeypatch.setenv("NEW_RELIC_METADATA_FOOBAR", "foobar")
monkeypatch.setenv("_NEW_RELIC_METADATA_WRONG", "wrong")
protocol = AgentProtocol.connect(
APP_NAME,
LINKED_APPS,
ENVIRONMENT,
finalize_application_settings(),
client_cls=HttpClientRecorder,
)
connect = HttpClientRecorder.SENT[1]
assert connect.params["method"] == "connect"
connect_payload = json_decode(connect.payload.decode("utf-8"))[0]
assert connect_payload["metadata"] == {"NEW_RELIC_METADATA_FOOBAR": "foobar"}
def test_ca_bundle_path(monkeypatch, ca_bundle_path):
# Pretend CA certificates are not available
class DefaultVerifyPaths(object):
cafile = None
def __init__(self, *args, **kwargs):
pass
monkeypatch.setattr(ssl, "DefaultVerifyPaths", DefaultVerifyPaths)
settings = finalize_application_settings({"ca_bundle_path": ca_bundle_path})
protocol = AgentProtocol(settings)
expected = ca_bundle_path or certs.where()
assert protocol.client._connection_kwargs["ca_certs"] == expected
def connect(
cls,
app_name,
linked_applications,
environment,
settings,
client_cls=ServerlessModeClient,
):
aws_lambda_metadata = settings.aws_lambda_metadata
settings = finalize_application_settings(
{"cross_application_tracer.enabled": False}, settings)
# Metadata must come from the original settings object since it
# can be modified later
settings.aws_lambda_metadata = aws_lambda_metadata
return cls(settings, client_cls=client_cls)
def connect(
cls,
app_name,
linked_applications,
environment,
settings,
client_cls=ApplicationModeClient,
):
with cls(settings, client_cls=client_cls) as preconnect:
redirect_host = preconnect.send("preconnect")["redirect_host"]
with cls(settings, host=redirect_host,
client_cls=client_cls) as protocol:
configuration = protocol.send(
"connect",
cls._connect_payload(app_name, linked_applications,
environment, settings),
)
# Apply High Security Mode to server_config, so the local
# security settings won't get overwritten when we overlay
# the server settings on top of them.
configuration = cls._apply_high_security_mode_fixups(
configuration, settings)
# The agent configuration for the application in constructed
# by taking a snapshot of the locally constructed
# configuration and overlaying it with that from the server,
# as well as creating the attribute filter.
settings = finalize_application_settings(configuration, settings)
with cls(settings, host=redirect_host,
client_cls=client_cls) as protocol:
protocol.send(
"agent_settings",
(global_settings_dump(settings, serializable=True), ))
if "messages" in configuration:
for item in configuration["messages"]:
message = item["message"]
level = item["level"]
logger_func = cls.LOGGER_FUNC_MAPPING.get(level, None)
if logger_func:
logger_func("%s", message)
return protocol
def test_serverless_protocol_finalize(capsys):
protocol = ServerlessModeProtocol(
finalize_application_settings(
{"aws_lambda_metadata": {"foo": "bar", "agent_version": "x"}}
)
)
response = protocol.send("metric_data", (1, 2, 3))
assert response is None
payload = protocol.finalize()
captured = capsys.readouterr()
assert captured.out.rstrip("\n") == payload
payload = json_decode(payload)
assert payload[:2] == [1, "NR_LAMBDA_MONITORING"]
data = serverless_payload_decode(payload[2])
assert data["data"] == {"metric_data": [1, 2, 3]}
assert data["metadata"]["foo"] == "bar"
assert data["metadata"]["agent_version"] != "x"
def session():
initialize_agent(app_name='Python Agent Test (test_max_payload_size)',
default_settings=_default_settings)
session = ApplicationSession("https://collector.newrelic.com", None,
finalize_application_settings())
# patch in application sesssion requests
original_request = session.requests_session.request
assert original_request
def request(*args, **kwargs):
assert False, "Outbound request made"
# Disable all requests
session.requests_session.request = request
yield session
# Re-enable requests
session.requests_session.request = original_request
def test_send(status_code):
HttpClientRecorder.STATUS_CODE = status_code
settings = finalize_application_settings({
"request_headers_map": {
"custom-header": u"value"
},
"agent_run_id": "RUN_TOKEN",
})
protocol = AgentProtocol(settings, client_cls=HttpClientRecorder)
response = protocol.send("metric_data", (1, 2, 3))
assert response is None
assert len(HttpClientRecorder.SENT) == 1
request = HttpClientRecorder.SENT[0]
assert request.method == "POST"
assert request.path == "/agent_listener/invoke_raw_method"
# Verify license key was there, but no way to validate the value
request.params.pop("license_key")
assert request.params == {
"method": "metric_data",
"marshal_format": "json",
"protocol_version": AgentProtocol.VERSION,
"run_id": "RUN_TOKEN",
}
assert request.headers == {
"Content-Type": "application/json",
"custom-header": u"value",
}
assert request.payload == b"[1,2,3]"
# Verify call to finalize is None
assert protocol.finalize() is None
def create_session(cls, license_key, app_name, linked_applications,
environment, settings):
"""Registers the agent for the specified application with the data
collector and retrieves the server side configuration. Returns a
session object if successful through which subsequent calls to the
data collector are made. If unsuccessful then None is returned.
"""
start = time.time()
# If no license key provided in the call, fall back to using that
# from the agent configuration file or environment variables.
# Flag an error if the result still seems invalid.
if not license_key:
license_key = global_settings().license_key
if not license_key:
_logger.error('A valid account license key cannot be found. '
'Has a license key been specified in the agent configuration '
'file or via the NEW_RELIC_LICENSE_KEY environment variable?')
try:
# First need to ask the primary data collector which of the many
# data collector instances we should use for this agent run.
_logger.debug('Connecting to data collector to register agent '
'with license_key=%r, app_name=%r, '
'linked_applications=%r, environment=%r and settings=%r.',
license_key, app_name, linked_applications, environment,
settings)
url = collector_url()
redirect_host = cls.send_request(None, url,
'get_redirect_host', license_key)
# Then we perform a connect to the actual data collector host
# we need to use. All communications after this point should go
# to the secondary data collector.
#
# We use the global requests session object for now as harvest
# for different applications are all done in turn. We will need
# to change this if use multiple threads as currently force
# session object to maintain only single connection to ensure
# that keep alive is effective.
payload = cls._create_connect_payload(app_name,
linked_applications, environment, settings)
url = collector_url(redirect_host)
server_config = cls.send_request(None, url, 'connect',
license_key, None, payload)
# Apply High Security Mode to server_config, so the local
# security settings won't get overwritten when we overlay
# the server settings on top of them.
server_config = apply_high_security_mode_fixups(settings,
server_config)
# The agent configuration for the application in constructed
# by taking a snapshot of the locally constructed
# configuration and overlaying it with that from the server,
# as well as creating the attribute filter.
application_config = finalize_application_settings(server_config)
except NetworkInterfaceException:
# The reason for errors of this type have already been logged.
# No matter what the error we just pass back None. The upper
# layer needs to count how many success times this has failed
# and escalate things with a more sever error.
pass
except Exception:
# Any other errors are going to be unexpected and likely will
# indicate an issue with the implementation of the agent.
_logger.exception('Unexpected exception when attempting to '
'register the agent with the data collector. Please '
'report this problem to New Relic support for further '
'investigation.')
pass
else:
# Everything fine so we create the session object through which
# subsequent communication with data collector will be done.
session = cls(url, license_key, application_config)
duration = time.time() - start
# Log successful agent registration and any server side messages.
_logger.info('Successfully registered New Relic Python agent '
'where app_name=%r, pid=%r, redirect_host=%r and '
'agent_run_id=%r, in %.2f seconds.', app_name,
os.getpid(), redirect_host, session.agent_run_id,
duration)
if getattr(application_config, 'high_security', False):
_logger.info('High Security Mode is being applied to all '
'communications between the agent and the data '
'collector for this session.')
logger_func_mapping = {
'ERROR': _logger.error,
'WARN': _logger.warning,
'INFO': _logger.info,
'VERBOSE': _logger.debug,
}
if 'messages' in server_config:
for item in server_config['messages']:
message = item['message']
level = item['level']
logger_func = logger_func_mapping.get(level, None)
if logger_func:
logger_func('%s', message)
return session
def test_connect(with_aws, with_pcf, with_gcp, with_azure, with_docker,
with_kubernetes, with_ip):
global AWS, AZURE, GCP, PCF, BOOT_ID, DOCKER, KUBERNETES, IP_ADDRESS
if not with_aws:
AWS = Exception
if not with_pcf:
PCF = Exception
if not with_gcp:
GCP = Exception
if not with_azure:
AZURE = Exception
if not with_docker:
DOCKER = Exception
if not with_kubernetes:
KUBERNETES = Exception
if not with_ip:
IP_ADDRESS = None
settings = finalize_application_settings({
"browser_monitoring.loader": BROWSER_MONITORING_LOADER,
"browser_monitoring.debug": BROWSER_MONITORING_DEBUG,
"capture_params": CAPTURE_PARAMS,
"process_host.display_name": DISPLAY_NAME,
"transaction_tracer.record_sql": RECORD_SQL,
"high_security": HIGH_SECURITY,
"labels": LABELS,
"utilization.detect_aws": with_aws,
"utilization.detect_pcf": with_pcf,
"utilization.detect_gcp": with_gcp,
"utilization.detect_azure": with_azure,
"utilization.detect_docker": with_docker,
"utilization.detect_kubernetes": with_kubernetes,
"event_harvest_config": {
"harvest_limits": {
"analytic_event_data": ANALYTIC_EVENT_DATA,
"span_event_data": SPAN_EVENT_DATA,
"custom_event_data": CUSTOM_EVENT_DATA,
"error_event_data": ERROR_EVENT_DATA,
}
},
})
protocol = AgentProtocol.connect(
APP_NAME,
LINKED_APPS,
ENVIRONMENT,
settings,
client_cls=HttpClientRecorder,
)
# verify there are exactly 3 calls to HttpClientRecorder
assert len(HttpClientRecorder.SENT) == 3
# Verify preconnect call
preconnect = HttpClientRecorder.SENT[0]
assert preconnect.params["method"] == "preconnect"
assert preconnect.payload == b"[]"
# Verify connect call
connect = HttpClientRecorder.SENT[1]
assert connect.params["method"] == "connect"
connect_payload = json_decode(connect.payload.decode("utf-8"))
connect_payload_asserts(
connect_payload,
with_aws=with_aws,
with_pcf=with_pcf,
with_gcp=with_gcp,
with_azure=with_azure,
with_docker=with_docker,
with_kubernetes=with_kubernetes,
)
# Verify agent_settings call is done with the finalized settings
agent_settings = HttpClientRecorder.SENT[2]
assert agent_settings.params["method"] == "agent_settings"
agent_settings_payload = json_decode(
agent_settings.payload.decode("utf-8"))
assert len(agent_settings_payload) == 1
agent_settings_payload = agent_settings_payload[0]
# Finalized settings will have a non-None agent_run_id
assert agent_settings_payload["agent_run_id"] is not None
assert protocol.configuration.agent_run_id is not None
# Verify that agent settings sent have converted null, containers, and
# unserializable types to string
assert agent_settings_payload["proxy_host"] == "None"
assert agent_settings_payload["attributes.include"] == "[]"
assert agent_settings_payload["feature_flag"] == str(set())
assert isinstance(agent_settings_payload["attribute_filter"],
six.string_types)
# Verify that the connection is closed
assert HttpClientRecorder.STATE == 0
def test_close_connection():
protocol = AgentProtocol(finalize_application_settings(),
client_cls=HttpClientRecorder)
protocol.close_connection()
assert HttpClientRecorder.STATE == -1
self.harvest_flexible += 1
else:
self.harvest_default += 1
if shutdown:
self.is_alive = False
class FakeAgent(Agent):
def __init__(self, *args, **kwargs):
super(FakeAgent, self).__init__(*args, **kwargs)
self._applications = {'fake': FakeApplication()}
SETTINGS = finalize_application_settings({
'enabled': True,
'debug.disable_harvest_until_shutdown': False,
})
@pytest.fixture
def agent():
agent = FakeAgent(SETTINGS)
yield agent
agent.shutdown_agent(timeout=5)
assert not agent._harvest_thread.is_alive()
_override_settings = {
'event_harvest_config.report_period_ms': 80.0 * 1000.0,
}
def transaction_node(request):
default_capacity = SampledDataSet().capacity
num_events = default_capacity + 1
custom_events = SampledDataSet(capacity=num_events)
for _ in range(num_events):
event = create_custom_event("Custom", {})
custom_events.add(event)
error = ErrorNode(
timestamp=0,
type="foo:bar",
message="oh no! your foo had a bar",
expected=False,
span_id=None,
stack_trace="",
custom_params={},
file_name=None,
line_number=None,
source=None,
)
errors = tuple(error for _ in range(num_events))
function = FunctionNode(
group="Function",
name="foo",
children=(),
start_time=0,
end_time=1,
duration=1,
exclusive=1,
label=None,
params=None,
rollup=None,
guid="GUID",
agent_attributes={},
user_attributes={},
)
children = tuple(function for _ in range(num_events))
root = RootNode(
name="Function/main",
children=children,
start_time=1524764430.0,
end_time=1524764430.1,
duration=0.1,
exclusive=0.1,
guid=None,
agent_attributes={},
user_attributes={},
path="OtherTransaction/Function/main",
trusted_parent_span=None,
tracing_vendors=None,
)
node = TransactionNode(
settings=finalize_application_settings({"agent_run_id": "1234567"}),
path="OtherTransaction/Function/main",
type="OtherTransaction",
group="Function",
base_name="main",
name_for_metric="Function/main",
port=None,
request_uri=None,
queue_start=0.0,
start_time=1524764430.0,
end_time=1524764430.1,
last_byte_time=0.0,
total_time=0.1,
response_time=0.1,
duration=0.1,
exclusive=0.1,
root=root,
errors=errors,
slow_sql=(),
custom_events=custom_events,
apdex_t=0.5,
suppress_apdex=False,
custom_metrics=CustomMetrics(),
guid="4485b89db608aece",
cpu_time=0.0,
suppress_transaction_trace=False,
client_cross_process_id=None,
referring_transaction_guid=None,
record_tt=False,
synthetics_resource_id=None,
synthetics_job_id=None,
synthetics_monitor_id=None,
synthetics_header=None,
is_part_of_cat=False,
trip_id="4485b89db608aece",
path_hash=None,
referring_path_hash=None,
alternate_path_hashes=[],
trace_intrinsics={},
distributed_trace_intrinsics={},
agent_attributes=[],
user_attributes=[],
priority=1.0,
parent_transport_duration=None,
parent_span=None,
parent_type=None,
parent_account=None,
parent_app=None,
parent_tx=None,
parent_transport_type=None,
sampled=True,
root_span_guid=None,
trace_id="4485b89db608aece",
loop_time=0.0,
)
return node
def test_protocol_http_error_causes_retry():
protocol = AgentProtocol(finalize_application_settings(),
client_cls=HttpClientException)
with pytest.raises(RetryDataForRequest):
protocol.send("analytic_event_data")
def create_session(cls, license_key, app_name, linked_applications,
environment, settings):
"""Registers the agent for the specified application with the data
collector and retrieves the server side configuration. Returns a
session object if successful through which subsequent calls to the
data collector are made. If unsucessful then None is returned.
"""
start = time.time()
# If no license key provided in the call, fallback to using that
# from the agent configuration file or environment variables.
# Flag an error if the result still seems invalid.
if not license_key:
license_key = global_settings().license_key
if not license_key:
_logger.error('A valid account license key cannot be found. '
'Has a license key been specified in the agent configuration '
'file or via the NEW_RELIC_LICENSE_KEY environment variable?')
try:
# First need to ask the primary data collector which of the many
# data collector instances we should use for this agent run.
_logger.debug('Connecting to data collector to register agent '
'with license_key=%r, app_name=%r, '
'linked_applications=%r, environment=%r and settings=%r.',
license_key, app_name, linked_applications, environment,
settings)
url = collector_url()
with InternalTrace('Supportability/Python/Collector/Calls/'
'get_redirect_host'):
redirect_host = cls.send_request(None, url,
'get_redirect_host', license_key)
# Then we perform a connect to the actual data collector host
# we need to use. All communications after this point should go
# to the secondary data collector.
#
# We use the global requests session object for now as harvest
# for different applications are all done in turn. We will need
# to change this if use multiple threads as currently force
# session object to maintain only single connection to ensure
# that keep alive is effective.
payload = cls._create_connect_payload(app_name,
linked_applications, environment, settings)
url = collector_url(redirect_host)
with InternalTrace('Supportability/Python/Collector/Calls/'
'connect'):
server_config = cls.send_request(None, url, 'connect',
license_key, None, payload)
# Apply High Security Mode to server_config, so the local
# security settings won't get overwritten when we overlay
# the server settings on top of them.
server_config = apply_high_security_mode_fixups(settings,
server_config)
# The agent configuration for the application in constructed
# by taking a snapshot of the locally constructed
# configuration and overlaying it with that from the server,
# as well as creating the attribute filter.
application_config = finalize_application_settings(server_config)
except NetworkInterfaceException:
# The reason for errors of this type have already been logged.
# No matter what the error we just pass back None. The upper
# layer needs to count how many success times this has failed
# and escalate things with a more sever error.
pass
except Exception:
# Any other errors are going to be unexpected and likely will
# indicate an issue with the implementation of the agent.
_logger.exception('Unexpected exception when attempting to '
'register the agent with the data collector. Please '
'report this problem to New Relic support for further '
'investigation.')
pass
else:
# Everything fine so we create the session object through which
# subsequent communication with data collector will be done.
session = cls(url, license_key, application_config)
duration = time.time() - start
# Log successful agent registration and any server side messages.
_logger.info('Successfully registered New Relic Python agent '
'where app_name=%r, pid=%r, redirect_host=%r and '
'agent_run_id=%r, in %.2f seconds.', app_name,
os.getpid(), redirect_host, session.agent_run_id,
duration)
if getattr(application_config, 'high_security', False):
_logger.info('High Security Mode is being applied to all '
'communications between the agent and the data '
'collector for this session.')
logger_func_mapping = {
'ERROR': _logger.error,
'WARN': _logger.warning,
'INFO': _logger.info,
'VERBOSE': _logger.debug,
}
if 'messages' in server_config:
for item in server_config['messages']:
message = item['message']
level = item['level']
logger_func = logger_func_mapping.get(level, None)
if logger_func:
logger_func('%s', message)
return session
