def test_tracing_load():
"""
Test that all of the traces sent through the agent get the proper service
correlation datapoint.
"""
port = random.randint(5001, 20000)
with Agent.run(
dedent(
f"""
hostname: "testhost"
writer:
sendTraceHostCorrelationMetrics: true
traceHostCorrelationMetricsInterval: 1s
staleServiceTimeout: 7s
monitors:
- type: trace-forwarder
listenAddress: localhost:{port}
"""
)
) as agent:
assert wait_for(p(tcp_port_open_locally, port)), "trace forwarder port never opened!"
for i in range(0, 100):
spans = _test_trace()
spans[0]["localEndpoint"]["serviceName"] += f"-{i}"
spans[1]["localEndpoint"]["serviceName"] += f"-{i}"
resp = retry_on_ebadf(
lambda: requests.post(
f"http://localhost:{port}/v1/trace",
headers={"Content-Type": "application/json"},
data=json.dumps(spans), # pylint:disable=cell-var-from-loop
)
)()
assert resp.status_code == 200
for i in range(0, 100):
assert wait_for(
p(
has_datapoint,
agent.fake_services,
metric_name="sf.int.service.heartbeat",
dimensions={"sf_hasService": f"myapp-{i}", "host": "testhost"},
)
), "Didn't get host correlation datapoint"
assert wait_for(
p(
has_datapoint,
agent.fake_services,
metric_name="sf.int.service.heartbeat",
dimensions={"sf_hasService": f"file-server-{i}", "host": "testhost"},
)
), "Didn't get host correlation datapoint"
time.sleep(10)
agent.fake_services.reset_datapoints()
assert ensure_never(
p(has_datapoint, agent.fake_services, metric_name="sf.int.service.heartbeat"), timeout_seconds=5
), "Got infra correlation metric when it should have been expired"
def run_pprof(self, profile_type, sample_index=None, unit=""):
sample_index_flag = ""
if sample_index:
sample_index_flag = "-sample_index=" + sample_index
command = (
f'go tool pprof -text -compact_labels -lines {sample_index_flag} -unit "{unit}" '
+ f"{self._base_url}/debug/pprof/{profile_type}")
# pylint: disable=unexpected-keyword-arg
profile_text = retry_on_ebadf(lambda: subprocess.check_output(
command, shell=True, close_fds=False))()
return self._parse_profile(profile_text)
def socat_https_proxy(container, target_host, target_port, source_host,
bind_addr):
cert = "/%s.cert" % source_host
key = "/%s.key" % source_host
socat_bin = DOCKERFILES_DIR / "socat"
stopped = False
socket_path = "/tmp/scratch/%s-%s" % (source_host, container.id[:12])
# Keep the socat instance in the container running across container
# restarts
def keep_running_in_container(cont, sock):
while not stopped:
try:
cont.exec_run([
"socat",
"-v",
"OPENSSL-LISTEN:443,cert=%s,key=%s,verify=0,bind=%s,fork" %
(cert, key, bind_addr),
"UNIX-CONNECT:%s" % sock,
])
except docker.errors.APIError:
print("socat died, restarting...")
time.sleep(0.1)
threading.Thread(target=keep_running_in_container,
args=(container, socket_path),
daemon=True).start()
# pylint: disable=consider-using-with
proc = retry_on_ebadf(lambda: subprocess.Popen(
[
socat_bin, "-v",
"UNIX-LISTEN:%s,fork" % socket_path,
"TCP4:%s:%d" % (target_host, target_port)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=False,
))()
get_local_out = pull_from_reader_in_background(proc.stdout)
try:
yield
finally:
stopped = True
# The socat instance in the container will die with the container
proc.kill()
print(get_local_out())