async def test_send_metrics_with_same_ts(tcp_server, tcp_client, timestamp):
tcp_client._storage = RawStorage()
metrics = [
Metric(name='foo', value=44, timestamp=timestamp - 1),
Metric(name='foo', value=11, timestamp=timestamp),
Metric(name='foo', value=7, timestamp=timestamp),
Metric(name='foo', value=24, timestamp=timestamp)
]
for metric in metrics:
tcp_client.add(metric)
await asyncio.sleep(2)
await tcp_server.wait_data()
lines = list(filter(None, tcp_server.data.split(b"\n")))
assert len(lines) == 4
for metric in metrics:
line = lines.pop(0)
name, value, ts = line.decode().strip().split(" ")
assert name == metric.name
assert int(value) == metric.value
assert int(ts) == metric.timestamp
async def test_accepts_metrics(proxy_carbon_client, storage):
metrics = [
Metric('foo.bar.spam', 42, 1548934966),
Metric('foo.bar.spam', 33, 1548934967),
Metric('foo.bar.spam', 55, 1548934968)
]
for metric in metrics:
proxy_carbon_client.add(metric)
await asyncio.sleep(1)
assert storage.mock_storage == [
('foo.bar.spam', (1548934966.0, 42)),
('foo.bar.spam', (1548934967.0, 33)),
('foo.bar.spam', (1548934968.0, 55)),
]
async def test_udp_simple(event_loop: asyncio.AbstractEventLoop,
unused_tcp_port):
protocol = UDPServerProtocol()
await event_loop.create_datagram_endpoint(
lambda: protocol,
local_addr=('127.0.0.1', unused_tcp_port)
)
client = UDPClient("127.0.0.1", port=unused_tcp_port, namespace='',
storage=TotalStorage())
task = event_loop.create_task(client.run())
now = time.time()
metric = Metric(name='foo', value=42, timestamp=now)
client.add(metric)
data = await protocol.queue.get()
lines = list(map(lambda x: x.decode(), filter(None, data.split(b'\n'))))
assert len(lines) == 1
for line in lines:
name, value, ts = line.split(' ')
value = float(value)
ts = float(ts)
assert name == 'foo'
assert value == 42
assert ts < time.time()
task.cancel()
await asyncio.wait([task])
async def test_pickle_many(event_loop, unused_tcp_port):
client = PickleClient('127.0.0.1', port=unused_tcp_port, namespace='')
count = 9991
now = time.time() - 1
for i in range(count):
metric = Metric(name='foo', value=i, timestamp=now - i)
client.add(metric)
data = list()
event = asyncio.Event(loop=event_loop)
async def handler(reader: asyncio.StreamReader,
writer: asyncio.StreamWriter):
nonlocal data
header_size = struct.calcsize('!L')
while not reader.at_eof():
try:
header = await reader.readexactly(header_size)
except asyncio.IncompleteReadError:
break
payload_size = struct.unpack("!L", header)[0]
try:
payload = await reader.readexactly(payload_size)
except asyncio.IncompleteReadError:
break
for metric in pickle.loads(payload):
data.append(metric)
if len(data) == count:
event.set()
writer.close()
reader.feed_eof()
server = await asyncio.start_server(handler,
'127.0.0.1',
unused_tcp_port,
loop=event_loop)
await client.send()
await event.wait()
for idx, metric in enumerate(sorted(data, key=lambda x: x[1][1])):
name, payload = metric
ts, value = payload
assert name == 'foo'
assert value == idx
server.close()
async def test_tcp_simple(tcp_client, tcp_server, event_loop):
metric = Metric(name='foo', value=42)
tcp_client.add(metric)
await tcp_server.wait_data()
name, value, ts = tcp_server.data.decode().strip().split(" ")
assert name == metric.name
assert int(float(value)) == int(metric.value)
assert int(float(ts)) == int(metric.timestamp)
async def test_tcp_many(tcp_client, tcp_server, event_loop):
now = int(time.time()) - 86400
for i in range(199):
metric = Metric(name='foo', value=42, timestamp=now + i)
tcp_client.add(metric)
await tcp_server.wait_data()
for i in range(199):
metric = Metric(name='foo', value=42, timestamp=now - i)
tcp_client.add(metric)
await tcp_server.wait_data()
lines = list(filter(None, tcp_server.data.split(b"\n")))
assert len(lines) == 398
for line in lines:
name, value, ts = line.decode().strip().split(" ")
assert name == 'foo'
assert int(float(value)) == 42
async def test_sum_metrics_with_the_same_ts(tcp_server, tcp_client, timestamp):
for val in [11, 7, 24]:
metric = Metric(name='foo', value=val, timestamp=timestamp)
tcp_client.add(metric)
await tcp_server.wait_data()
lines = list(filter(None, tcp_server.data.split(b"\n")))
assert len(lines) == 1
line = lines[0]
name, value, ts = line.decode().strip().split(" ")
assert name == 'foo'
assert int(value) == 42
assert int(ts) == timestamp
def __iter__(self) -> AsyncIterable[Metric]:
current_time = int(time.time())
for name, metrics in self._metrics.items(): # type: Counter
returning = list()
while metrics:
ts, value = metrics.popitem()
if ts >= current_time:
returning.append((ts, value))
continue
yield Metric(name=name, timestamp=ts, value=value)
for ts, value in returning:
metrics[ts] += value
async def test_udp_many(event_loop: asyncio.AbstractEventLoop,
unused_tcp_port):
count = 99991
protocol = UDPServerProtocol()
await event_loop.create_datagram_endpoint(
lambda: protocol,
local_addr=('127.0.0.1', unused_tcp_port)
)
client = UDPClient("127.0.0.1", port=unused_tcp_port, namespace='',
storage=TotalStorage())
now = time.time() - 5
for i in range(count):
metric = Metric(name='foo', value=i, timestamp=now - i)
client.add(metric)
await client.send()
await asyncio.sleep(0.1)
protocol.close()
data = b''
chunk = await protocol.queue.get()
while chunk is not None:
data += chunk
chunk = await protocol.queue.get()
lines = list(map(lambda x: x.decode(), filter(None, data.split(b'\n'))))
assert len(lines) == count
lines = sorted(
map(lambda x: x.split(' '), lines),
key=lambda x: int(x[1]))
for idx, (name, value, ts) in enumerate(lines):
value = float(value)
ts = float(ts)
assert name == 'foo'
assert value == idx
assert ts < time.time()
def __iter__(self) -> AsyncIterable[Metric]:
current_time = int(time.time())
for name, metrics in self._metrics.items(): # type: Counter
returning = list()
for ts, value in metrics:
if ts >= current_time:
returning.append((ts, value))
continue
yield Metric(name=name, timestamp=ts, value=value)
metrics.clear()
if not returning:
continue
metrics.extend(returning)
log.info(
"%d metric(s) of %r weren't sent yet because they're "
"newer than the current timestamp %d", len(returning), name,
current_time)
async def test_tcp_reconnect(event_loop: asyncio.AbstractEventLoop,
unused_tcp_port):
async def handler(reader, writer):
await reader.read(10)
writer.close()
reader.feed_eof()
server = await asyncio.start_server(
handler, '127.0.0.1', unused_tcp_port, loop=event_loop
)
client = TCPClient('127.0.0.1', port=unused_tcp_port, namespace='')
count = 19907
now = time.time() - 1
for i in range(count):
metric = Metric(name='foo', value=i, timestamp=now - i)
client.add(metric)
server.close()
await server.wait_closed()
with pytest.raises(ConnectionError):
await client.send()
event = asyncio.Event(loop=event_loop)
data = b''
async def handler(reader, writer):
nonlocal data
while not reader.at_eof():
try:
data += await reader.read(1024)
except:
break
try:
data += await reader.read(1024)
except:
pass
event.set()
server = await asyncio.start_server(
handler, '127.0.0.1', unused_tcp_port, loop=event_loop
)
await client.send()
await event.wait()
server.close()
await server.wait_closed()
lines = sorted(
map(
lambda x: x.split(' '),
filter(None, map(lambda x: x.decode(), data.split(b"\n")))
),
key=lambda x: int(x[1])
)
for idx, (name, value, ts) in enumerate(lines):
value = float(value)
assert name == 'foo'
assert idx == value
assert len(lines) == count