async def _tr_commit(self, tr):
i_samples = []
g_samples = []
for metric, df in self._tr_dfs.get(tr.tid, {}).items():
i_smpls = df[df.op == 'i']
i_smpls = i_smpls[~i_smpls.t.duplicated(keep='last')]
for index, row in i_smpls.iterrows():
i_samples.append(
Sample(metric=metric, t=row.t, value=row.value_orig))
g_smpls = df[df['op'] == 'g']
g_smpls = g_smpls[~g_smpls.t.duplicated(keep='last')]
if metric in self._hooked:
for index, row in g_smpls.iterrows():
g_samples.append(
(index, Sample(metric=metric, t=row.t,
value=row.value)))
for t, sample in g_samples:
self._store(metric=sample.metric,
t=sample.t,
value=sample.value,
tm=index)
for metric, ranges in self._tr_synced_ranges.get(tr.tid, {}).items():
if metric in self._hooked:
for r in ranges:
self._add_synced_range(metric, r['t'], r['its'], r['ets'])
if len(i_samples) > 0:
await prproc.send_samples(i_samples, irt=tr.irt)
items = [
s.metric for s in i_samples if
isinstance(s.metric, Datasource) and s.metric.supplies != None
]
info_metrics = set()
for m in items:
if not m in self._metrics_info or sorted(
m.supplies) != self._metrics_info[m]['supplies']:
info_metrics.add(m)
info_metrics = list(info_metrics)
if len(info_metrics) > 0:
await prproc.send_info(list(info_metrics), irt=tr.irt)
for m in info_metrics:
try:
self._metrics_info[m]['supplies'] = sorted(m.supplies)
except KeyError:
self._metrics_info[m] = {
'supplies': sorted(m.supplies)
}
async def send_stdin(s, uri):
data = sys.stdin.read()
sample = Sample(metric=Datasource(uri, session=s),
t=TimeUUID(),
value=data)
await s.login()
result = await prproc.send_samples([sample])
if not result['success']:
sys.stderr.write('Error sending data to Komlog.\n')
for err in result['errors']:
sys.stderr.write(str(err['error']) + '\n')
await s.close()
def insert(self, metric, t, value):
sample = Sample(metric=metric, t=t, value=value)
tr = asyncio.Task.current_task().get_tr()
if tr:
tid = tr.tid
self._store(sample.metric,
sample.t,
sample.value,
tm=time.monotonic(),
op='i',
tid=tid)
tr.add_dirty_item(self)
else:
self._store(sample.metric,
sample.t,
sample.value,
tm=time.monotonic())
async def test_process_message_send_data_interval_success_dp_data(self):
''' process_message_send_data_interval should store contents in session store '''
username = '******'
privkey = crypto.generate_rsa_key()
uri = 'my_datapoint'
m_type = Metrics.DATAPOINT
start = TimeUUID(1)
end = TimeUUID(3000)
data_json = json.dumps([(TimeUUID(i).hex, i) for i in range(1, 100)])
data = json.loads(data_json)
session = KomlogSession(username=username, privkey=privkey)
msg = messages.SendDataInterval(uri, m_type, start, end, data)
self.assertIsNone(
prmsg.process_message_send_data_interval(msg, session))
for d in data:
smp = Sample(Datapoint(uri, session), TimeUUID(s=d[0]), d[1])
self.assertTrue(session.store.is_in(smp.metric, smp.t, smp.value))
sessionIndex.unregister_session(session.sid)
async def _request_data_range(self, metric, its, ets, count):
response = await prproc.request_data(metric, its, ets, count)
tr = asyncio.Task.current_task().get_tr()
if tr:
tr.add_dirty_item(self)
tid = tr.tid
op = 'g'
else:
tid = None
op = None
d = response['data']
for r in d:
sample = Sample(metric, r[0], r[1])
self._store(sample.metric,
sample.t,
sample.value,
tm=time.monotonic(),
op=op,
tid=tid)
if count != None and count > 0 and len(d) == count:
its = min(r[0] for r in d)
ets = max(r[0] for r in d)
else:
if its == None:
if len(d) > 0 and count == None:
its = min(r[0] for r in d)
else:
its = timeuuid.MIN_TIMEUUID
if ets == None:
if len(d) > 0 and count == None:
ets = max(r[0] for r in d)
else:
ets = timeuuid.MAX_TIMEUUID
if its and ets:
self._add_synced_range(metric, time.monotonic(), its, ets, tid)
return {'count': len(d)}
async def test_process_message_send_multi_data_success_no_notify_already_stored_values(
self):
''' process_message_send_multi_data should store contents and notify tmIndex '''
try:
username = '******'
privkey = crypto.generate_rsa_key()
t = TimeUUID()
uris = [{
'uri': 'datasource1',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource2',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource3',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource4',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datapoint1',
'type': Metrics.DATAPOINT.value,
'content': '1232'
}, {
'uri': 'datapoint2',
'type': Metrics.DATAPOINT.value,
'content': '1233'
}, {
'uri': 'datapoint3',
'type': Metrics.DATAPOINT.value,
'content': '1234'
}, {
'uri': 'datapoint4',
'type': Metrics.DATAPOINT.value,
'content': '1235'
}]
msg = messages.SendMultiData(t, uris)
session = KomlogSession(username=username, privkey=privkey)
bck = tmIndex.metrics_updated
tmIndex.metrics_updated = Mock(return_value=None)
self.assertIsNone(
prmsg.process_message_send_multi_data(msg, session))
metrics = [
Datasource('datasource1', session=session),
Datasource('datasource2', session=session),
Datasource('datasource3', session=session),
Datasource('datasource4', session=session),
Datapoint('datapoint1', session=session),
Datapoint('datapoint2', session=session),
Datapoint('datapoint3', session=session),
Datapoint('datapoint4', session=session)
]
for uri in uris:
if uri['type'] == Metrics.DATASOURCE.value:
smp = Sample(Datasource(uri['uri'], session=session), t,
uri['content'])
else:
smp = Sample(Datapoint(uri['uri'], session=session), t,
uri['content'])
self.assertTrue(
session.store.is_in(smp.metric, smp.t, smp.value))
self.assertEqual(tmIndex.metrics_updated.call_args[1]['t'], t)
self.assertEqual(tmIndex.metrics_updated.call_args[1]['metrics'],
metrics)
updated_uris = [{
'uri': 'datasource1',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource2',
'type': Metrics.DATASOURCE.value,
'content': 'ds data2'
}, {
'uri': 'datasource3',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource4',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource5',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datapoint1',
'type': Metrics.DATAPOINT.value,
'content': '1233'
}, {
'uri': 'datapoint2',
'type': Metrics.DATAPOINT.value,
'content': '1234'
}, {
'uri': 'datapoint3',
'type': Metrics.DATAPOINT.value,
'content': '1234'
}, {
'uri': 'datapoint4',
'type': Metrics.DATAPOINT.value,
'content': '1235'
}, {
'uri': 'datapoint5',
'type': Metrics.DATAPOINT.value,
'content': '1235'
}]
msg = messages.SendMultiData(t, updated_uris)
updated_metrics = [
Datasource('datasource2', session=session),
Datasource('datasource5', session=session),
Datapoint('datapoint1', session=session),
Datapoint('datapoint2', session=session),
Datapoint('datapoint5', session=session)
]
self.assertIsNone(
prmsg.process_message_send_multi_data(msg, session))
for uri in updated_uris:
if uri['type'] == Metrics.DATASOURCE.value:
smp = Sample(Datasource(uri['uri']), t, uri['content'])
else:
smp = Sample(Datapoint(uri['uri']), t, uri['content'])
self.assertTrue(
session.store.is_in(smp.metric, smp.t, smp.value))
self.assertEqual(tmIndex.metrics_updated.call_args[1]['t'], t)
self.assertEqual(tmIndex.metrics_updated.call_args[1]['metrics'],
updated_metrics)
sessionIndex.unregister_session(session.sid)
tmIndex.metrics_updated = bck
except:
tmIndex.metrics_updated = bck
raise
async def test_send_samples_success(self):
''' send_samples should create messages and send through the sessions to Komlog '''
username1 = 'username1'
privkey1 = crypto.generate_rsa_key()
username2 = 'username2'
privkey2 = crypto.generate_rsa_key()
session1 = KomlogSession(username=username1, privkey=privkey1)
session2 = KomlogSession(username=username2, privkey=privkey2)
session1.send_message = test.AsyncMock(return_value=None)
session2.send_message = test.AsyncMock(return_value=None)
t_common = TimeUUID()
t1 = TimeUUID()
t2 = TimeUUID()
samples_s1 = [
Sample(Datasource('datasource1', session=session1), t_common,
'value'),
Sample(Datasource('datasource2', session=session1), t_common,
'value'),
Sample(Datasource('datasource3', session=session1), t_common,
'value'),
Sample(Datasource('datasource4', session=session1), t_common,
'value'),
Sample(Datapoint('datapoint1', session=session1), t_common, 1),
Sample(Datapoint('datapoint2', session=session1), t_common, 1),
Sample(Datapoint('datapoint3', session=session1), t_common, 1),
Sample(Datapoint('datapoint4', session=session1), t_common, 1),
Sample(Datasource('datasource5', session=session1), t1, 'value'),
Sample(Datapoint('datapoint5', session=session1), t2, 1),
]
samples_s2 = [
Sample(Datasource('datasource1', session=session2), t_common,
'value'),
Sample(Datasource('datasource2', session=session2), t_common,
'value'),
Sample(Datasource('datasource3', session=session2), t_common,
'value'),
Sample(Datasource('datasource4', session=session2), t_common,
'value'),
Sample(Datapoint('datapoint1', session=session2), t_common, 1),
Sample(Datapoint('datapoint2', session=session2), t_common, 1),
Sample(Datapoint('datapoint3', session=session2), t_common, 1),
Sample(Datapoint('datapoint4', session=session2), t_common, 1),
Sample(Datasource('datasource5', session=session2), t1, 'value'),
Sample(Datapoint('datapoint5', session=session2), t2, 1),
]
total_samples = []
for smp in samples_s1:
total_samples.append(smp)
for smp in samples_s2:
total_samples.append(smp)
response = await prproc.send_samples(total_samples)
self.assertEqual(session1.send_message.call_count, 3)
self.assertEqual(session2.send_message.call_count, 3)
self.assertEqual(response['success'], False)
self.assertEqual(len(response['errors']), 6)
for i, m in enumerate(response['errors']):
self.assertEqual(m['success'], False)
self.assertEqual(m['error'], 'Unexpected message type')
msg = m['msg']
if i == 0:
self.assertTrue(isinstance(msg, messages.SendMultiData))
self.assertEqual(msg.t, t_common)
self.assertEqual(msg.uris, [{
'uri': s.metric.uri,
'type': s.metric._m_type_,
'content': s.value
} for s in samples_s1[:-2]])
elif i == 1:
self.assertTrue(isinstance(msg, messages.SendDsData))
self.assertEqual(msg.uri, samples_s1[-2].metric.uri)
self.assertEqual(msg.t, samples_s1[-2].t)
self.assertEqual(msg.content, samples_s1[-2].value)
elif i == 2:
self.assertTrue(isinstance(msg, messages.SendDpData))
self.assertEqual(msg.uri, samples_s1[-1].metric.uri)
self.assertEqual(msg.t, samples_s1[-1].t)
self.assertEqual(msg.content, samples_s1[-1].value)
elif i == 3:
self.assertTrue(isinstance(msg, messages.SendMultiData))
self.assertEqual(msg.t, t_common)
self.assertEqual(msg.uris, [{
'uri': s.metric.uri,
'type': s.metric._m_type_,
'content': s.value
} for s in samples_s2[:-2]])
elif i == 4:
self.assertTrue(isinstance(msg, messages.SendDsData))
self.assertEqual(msg.uri, samples_s2[-2].metric.uri)
self.assertEqual(msg.t, samples_s2[-2].t)
self.assertEqual(msg.content, samples_s2[-2].value)
elif i == 5:
self.assertTrue(isinstance(msg, messages.SendDpData))
self.assertEqual(msg.uri, samples_s2[-1].metric.uri)
self.assertEqual(msg.t, samples_s2[-1].t)
self.assertEqual(msg.content, samples_s2[-1].value)
sessionIndex.unregister_session(session1.sid)
sessionIndex.unregister_session(session2.sid)