async def find_missing(self, content):
devs = []
fss = []
dev_block = False
fs_block = False
for line in content.split('\n'):
fields = line.split()
if dev_block and fields:
dev_name = fields[0]
final_name = ''
for i,c in enumerate(dev_name):
if validation.is_local_uri(c):
final_name += c
elif i>0:
final_name += '_'
if validation.is_local_uri(final_name):
devs.append(final_name)
elif fs_block and fields:
fs_name = fields[-1]
final_name = ''
for i,c in enumerate(fs_name):
if validation.is_local_uri(c):
final_name += c
elif i>0:
final_name += '_'
if validation.is_local_uri(final_name):
fss.append(final_name)
else:
dev_block = False
fs_block = False
if fields and fields[0][0] == 'D':
dev_block = True
elif fields and fields[0][0:2] == 'MB':
fs_block = True
for dev in devs:
if not dev in self._devs:
logging.logger.debug('New device found: '+dev)
self._devs.append(dev)
for m in self._dev_metrics:
self.info.supplies.append('.'.join((dev,m)))
f_params = {
'util':Datapoint(uri='.'.join((self.info.uri,dev,'util')))
}
tm = transfermethod(f=self.check_anom_dev, f_params=f_params)
await tm.bind()
self.tms.append(tm)
for fs in fss:
if not fs in self._fss:
logging.logger.debug('New filesystem found: '+fs)
self._fss.append(fs)
for m in self._fs_metrics:
self.info.supplies.append('.'.join((fs,m)))
f_params = {
'fsused':Datapoint(uri='.'.join((self.info.uri,fs,'fsused'))),
'ufsused':Datapoint(uri='.'.join((self.info.uri,fs,'ufsused'))),
'pIused':Datapoint(uri='.'.join((self.info.uri,fs,'pIused')))
}
tm = transfermethod(f=self.check_anom_fs, f_params=f_params)
await tm.bind()
self.tms.append(tm)
def test_cpu_creation(self):
''' creating a CPU object should succeed '''
root_uri = 'host.resources'
my_cpu = CPU(root_uri)
self.assertTrue(isinstance(my_cpu.info, Datasource))
self.assertEqual(my_cpu.info.uri, '.'.join((root_uri, 'cpu')))
self.assertTrue('user' in my_cpu.use_metrics)
self.assertTrue('nice' in my_cpu.use_metrics)
self.assertTrue('system' in my_cpu.use_metrics)
self.assertTrue('steal' in my_cpu.use_metrics)
self.assertTrue('iowait' in my_cpu.use_metrics)
self.assertTrue('idle' in my_cpu.use_metrics)
self.assertTrue('cpu_count' in my_cpu.use_metrics)
self.assertTrue('t_runnable' in my_cpu.use_metrics)
self.assertTrue('ldavg1' in my_cpu.use_metrics)
self.assertEqual(my_cpu.use_metrics['user'],
Datapoint('.'.join((my_cpu.info.uri, 'all.user'))))
self.assertEqual(my_cpu.use_metrics['nice'],
Datapoint('.'.join((my_cpu.info.uri, 'all.nice'))))
self.assertEqual(my_cpu.use_metrics['system'],
Datapoint('.'.join((my_cpu.info.uri, 'all.system'))))
self.assertEqual(my_cpu.use_metrics['steal'],
Datapoint('.'.join((my_cpu.info.uri, 'all.steal'))))
self.assertEqual(my_cpu.use_metrics['iowait'],
Datapoint('.'.join((my_cpu.info.uri, 'all.iowait'))))
self.assertEqual(my_cpu.use_metrics['idle'],
Datapoint('.'.join((my_cpu.info.uri, 'all.idle'))))
self.assertEqual(my_cpu.use_metrics['cpu_count'],
Datapoint('.'.join((my_cpu.info.uri, 'cpu_count'))))
self.assertEqual(
my_cpu.use_metrics['t_runnable'],
Datapoint('.'.join((my_cpu.info.uri, 'tasks.runnable'))))
self.assertEqual(my_cpu.use_metrics['ldavg1'],
Datapoint('.'.join((my_cpu.info.uri, 'ldavg.1min'))))
self.assertEqual(len(my_cpu.tms), 1)
def test_memory_creation(self):
''' creating a Memory object should succeed '''
root_uri = 'host.resources'
my_mem = Memory(root_uri)
self.assertTrue(isinstance(my_mem.info, Datasource))
self.assertEqual(my_mem.info.uri, '.'.join((root_uri, 'memory')))
self.assertTrue('memused' in my_mem.use_metrics)
self.assertTrue('majflt' in my_mem.use_metrics)
self.assertEqual(my_mem.use_metrics['memused'],
Datapoint('.'.join((my_mem.info.uri, 'memused'))))
self.assertEqual(my_mem.use_metrics['majflt'],
Datapoint('.'.join((my_mem.info.uri, 'majflt'))))
self.assertEqual(len(my_mem.tms), 1)
def __init__(self, root_uri):
self.info = Datasource(
uri='.'.join((root_uri,'memory')),
supplies = ['pswpin','pswpout','pgpgin','pgpgout','fault','majflt','pgfree','pgscank','pgscand','pgsteal','vmeff','kbmemfree','kbmemused','memused','kbbuffers','kbcached','kbcommit','commit','kbactive','kbinact','kbdirty']
)
self.use_metrics= {
'memused':Datapoint('.'.join((self.info.uri,'memused'))),
'majflt':Datapoint('.'.join((self.info.uri,'majflt')))
}
self.tms = [
transfermethod(f=self.check_anom, schedule=OnUpdateSchedule(activation_metrics=self.use_metrics)),
]
def process_message_send_data_interval(msg, session, **kwargs):
if msg.m_type == Metrics.DATAPOINT:
metric = Datapoint(uri=msg.uri, session=session)
else:
metric = Datasource(uri=msg.uri, session=session)
for row in msg.data[::-1]:
session.store.insert(metric, row[0], row[1])
async def find_missing(self, content):
ifaces = []
iface_block = False
for line in content.split('\n'):
fields = line.split()
if iface_block and fields:
iface_name = fields[0]
final_name = ''
for i,c in enumerate(iface_name):
if validation.is_local_uri(c):
final_name += c
elif i>0:
final_name += '_'
if validation.is_local_uri(final_name):
ifaces.append(final_name)
else:
iface_block = False
if fields and fields[0] == 'IFACE':
iface_block = True
for iface in ifaces:
if not iface in self._ifaces:
logging.logger.debug('New network interface found: '+iface)
self._ifaces.append(iface)
for m in self._iface_metrics:
self.info.supplies.append('.'.join((iface,m)))
f_params = {
'ifutil':Datapoint(uri='.'.join((self.info.uri,iface,'ifutil')))
}
tm = transfermethod(f=self.check_anom, f_params=f_params)
await tm.bind()
self.tms.append(tm)
def process_message_send_multi_data(msg, session, **kwargs):
metrics=[]
for item in msg.uris:
if item['type'] == Metrics.DATASOURCE:
metric=Datasource(uri=item['uri'], session=session)
elif item['type'] == Metrics.DATAPOINT:
metric=Datapoint(uri=item['uri'], session=session)
if not session.store.is_in(metric=metric, t=msg.t, value=item['content']):
session.store.insert(metric, msg.t, item['content'])
metrics.append(metric)
tmIndex.metrics_updated(t=msg.t, metrics=metrics, irt=msg.seq)
def __init__(self, root_uri):
self.info = Datasource(
uri = '.'.join((root_uri,'cpu')),
supplies = ['ldavg.1min','ldavg.5min','ldavg.15min','cpu_count','all.user','all.nice','all.system','all.iowait','all.steal','all.idle','tasks.runnable','tasks.total','tasks.blocked']
)
self.use_metrics= {
'user':Datapoint('.'.join((self.info.uri,'all.user'))),
'nice':Datapoint('.'.join((self.info.uri,'all.nice'))),
'system':Datapoint('.'.join((self.info.uri,'all.system'))),
'steal':Datapoint('.'.join((self.info.uri,'all.steal'))),
'iowait':Datapoint('.'.join((self.info.uri,'all.iowait'))),
'idle':Datapoint('.'.join((self.info.uri,'all.idle'))),
'cpu_count':Datapoint('.'.join((self.info.uri,'cpu_count'))),
't_runnable':Datapoint('.'.join((self.info.uri,'tasks.runnable'))),
'ldavg1':Datapoint('.'.join((self.info.uri,'ldavg.1min')))
}
self.tms = [
transfermethod(f=self.check_anom, schedule=OnUpdateSchedule(activation_metrics=self.use_metrics)),
]
async def test_process_message_send_data_interval_success_dp_data(self):
''' process_message_send_data_interval should store contents in session store '''
username = '******'
privkey = crypto.generate_rsa_key()
uri = 'my_datapoint'
m_type = Metrics.DATAPOINT
start = TimeUUID(1)
end = TimeUUID(3000)
data_json = json.dumps([(TimeUUID(i).hex, i) for i in range(1, 100)])
data = json.loads(data_json)
session = KomlogSession(username=username, privkey=privkey)
msg = messages.SendDataInterval(uri, m_type, start, end, data)
self.assertIsNone(
prmsg.process_message_send_data_interval(msg, session))
for d in data:
smp = Sample(Datapoint(uri, session), TimeUUID(s=d[0]), d[1])
self.assertTrue(session.store.is_in(smp.metric, smp.t, smp.value))
sessionIndex.unregister_session(session.sid)
async def test_process_message_send_multi_data_success_no_notify_already_stored_values(
self):
''' process_message_send_multi_data should store contents and notify tmIndex '''
try:
username = '******'
privkey = crypto.generate_rsa_key()
t = TimeUUID()
uris = [{
'uri': 'datasource1',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource2',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource3',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource4',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datapoint1',
'type': Metrics.DATAPOINT.value,
'content': '1232'
}, {
'uri': 'datapoint2',
'type': Metrics.DATAPOINT.value,
'content': '1233'
}, {
'uri': 'datapoint3',
'type': Metrics.DATAPOINT.value,
'content': '1234'
}, {
'uri': 'datapoint4',
'type': Metrics.DATAPOINT.value,
'content': '1235'
}]
msg = messages.SendMultiData(t, uris)
session = KomlogSession(username=username, privkey=privkey)
bck = tmIndex.metrics_updated
tmIndex.metrics_updated = Mock(return_value=None)
self.assertIsNone(
prmsg.process_message_send_multi_data(msg, session))
metrics = [
Datasource('datasource1', session=session),
Datasource('datasource2', session=session),
Datasource('datasource3', session=session),
Datasource('datasource4', session=session),
Datapoint('datapoint1', session=session),
Datapoint('datapoint2', session=session),
Datapoint('datapoint3', session=session),
Datapoint('datapoint4', session=session)
]
for uri in uris:
if uri['type'] == Metrics.DATASOURCE.value:
smp = Sample(Datasource(uri['uri'], session=session), t,
uri['content'])
else:
smp = Sample(Datapoint(uri['uri'], session=session), t,
uri['content'])
self.assertTrue(
session.store.is_in(smp.metric, smp.t, smp.value))
self.assertEqual(tmIndex.metrics_updated.call_args[1]['t'], t)
self.assertEqual(tmIndex.metrics_updated.call_args[1]['metrics'],
metrics)
updated_uris = [{
'uri': 'datasource1',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource2',
'type': Metrics.DATASOURCE.value,
'content': 'ds data2'
}, {
'uri': 'datasource3',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource4',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datasource5',
'type': Metrics.DATASOURCE.value,
'content': 'ds data'
}, {
'uri': 'datapoint1',
'type': Metrics.DATAPOINT.value,
'content': '1233'
}, {
'uri': 'datapoint2',
'type': Metrics.DATAPOINT.value,
'content': '1234'
}, {
'uri': 'datapoint3',
'type': Metrics.DATAPOINT.value,
'content': '1234'
}, {
'uri': 'datapoint4',
'type': Metrics.DATAPOINT.value,
'content': '1235'
}, {
'uri': 'datapoint5',
'type': Metrics.DATAPOINT.value,
'content': '1235'
}]
msg = messages.SendMultiData(t, updated_uris)
updated_metrics = [
Datasource('datasource2', session=session),
Datasource('datasource5', session=session),
Datapoint('datapoint1', session=session),
Datapoint('datapoint2', session=session),
Datapoint('datapoint5', session=session)
]
self.assertIsNone(
prmsg.process_message_send_multi_data(msg, session))
for uri in updated_uris:
if uri['type'] == Metrics.DATASOURCE.value:
smp = Sample(Datasource(uri['uri']), t, uri['content'])
else:
smp = Sample(Datapoint(uri['uri']), t, uri['content'])
self.assertTrue(
session.store.is_in(smp.metric, smp.t, smp.value))
self.assertEqual(tmIndex.metrics_updated.call_args[1]['t'], t)
self.assertEqual(tmIndex.metrics_updated.call_args[1]['metrics'],
updated_metrics)
sessionIndex.unregister_session(session.sid)
tmIndex.metrics_updated = bck
except:
tmIndex.metrics_updated = bck
raise
async def test_send_samples_success(self):
''' send_samples should create messages and send through the sessions to Komlog '''
username1 = 'username1'
privkey1 = crypto.generate_rsa_key()
username2 = 'username2'
privkey2 = crypto.generate_rsa_key()
session1 = KomlogSession(username=username1, privkey=privkey1)
session2 = KomlogSession(username=username2, privkey=privkey2)
session1.send_message = test.AsyncMock(return_value=None)
session2.send_message = test.AsyncMock(return_value=None)
t_common = TimeUUID()
t1 = TimeUUID()
t2 = TimeUUID()
samples_s1 = [
Sample(Datasource('datasource1', session=session1), t_common,
'value'),
Sample(Datasource('datasource2', session=session1), t_common,
'value'),
Sample(Datasource('datasource3', session=session1), t_common,
'value'),
Sample(Datasource('datasource4', session=session1), t_common,
'value'),
Sample(Datapoint('datapoint1', session=session1), t_common, 1),
Sample(Datapoint('datapoint2', session=session1), t_common, 1),
Sample(Datapoint('datapoint3', session=session1), t_common, 1),
Sample(Datapoint('datapoint4', session=session1), t_common, 1),
Sample(Datasource('datasource5', session=session1), t1, 'value'),
Sample(Datapoint('datapoint5', session=session1), t2, 1),
]
samples_s2 = [
Sample(Datasource('datasource1', session=session2), t_common,
'value'),
Sample(Datasource('datasource2', session=session2), t_common,
'value'),
Sample(Datasource('datasource3', session=session2), t_common,
'value'),
Sample(Datasource('datasource4', session=session2), t_common,
'value'),
Sample(Datapoint('datapoint1', session=session2), t_common, 1),
Sample(Datapoint('datapoint2', session=session2), t_common, 1),
Sample(Datapoint('datapoint3', session=session2), t_common, 1),
Sample(Datapoint('datapoint4', session=session2), t_common, 1),
Sample(Datasource('datasource5', session=session2), t1, 'value'),
Sample(Datapoint('datapoint5', session=session2), t2, 1),
]
total_samples = []
for smp in samples_s1:
total_samples.append(smp)
for smp in samples_s2:
total_samples.append(smp)
response = await prproc.send_samples(total_samples)
self.assertEqual(session1.send_message.call_count, 3)
self.assertEqual(session2.send_message.call_count, 3)
self.assertEqual(response['success'], False)
self.assertEqual(len(response['errors']), 6)
for i, m in enumerate(response['errors']):
self.assertEqual(m['success'], False)
self.assertEqual(m['error'], 'Unexpected message type')
msg = m['msg']
if i == 0:
self.assertTrue(isinstance(msg, messages.SendMultiData))
self.assertEqual(msg.t, t_common)
self.assertEqual(msg.uris, [{
'uri': s.metric.uri,
'type': s.metric._m_type_,
'content': s.value
} for s in samples_s1[:-2]])
elif i == 1:
self.assertTrue(isinstance(msg, messages.SendDsData))
self.assertEqual(msg.uri, samples_s1[-2].metric.uri)
self.assertEqual(msg.t, samples_s1[-2].t)
self.assertEqual(msg.content, samples_s1[-2].value)
elif i == 2:
self.assertTrue(isinstance(msg, messages.SendDpData))
self.assertEqual(msg.uri, samples_s1[-1].metric.uri)
self.assertEqual(msg.t, samples_s1[-1].t)
self.assertEqual(msg.content, samples_s1[-1].value)
elif i == 3:
self.assertTrue(isinstance(msg, messages.SendMultiData))
self.assertEqual(msg.t, t_common)
self.assertEqual(msg.uris, [{
'uri': s.metric.uri,
'type': s.metric._m_type_,
'content': s.value
} for s in samples_s2[:-2]])
elif i == 4:
self.assertTrue(isinstance(msg, messages.SendDsData))
self.assertEqual(msg.uri, samples_s2[-2].metric.uri)
self.assertEqual(msg.t, samples_s2[-2].t)
self.assertEqual(msg.content, samples_s2[-2].value)
elif i == 5:
self.assertTrue(isinstance(msg, messages.SendDpData))
self.assertEqual(msg.uri, samples_s2[-1].metric.uri)
self.assertEqual(msg.t, samples_s2[-1].t)
self.assertEqual(msg.content, samples_s2[-1].value)
sessionIndex.unregister_session(session1.sid)
sessionIndex.unregister_session(session2.sid)