def test_sort_by_iface_data(self):
self._cleanup()
for i in range(10):
data = {'name': 'iface-server-{0}'.format(i), 'last_check': 100}
server_id = server_model.collection.insert(data.copy())
for v in range(5):
device_data = {
'name': 'eth{0}'.format(v),
'server_id': server_id,
'last_update': 100,
}
device_id = interfaces_model.collection.insert(
device_data.copy())
data_dict = {
"server_id": server_id,
"device_id": device_id,
"i": i + v + 100,
"t": 100
}
interfaces_model.get_data_collection().insert(data_dict.copy())
result = map_model.sort_by(field='network:network.inbound')
assert len(result['sorted_data']) == 10
assert result['sorted_data'][0]['value'] == 113 # 100 + 9 + 4
assert result['sorted_data'][0]['unit'] == 'kb/s'
def test_sort_by_iface_data(self):
self._cleanup()
for i in range(10):
data = {
'name': 'iface-server-{0}'.format(i),
'last_check': 100
}
server_id = server_model.collection.insert(data.copy())
for v in range(5):
device_data = {
'name': 'eth{0}'.format(v),
'server_id': server_id,
'last_update': 100,
}
device_id = interfaces_model.collection.insert(device_data.copy())
data_dict = {
"server_id": server_id,
"device_id": device_id,
"i": i + v + 100,
"t": 100
}
interfaces_model.get_data_collection().insert(data_dict.copy())
result = map_model.sort_by(field='network:network.inbound')
assert len(result['sorted_data']) == 10
assert result['sorted_data'][0]['value'] == 113 # 100 + 9 + 4
assert result['sorted_data'][0]['unit'] == 'kb/s'
def _cleanup(self):
server_model.collection.remove()
system_model.data_collection.remove()
process_model.data_collection.remove()
process_model.collection.remove()
interfaces_model.collection.remove()
interfaces_model.get_data_collection().remove()
volumes_model.collection.remove()
volumes_model.get_data_collection().remove()
tags_model.collection.remove()
def _cleanup(self):
server_model.collection.remove()
system_model.data_collection.remove()
process_model.data_collection.remove()
process_model.collection.remove()
interfaces_model.collection.remove()
interfaces_model.get_data_collection().remove()
volumes_model.collection.remove()
volumes_model.get_data_collection().remove()
tags_model.collection.remove()
def delete_data(self, server_id=None, soft=None):
server_id = self.object_id(server_id)
params = {'server_id': server_id}
process_params = {'server': server_id}
processes = self.mongo.get_collection('processes')
processes.remove(process_params)
volumes = self.mongo.get_collection('volumes')
volumes.remove(params)
interfaces = self.mongo.get_collection('interfaces')
interfaces.remove(params)
plugins_col = self.mongo.get_collection('plugins')
plugin_gauges_col = self.mongo.get_collection('plugin_gauges')
plugins_for_server = plugins_col.find(params)
for p in plugins_for_server:
plugin_gauges_col.remove({'plugin_id': p['_id']})
# Soft delete - deletes process names, plugin names, network and disk volumes.
# Leaves the data which already has TTL and will be automatically deleted according to the 'Keep data' option
if soft == False:
process_data_collection = process_model.data_collection
process_data_collection.remove(params)
system_data_collection = self.data_collection
system_data_collection.remove(params)
volume_data_collection = volumes_model.get_data_collection()
volume_data_collection.remove(params)
interface_data_collection = interfaces_model.get_data_collection()
interface_data_collection.remove(params)
def _cleanup(self):
self.collection.remove()
data_collection = volumes_model.get_data_collection(server_id=self.server_id)
data_collection.remove()
data_collection = interfaces_model.get_data_collection(server_id=self.server_id)
data_collection.remove()
def _cleanup(self):
data_collection = system_model.data_collection
data_collection.remove()
network_collection = interfaces_model.get_data_collection(
server_id=self.server['_id'])
network_collection.remove()
disk_collection = volumes_model.get_data_collection(
server_id=self.server['_id'])
disk_collection.remove()
def cleanup(self, server, date_before=None):
process_data_collection = process_model.data_collection
params = {"t": {"$lte": date_before}, 'server_id': server['_id']}
process_data_collection.remove(params)
system_params = {"time": {"$lte": date_before}}
system_data_collection = self.data_collection
system_data_collection.remove(system_params)
volume_data_collection = volumes_model.get_data_collection()
volume_data_collection.remove(params)
interfaces_data_collection = interfaces_model.get_data_collection()
interfaces_data_collection.remove(params)
def save_network_data_test(self):
self._cleanup()
data_collection = interfaces_model.get_data_collection(server_id=self.server_id)
device_id = interfaces_model.get_or_create(server_id=self.server_id, name='eth1')
network_data = {
'eth1': {"inbound" : 991, "outbound" : 22, }
}
interfaces_model.save_data(self.server, network_data, time=1)
for r in data_collection.find():
eq_(r['device_id'], device_id['_id'])
eq_(r['i'], 991)
eq_(r['o'], 22)
eq_(r['t'], 1)
self._cleanup()
def cleanup_test(self):
self.collection.remove()
self.collection.insert({"name" : "testserver", "key": "test_me"})
server = self.collection.find_one()
date_before = 100
process_collection = process_model.data_collection
process_collection.remove()
system_collection = system_model.data_collection
system_collection.remove()
interface_collection = interfaces_model.get_data_collection()
interface_collection.remove()
volume_collection = volumes_model.get_data_collection()
volume_collection.remove()
for i in range(0, date_before):
process_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
system_collection.insert({'i' : 'test', 'time': i, 'server_id': server['_id']})
interface_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
volume_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
params = {'server_id': server['_id']}
self.model.cleanup(server, date_before=date_before)
process_entries = process_collection.find(params).count()
eq_(process_entries, 0)
system_entries = system_collection.find(params).count()
eq_(system_entries, 0)
interface_entries = interface_collection.find(params).count()
eq_(interface_entries, 0)
volume_entries = volume_collection.find(params).count()
eq_(volume_entries, 0)
system_collection.remove()
process_collection.remove()
interface_collection.remove()
volume_collection.remove()
entries = interface_collection.find().count()
eq_(entries, 0)
for i in range(0, 300):
process_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
system_collection.insert({'i' : 'test', 'time': i, 'server_id': server['_id']})
interface_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
volume_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
process_collection.ensure_index('server_id', background=True)
process_collection.ensure_index('t', background=True)
system_collection.ensure_index('time', background=True)
system_collection.ensure_index('server_id', background=True)
interface_collection.ensure_index('t', background=True)
interface_collection.ensure_index('server_id', background=True)
volume_collection.ensure_index('t', background=True)
volume_collection.ensure_index('server_id', background=True)
self.model.cleanup(server, date_before=date_before)
process_entries = process_collection.find(params).count()
eq_(process_entries, 199)
for p in process_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
system_entries = system_collection.find(params).count()
eq_(system_entries, 199)
for p in system_collection.find(sort=[('time', self.model.asc)]):
assert p['time'] > date_before
entries = interface_collection.find(params).count()
eq_(entries, 199)
for p in interface_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
entries = volume_collection.find(params).count()
eq_(entries, 199)
for p in volume_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
process_collection.drop()
system_collection.drop()
interface_collection.drop()
volume_collection.drop()
def cleanup_test(self):
self.collection.remove()
self.collection.insert({"name": "testserver", "key": "test_me"})
server = self.collection.find_one()
date_before = 100
process_collection = process_model.data_collection
process_collection.remove()
system_collection = system_model.data_collection
system_collection.remove()
interface_collection = interfaces_model.get_data_collection()
interface_collection.remove()
volume_collection = volumes_model.get_data_collection()
volume_collection.remove()
for i in range(0, date_before):
process_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
system_collection.insert({
'i': 'test',
'time': i,
'server_id': server['_id']
})
interface_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
volume_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
params = {'server_id': server['_id']}
self.model.cleanup(server, date_before=date_before)
process_entries = process_collection.find(params).count()
eq_(process_entries, 0)
system_entries = system_collection.find(params).count()
eq_(system_entries, 0)
interface_entries = interface_collection.find(params).count()
eq_(interface_entries, 0)
volume_entries = volume_collection.find(params).count()
eq_(volume_entries, 0)
system_collection.remove()
process_collection.remove()
interface_collection.remove()
volume_collection.remove()
entries = interface_collection.find().count()
eq_(entries, 0)
for i in range(0, 300):
process_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
system_collection.insert({
'i': 'test',
'time': i,
'server_id': server['_id']
})
interface_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
volume_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
process_collection.ensure_index('server_id', background=True)
process_collection.ensure_index('t', background=True)
system_collection.ensure_index('time', background=True)
system_collection.ensure_index('server_id', background=True)
interface_collection.ensure_index('t', background=True)
interface_collection.ensure_index('server_id', background=True)
volume_collection.ensure_index('t', background=True)
volume_collection.ensure_index('server_id', background=True)
self.model.cleanup(server, date_before=date_before)
process_entries = process_collection.find(params).count()
eq_(process_entries, 199)
for p in process_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
system_entries = system_collection.find(params).count()
eq_(system_entries, 199)
for p in system_collection.find(sort=[('time', self.model.asc)]):
assert p['time'] > date_before
entries = interface_collection.find(params).count()
eq_(entries, 199)
for p in interface_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
entries = volume_collection.find(params).count()
eq_(entries, 199)
for p in volume_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
process_collection.drop()
system_collection.drop()
interface_collection.drop()
volume_collection.drop()
def save_data_test(self):
self._cleanup()
expires_at = datetime.utcnow() + timedelta(hours=24)
last_check = 99999
system_data = {
u'disk': {
u'sda1': {
u'used': u'21350',
u'percent': u'46',
u'free': u'25237',
u'volume': u'/dev/sda1',
u'path': u'/',
u'total': u'49086'
}
},
u'memory': {
u'used_percent': 34,
u'swap_used_mb': 0,
u'total_mb': 3954,
u'free_mb': 2571,
u'swap_used_percent': 0,
u'swap_free_mb': 0,
u'used_mb': 1383,
u'swap_total_mb': 0
},
u'loadavg': {
u'cores': 4,
u'fifteen_minutes': u'0.36',
u'minute': u'0.12',
u'five_minutes': u'0.31'
},
u'network': {
u'eth3': {
u'inbound': u'6.05',
u'outbound': u'1.97'
}
},
u'cpu': {
u'iowait': u'0.00',
u'system': u'1.32',
u'idle': u'98.68',
u'user': u'0.00',
u'steal': u'0.00',
u'nice': u'0.00'
}
}
system_model.save_data(self.server,
system_data,
time=last_check,
expires_at=expires_at)
disk_collection = volumes_model.get_data_collection(
server_id=self.server['_id'])
disk = volumes_model.get_by_name(server=self.server, name='sda1')
eq_(disk_collection.find().count(), 1)
for r in disk_collection.find():
eq_(r['t'], last_check)
eq_(r['total'], "49086")
eq_(r['used'], "21350")
eq_(r['device_id'], disk['_id'])
eq_(r['expires_at'].date(), expires_at.date())
network_collection = interfaces_model.get_data_collection(
server_id=self.server['_id'])
adapter = interfaces_model.get_by_name(server=self.server, name='eth3')
eq_(network_collection.find().count(), 1)
for r in network_collection.find():
eq_(r['t'], last_check)
eq_(r['i'], "6.05")
eq_(r['o'], "1.97")
eq_(r['device_id'], adapter['_id'])
eq_(r['expires_at'].date(), expires_at.date())
data_collection = system_model.data_collection
for r in data_collection.find():
eq_(r['time'], last_check)
eq_(r['memory']['free_mb'], 2571)
eq_(r['loadavg']['fifteen_minutes'], '0.36')
eq_(r['cpu']['system'], '1.32')
eq_(r['expires_at'].date(), expires_at.date())
server_updated = server_model.get_by_id(self.server['_id'])
eq_(server_updated['last_check'], last_check)
self._cleanup()
def get_network_data_test(self):
self._cleanup()
interface = interfaces_model.get_or_create(
server_id=self.server['_id'], name='test_interface')
collection = interfaces_model.get_data_collection()
for i in range(0, 30):
collection.insert({
'server_id': self.server_id,
'device_id': interface['_id'],
't': i,
'i': 1,
'o': 2
})
result = system_model.get_device_data_after(timestamp=10,
enddate=20,
server=self.server,
check='network',
device_id=interface['_id'])
inbound = result[0]['data']
eq_(len(inbound), 11)
for entry in inbound:
assert entry['x'] >= 10
assert entry['x'] <= 20
assert entry['y'] == 1
assert type(entry['y']) is float
outbound = result[1]['data']
eq_(len(outbound), 11)
for entry in outbound:
assert entry['x'] >= 10
assert entry['x'] <= 20
assert entry['y'] == 2
# Test global data - used in the dashboards
all_servers = server_model.get_all()
result = system_model.get_global_device_data_after(
timestamp=10,
enddate=20,
filtered_servers=all_servers,
check='network',
key='i')
used_percent = result[0]['data']
eq_(len(used_percent), 11)
assert result[0]['name'] == "{0}.test_interface".format(
self.server['name'])
for entry in used_percent:
assert entry['x'] >= 10
assert entry['x'] <= 20
assert entry['y'] == 1.0
assert type(entry['y']) is float
self._cleanup()