def test_sort_by_volume_data(self):
self._cleanup()
for i in range(10):
data = {'name': 'volume-server-{0}'.format(i), 'last_check': 100}
server_id = server_model.collection.insert(data.copy())
for v in range(5):
volume = {
'name': 'sda-{0}'.format(v),
'server_id': server_id,
'last_update': 100,
}
volume_id = volumes_model.collection.insert(volume.copy())
volume_data_dict = {
"server_id": server_id,
"device_id": volume_id,
"percent": i + v + 10,
"t": 100
}
volumes_model.get_data_collection().insert(
volume_data_dict.copy())
result = map_model.sort_by(field='disk:disk.used_percent')
assert len(result['sorted_data']) == 10
assert result['sorted_data'][0]['value'] == 23 # 10 + 9 + 4
assert result['sorted_data'][0]['unit'] == '%'
def test_sort_by_volume_data(self):
self._cleanup()
for i in range(10):
data = {
'name': 'volume-server-{0}'.format(i),
'last_check': 100
}
server_id = server_model.collection.insert(data.copy())
for v in range(5):
volume = {
'name': 'sda-{0}'.format(v),
'server_id': server_id,
'last_update': 100,
}
volume_id = volumes_model.collection.insert(volume.copy())
volume_data_dict = {
"server_id": server_id,
"device_id": volume_id,
"percent": i + v + 10,
"t": 100
}
volumes_model.get_data_collection().insert(volume_data_dict.copy())
result = map_model.sort_by(field='disk:disk.used_percent')
assert len(result['sorted_data']) == 10
assert result['sorted_data'][0]['value'] == 23 # 10 + 9 + 4
assert result['sorted_data'][0]['unit'] == '%'
def _cleanup(self):
server_model.collection.remove()
system_model.data_collection.remove()
process_model.data_collection.remove()
process_model.collection.remove()
interfaces_model.collection.remove()
interfaces_model.get_data_collection().remove()
volumes_model.collection.remove()
volumes_model.get_data_collection().remove()
tags_model.collection.remove()
def _cleanup(self):
server_model.collection.remove()
system_model.data_collection.remove()
process_model.data_collection.remove()
process_model.collection.remove()
interfaces_model.collection.remove()
interfaces_model.get_data_collection().remove()
volumes_model.collection.remove()
volumes_model.get_data_collection().remove()
tags_model.collection.remove()
def save_disk_data_test(self):
self._cleanup()
data_collection = volumes_model.get_data_collection(server_id=self.server_id)
device_id = volumes_model.get_or_create(server_id=self.server_id, name='sda1')
volume_data = {
'sda1': {"used" : "19060", "percent" : "41", "free" : "27527", "total": 62}
}
volumes_model.save_data(server=self.server, data=volume_data, time=1)
for r in data_collection.find():
eq_(r['device_id'], device_id['_id'])
eq_(r['percent'], '41')
eq_(r['used'], "19060")
eq_(r['t'], 1)
golang_data = [{"name":"sda1","path":"/","fstype":"ext2/ext3","total":"36236.00",
"free":"3194.00","used":"31282.00","percent":86.33}]
data_collection.remove()
volumes_model.save_data(server=self.server, data=golang_data, time=1)
for r in data_collection.find():
eq_(r['device_id'], device_id['_id'])
eq_(r['percent'], 86.33)
eq_(r['used'], "31282.00")
eq_(r['t'], 1)
self._cleanup()
def delete_data(self, server_id=None, soft=None):
server_id = self.object_id(server_id)
params = {'server_id': server_id}
process_params = {'server': server_id}
processes = self.mongo.get_collection('processes')
processes.remove(process_params)
volumes = self.mongo.get_collection('volumes')
volumes.remove(params)
interfaces = self.mongo.get_collection('interfaces')
interfaces.remove(params)
plugins_col = self.mongo.get_collection('plugins')
plugin_gauges_col = self.mongo.get_collection('plugin_gauges')
plugins_for_server = plugins_col.find(params)
for p in plugins_for_server:
plugin_gauges_col.remove({'plugin_id': p['_id']})
# Soft delete - deletes process names, plugin names, network and disk volumes.
# Leaves the data which already has TTL and will be automatically deleted according to the 'Keep data' option
if soft == False:
process_data_collection = process_model.data_collection
process_data_collection.remove(params)
system_data_collection = self.data_collection
system_data_collection.remove(params)
volume_data_collection = volumes_model.get_data_collection()
volume_data_collection.remove(params)
interface_data_collection = interfaces_model.get_data_collection()
interface_data_collection.remove(params)
def _cleanup(self):
self.collection.remove()
data_collection = volumes_model.get_data_collection(server_id=self.server_id)
data_collection.remove()
data_collection = interfaces_model.get_data_collection(server_id=self.server_id)
data_collection.remove()
def _cleanup(self):
data_collection = system_model.data_collection
data_collection.remove()
network_collection = interfaces_model.get_data_collection(
server_id=self.server['_id'])
network_collection.remove()
disk_collection = volumes_model.get_data_collection(
server_id=self.server['_id'])
disk_collection.remove()
def cleanup(self, server, date_before=None):
process_data_collection = process_model.data_collection
params = {"t": {"$lte": date_before}, 'server_id': server['_id']}
process_data_collection.remove(params)
system_params = {"time": {"$lte": date_before}}
system_data_collection = self.data_collection
system_data_collection.remove(system_params)
volume_data_collection = volumes_model.get_data_collection()
volume_data_collection.remove(params)
interfaces_data_collection = interfaces_model.get_data_collection()
interfaces_data_collection.remove(params)
def cleanup_test(self):
self.collection.remove()
self.collection.insert({"name" : "testserver", "key": "test_me"})
server = self.collection.find_one()
date_before = 100
process_collection = process_model.data_collection
process_collection.remove()
system_collection = system_model.data_collection
system_collection.remove()
interface_collection = interfaces_model.get_data_collection()
interface_collection.remove()
volume_collection = volumes_model.get_data_collection()
volume_collection.remove()
for i in range(0, date_before):
process_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
system_collection.insert({'i' : 'test', 'time': i, 'server_id': server['_id']})
interface_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
volume_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
params = {'server_id': server['_id']}
self.model.cleanup(server, date_before=date_before)
process_entries = process_collection.find(params).count()
eq_(process_entries, 0)
system_entries = system_collection.find(params).count()
eq_(system_entries, 0)
interface_entries = interface_collection.find(params).count()
eq_(interface_entries, 0)
volume_entries = volume_collection.find(params).count()
eq_(volume_entries, 0)
system_collection.remove()
process_collection.remove()
interface_collection.remove()
volume_collection.remove()
entries = interface_collection.find().count()
eq_(entries, 0)
for i in range(0, 300):
process_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
system_collection.insert({'i' : 'test', 'time': i, 'server_id': server['_id']})
interface_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
volume_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']})
process_collection.ensure_index('server_id', background=True)
process_collection.ensure_index('t', background=True)
system_collection.ensure_index('time', background=True)
system_collection.ensure_index('server_id', background=True)
interface_collection.ensure_index('t', background=True)
interface_collection.ensure_index('server_id', background=True)
volume_collection.ensure_index('t', background=True)
volume_collection.ensure_index('server_id', background=True)
self.model.cleanup(server, date_before=date_before)
process_entries = process_collection.find(params).count()
eq_(process_entries, 199)
for p in process_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
system_entries = system_collection.find(params).count()
eq_(system_entries, 199)
for p in system_collection.find(sort=[('time', self.model.asc)]):
assert p['time'] > date_before
entries = interface_collection.find(params).count()
eq_(entries, 199)
for p in interface_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
entries = volume_collection.find(params).count()
eq_(entries, 199)
for p in volume_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
process_collection.drop()
system_collection.drop()
interface_collection.drop()
volume_collection.drop()
def get_data_collection_test(self):
result = volumes_model.get_data_collection()
eq_(result.name, "volume_data")
def cleanup_test(self):
self.collection.remove()
self.collection.insert({"name": "testserver", "key": "test_me"})
server = self.collection.find_one()
date_before = 100
process_collection = process_model.data_collection
process_collection.remove()
system_collection = system_model.data_collection
system_collection.remove()
interface_collection = interfaces_model.get_data_collection()
interface_collection.remove()
volume_collection = volumes_model.get_data_collection()
volume_collection.remove()
for i in range(0, date_before):
process_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
system_collection.insert({
'i': 'test',
'time': i,
'server_id': server['_id']
})
interface_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
volume_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
params = {'server_id': server['_id']}
self.model.cleanup(server, date_before=date_before)
process_entries = process_collection.find(params).count()
eq_(process_entries, 0)
system_entries = system_collection.find(params).count()
eq_(system_entries, 0)
interface_entries = interface_collection.find(params).count()
eq_(interface_entries, 0)
volume_entries = volume_collection.find(params).count()
eq_(volume_entries, 0)
system_collection.remove()
process_collection.remove()
interface_collection.remove()
volume_collection.remove()
entries = interface_collection.find().count()
eq_(entries, 0)
for i in range(0, 300):
process_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
system_collection.insert({
'i': 'test',
'time': i,
'server_id': server['_id']
})
interface_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
volume_collection.insert({
'i': 'test',
't': i,
'server_id': server['_id']
})
process_collection.ensure_index('server_id', background=True)
process_collection.ensure_index('t', background=True)
system_collection.ensure_index('time', background=True)
system_collection.ensure_index('server_id', background=True)
interface_collection.ensure_index('t', background=True)
interface_collection.ensure_index('server_id', background=True)
volume_collection.ensure_index('t', background=True)
volume_collection.ensure_index('server_id', background=True)
self.model.cleanup(server, date_before=date_before)
process_entries = process_collection.find(params).count()
eq_(process_entries, 199)
for p in process_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
system_entries = system_collection.find(params).count()
eq_(system_entries, 199)
for p in system_collection.find(sort=[('time', self.model.asc)]):
assert p['time'] > date_before
entries = interface_collection.find(params).count()
eq_(entries, 199)
for p in interface_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
entries = volume_collection.find(params).count()
eq_(entries, 199)
for p in volume_collection.find(sort=[('t', self.model.asc)]):
assert p['t'] > date_before
process_collection.drop()
system_collection.drop()
interface_collection.drop()
volume_collection.drop()
def save_data_test(self):
self._cleanup()
expires_at = datetime.utcnow() + timedelta(hours=24)
last_check = 99999
system_data = {
u'disk': {
u'sda1': {
u'used': u'21350',
u'percent': u'46',
u'free': u'25237',
u'volume': u'/dev/sda1',
u'path': u'/',
u'total': u'49086'
}
},
u'memory': {
u'used_percent': 34,
u'swap_used_mb': 0,
u'total_mb': 3954,
u'free_mb': 2571,
u'swap_used_percent': 0,
u'swap_free_mb': 0,
u'used_mb': 1383,
u'swap_total_mb': 0
},
u'loadavg': {
u'cores': 4,
u'fifteen_minutes': u'0.36',
u'minute': u'0.12',
u'five_minutes': u'0.31'
},
u'network': {
u'eth3': {
u'inbound': u'6.05',
u'outbound': u'1.97'
}
},
u'cpu': {
u'iowait': u'0.00',
u'system': u'1.32',
u'idle': u'98.68',
u'user': u'0.00',
u'steal': u'0.00',
u'nice': u'0.00'
}
}
system_model.save_data(self.server,
system_data,
time=last_check,
expires_at=expires_at)
disk_collection = volumes_model.get_data_collection(
server_id=self.server['_id'])
disk = volumes_model.get_by_name(server=self.server, name='sda1')
eq_(disk_collection.find().count(), 1)
for r in disk_collection.find():
eq_(r['t'], last_check)
eq_(r['total'], "49086")
eq_(r['used'], "21350")
eq_(r['device_id'], disk['_id'])
eq_(r['expires_at'].date(), expires_at.date())
network_collection = interfaces_model.get_data_collection(
server_id=self.server['_id'])
adapter = interfaces_model.get_by_name(server=self.server, name='eth3')
eq_(network_collection.find().count(), 1)
for r in network_collection.find():
eq_(r['t'], last_check)
eq_(r['i'], "6.05")
eq_(r['o'], "1.97")
eq_(r['device_id'], adapter['_id'])
eq_(r['expires_at'].date(), expires_at.date())
data_collection = system_model.data_collection
for r in data_collection.find():
eq_(r['time'], last_check)
eq_(r['memory']['free_mb'], 2571)
eq_(r['loadavg']['fifteen_minutes'], '0.36')
eq_(r['cpu']['system'], '1.32')
eq_(r['expires_at'].date(), expires_at.date())
server_updated = server_model.get_by_id(self.server['_id'])
eq_(server_updated['last_check'], last_check)
self._cleanup()
def get_disk_data_test(self):
self._cleanup()
volume = volumes_model.get_or_create(server_id=self.server['_id'],
name='test_volume')
collection = volumes_model.get_data_collection()
for i in range(0, 30):
collection.insert({
'server_id': self.server_id,
'device_id': volume['_id'],
't': i,
'total': 12,
'used': 2,
'percent': 60.0
})
result = system_model.get_device_data_after(timestamp=10,
enddate=20,
server=self.server,
check='disk',
device_id=volume['_id'])
total = result[1]['data']
eq_(len(total), 11)
for entry in total:
assert entry['x'] >= 10
assert entry['x'] <= 20
assert entry['y'] == 12
assert type(entry['y']) is float
used = result[0]['data']
eq_(len(used), 11)
for entry in used:
assert entry['x'] >= 10
assert entry['x'] <= 20
assert entry['y'] == 2
# Test global data - used in the dashboards
all_servers = server_model.get_all()
result = system_model.get_global_device_data_after(
timestamp=10,
enddate=20,
filtered_servers=all_servers,
check='disk',
key='percent')
used_percent = result[0]['data']
eq_(len(used_percent), 11)
assert result[0]['name'] == "{0}.test_volume".format(
self.server['name'])
for entry in used_percent:
assert entry['x'] >= 10
assert entry['x'] <= 20
assert entry['y'] == 60.0
assert type(entry['y']) is float
self._cleanup()