def test_sort_by_volume_data(self): self._cleanup() for i in range(10): data = {'name': 'volume-server-{0}'.format(i), 'last_check': 100} server_id = server_model.collection.insert(data.copy()) for v in range(5): volume = { 'name': 'sda-{0}'.format(v), 'server_id': server_id, 'last_update': 100, } volume_id = volumes_model.collection.insert(volume.copy()) volume_data_dict = { "server_id": server_id, "device_id": volume_id, "percent": i + v + 10, "t": 100 } volumes_model.get_data_collection().insert( volume_data_dict.copy()) result = map_model.sort_by(field='disk:disk.used_percent') assert len(result['sorted_data']) == 10 assert result['sorted_data'][0]['value'] == 23 # 10 + 9 + 4 assert result['sorted_data'][0]['unit'] == '%'
def test_sort_by_volume_data(self): self._cleanup() for i in range(10): data = { 'name': 'volume-server-{0}'.format(i), 'last_check': 100 } server_id = server_model.collection.insert(data.copy()) for v in range(5): volume = { 'name': 'sda-{0}'.format(v), 'server_id': server_id, 'last_update': 100, } volume_id = volumes_model.collection.insert(volume.copy()) volume_data_dict = { "server_id": server_id, "device_id": volume_id, "percent": i + v + 10, "t": 100 } volumes_model.get_data_collection().insert(volume_data_dict.copy()) result = map_model.sort_by(field='disk:disk.used_percent') assert len(result['sorted_data']) == 10 assert result['sorted_data'][0]['value'] == 23 # 10 + 9 + 4 assert result['sorted_data'][0]['unit'] == '%'
def _cleanup(self): server_model.collection.remove() system_model.data_collection.remove() process_model.data_collection.remove() process_model.collection.remove() interfaces_model.collection.remove() interfaces_model.get_data_collection().remove() volumes_model.collection.remove() volumes_model.get_data_collection().remove() tags_model.collection.remove()
def _cleanup(self): server_model.collection.remove() system_model.data_collection.remove() process_model.data_collection.remove() process_model.collection.remove() interfaces_model.collection.remove() interfaces_model.get_data_collection().remove() volumes_model.collection.remove() volumes_model.get_data_collection().remove() tags_model.collection.remove()
def save_disk_data_test(self): self._cleanup() data_collection = volumes_model.get_data_collection(server_id=self.server_id) device_id = volumes_model.get_or_create(server_id=self.server_id, name='sda1') volume_data = { 'sda1': {"used" : "19060", "percent" : "41", "free" : "27527", "total": 62} } volumes_model.save_data(server=self.server, data=volume_data, time=1) for r in data_collection.find(): eq_(r['device_id'], device_id['_id']) eq_(r['percent'], '41') eq_(r['used'], "19060") eq_(r['t'], 1) golang_data = [{"name":"sda1","path":"/","fstype":"ext2/ext3","total":"36236.00", "free":"3194.00","used":"31282.00","percent":86.33}] data_collection.remove() volumes_model.save_data(server=self.server, data=golang_data, time=1) for r in data_collection.find(): eq_(r['device_id'], device_id['_id']) eq_(r['percent'], 86.33) eq_(r['used'], "31282.00") eq_(r['t'], 1) self._cleanup()
def delete_data(self, server_id=None, soft=None): server_id = self.object_id(server_id) params = {'server_id': server_id} process_params = {'server': server_id} processes = self.mongo.get_collection('processes') processes.remove(process_params) volumes = self.mongo.get_collection('volumes') volumes.remove(params) interfaces = self.mongo.get_collection('interfaces') interfaces.remove(params) plugins_col = self.mongo.get_collection('plugins') plugin_gauges_col = self.mongo.get_collection('plugin_gauges') plugins_for_server = plugins_col.find(params) for p in plugins_for_server: plugin_gauges_col.remove({'plugin_id': p['_id']}) # Soft delete - deletes process names, plugin names, network and disk volumes. # Leaves the data which already has TTL and will be automatically deleted according to the 'Keep data' option if soft == False: process_data_collection = process_model.data_collection process_data_collection.remove(params) system_data_collection = self.data_collection system_data_collection.remove(params) volume_data_collection = volumes_model.get_data_collection() volume_data_collection.remove(params) interface_data_collection = interfaces_model.get_data_collection() interface_data_collection.remove(params)
def _cleanup(self): self.collection.remove() data_collection = volumes_model.get_data_collection(server_id=self.server_id) data_collection.remove() data_collection = interfaces_model.get_data_collection(server_id=self.server_id) data_collection.remove()
def _cleanup(self): data_collection = system_model.data_collection data_collection.remove() network_collection = interfaces_model.get_data_collection( server_id=self.server['_id']) network_collection.remove() disk_collection = volumes_model.get_data_collection( server_id=self.server['_id']) disk_collection.remove()
def cleanup(self, server, date_before=None): process_data_collection = process_model.data_collection params = {"t": {"$lte": date_before}, 'server_id': server['_id']} process_data_collection.remove(params) system_params = {"time": {"$lte": date_before}} system_data_collection = self.data_collection system_data_collection.remove(system_params) volume_data_collection = volumes_model.get_data_collection() volume_data_collection.remove(params) interfaces_data_collection = interfaces_model.get_data_collection() interfaces_data_collection.remove(params)
def cleanup_test(self): self.collection.remove() self.collection.insert({"name" : "testserver", "key": "test_me"}) server = self.collection.find_one() date_before = 100 process_collection = process_model.data_collection process_collection.remove() system_collection = system_model.data_collection system_collection.remove() interface_collection = interfaces_model.get_data_collection() interface_collection.remove() volume_collection = volumes_model.get_data_collection() volume_collection.remove() for i in range(0, date_before): process_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']}) system_collection.insert({'i' : 'test', 'time': i, 'server_id': server['_id']}) interface_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']}) volume_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']}) params = {'server_id': server['_id']} self.model.cleanup(server, date_before=date_before) process_entries = process_collection.find(params).count() eq_(process_entries, 0) system_entries = system_collection.find(params).count() eq_(system_entries, 0) interface_entries = interface_collection.find(params).count() eq_(interface_entries, 0) volume_entries = volume_collection.find(params).count() eq_(volume_entries, 0) system_collection.remove() process_collection.remove() interface_collection.remove() volume_collection.remove() entries = interface_collection.find().count() eq_(entries, 0) for i in range(0, 300): process_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']}) system_collection.insert({'i' : 'test', 'time': i, 'server_id': server['_id']}) interface_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']}) volume_collection.insert({'i' : 'test', 't': i, 'server_id': server['_id']}) process_collection.ensure_index('server_id', background=True) process_collection.ensure_index('t', background=True) system_collection.ensure_index('time', background=True) system_collection.ensure_index('server_id', background=True) interface_collection.ensure_index('t', background=True) interface_collection.ensure_index('server_id', background=True) volume_collection.ensure_index('t', background=True) volume_collection.ensure_index('server_id', background=True) self.model.cleanup(server, date_before=date_before) process_entries = process_collection.find(params).count() eq_(process_entries, 199) for p in process_collection.find(sort=[('t', self.model.asc)]): assert p['t'] > date_before system_entries = system_collection.find(params).count() eq_(system_entries, 199) for p in system_collection.find(sort=[('time', self.model.asc)]): assert p['time'] > date_before entries = interface_collection.find(params).count() eq_(entries, 199) for p in interface_collection.find(sort=[('t', self.model.asc)]): assert p['t'] > date_before entries = volume_collection.find(params).count() eq_(entries, 199) for p in volume_collection.find(sort=[('t', self.model.asc)]): assert p['t'] > date_before process_collection.drop() system_collection.drop() interface_collection.drop() volume_collection.drop()
def get_data_collection_test(self): result = volumes_model.get_data_collection() eq_(result.name, "volume_data")
def cleanup_test(self): self.collection.remove() self.collection.insert({"name": "testserver", "key": "test_me"}) server = self.collection.find_one() date_before = 100 process_collection = process_model.data_collection process_collection.remove() system_collection = system_model.data_collection system_collection.remove() interface_collection = interfaces_model.get_data_collection() interface_collection.remove() volume_collection = volumes_model.get_data_collection() volume_collection.remove() for i in range(0, date_before): process_collection.insert({ 'i': 'test', 't': i, 'server_id': server['_id'] }) system_collection.insert({ 'i': 'test', 'time': i, 'server_id': server['_id'] }) interface_collection.insert({ 'i': 'test', 't': i, 'server_id': server['_id'] }) volume_collection.insert({ 'i': 'test', 't': i, 'server_id': server['_id'] }) params = {'server_id': server['_id']} self.model.cleanup(server, date_before=date_before) process_entries = process_collection.find(params).count() eq_(process_entries, 0) system_entries = system_collection.find(params).count() eq_(system_entries, 0) interface_entries = interface_collection.find(params).count() eq_(interface_entries, 0) volume_entries = volume_collection.find(params).count() eq_(volume_entries, 0) system_collection.remove() process_collection.remove() interface_collection.remove() volume_collection.remove() entries = interface_collection.find().count() eq_(entries, 0) for i in range(0, 300): process_collection.insert({ 'i': 'test', 't': i, 'server_id': server['_id'] }) system_collection.insert({ 'i': 'test', 'time': i, 'server_id': server['_id'] }) interface_collection.insert({ 'i': 'test', 't': i, 'server_id': server['_id'] }) volume_collection.insert({ 'i': 'test', 't': i, 'server_id': server['_id'] }) process_collection.ensure_index('server_id', background=True) process_collection.ensure_index('t', background=True) system_collection.ensure_index('time', background=True) system_collection.ensure_index('server_id', background=True) interface_collection.ensure_index('t', background=True) interface_collection.ensure_index('server_id', background=True) volume_collection.ensure_index('t', background=True) volume_collection.ensure_index('server_id', background=True) self.model.cleanup(server, date_before=date_before) process_entries = process_collection.find(params).count() eq_(process_entries, 199) for p in process_collection.find(sort=[('t', self.model.asc)]): assert p['t'] > date_before system_entries = system_collection.find(params).count() eq_(system_entries, 199) for p in system_collection.find(sort=[('time', self.model.asc)]): assert p['time'] > date_before entries = interface_collection.find(params).count() eq_(entries, 199) for p in interface_collection.find(sort=[('t', self.model.asc)]): assert p['t'] > date_before entries = volume_collection.find(params).count() eq_(entries, 199) for p in volume_collection.find(sort=[('t', self.model.asc)]): assert p['t'] > date_before process_collection.drop() system_collection.drop() interface_collection.drop() volume_collection.drop()
def save_data_test(self): self._cleanup() expires_at = datetime.utcnow() + timedelta(hours=24) last_check = 99999 system_data = { u'disk': { u'sda1': { u'used': u'21350', u'percent': u'46', u'free': u'25237', u'volume': u'/dev/sda1', u'path': u'/', u'total': u'49086' } }, u'memory': { u'used_percent': 34, u'swap_used_mb': 0, u'total_mb': 3954, u'free_mb': 2571, u'swap_used_percent': 0, u'swap_free_mb': 0, u'used_mb': 1383, u'swap_total_mb': 0 }, u'loadavg': { u'cores': 4, u'fifteen_minutes': u'0.36', u'minute': u'0.12', u'five_minutes': u'0.31' }, u'network': { u'eth3': { u'inbound': u'6.05', u'outbound': u'1.97' } }, u'cpu': { u'iowait': u'0.00', u'system': u'1.32', u'idle': u'98.68', u'user': u'0.00', u'steal': u'0.00', u'nice': u'0.00' } } system_model.save_data(self.server, system_data, time=last_check, expires_at=expires_at) disk_collection = volumes_model.get_data_collection( server_id=self.server['_id']) disk = volumes_model.get_by_name(server=self.server, name='sda1') eq_(disk_collection.find().count(), 1) for r in disk_collection.find(): eq_(r['t'], last_check) eq_(r['total'], "49086") eq_(r['used'], "21350") eq_(r['device_id'], disk['_id']) eq_(r['expires_at'].date(), expires_at.date()) network_collection = interfaces_model.get_data_collection( server_id=self.server['_id']) adapter = interfaces_model.get_by_name(server=self.server, name='eth3') eq_(network_collection.find().count(), 1) for r in network_collection.find(): eq_(r['t'], last_check) eq_(r['i'], "6.05") eq_(r['o'], "1.97") eq_(r['device_id'], adapter['_id']) eq_(r['expires_at'].date(), expires_at.date()) data_collection = system_model.data_collection for r in data_collection.find(): eq_(r['time'], last_check) eq_(r['memory']['free_mb'], 2571) eq_(r['loadavg']['fifteen_minutes'], '0.36') eq_(r['cpu']['system'], '1.32') eq_(r['expires_at'].date(), expires_at.date()) server_updated = server_model.get_by_id(self.server['_id']) eq_(server_updated['last_check'], last_check) self._cleanup()
def get_disk_data_test(self): self._cleanup() volume = volumes_model.get_or_create(server_id=self.server['_id'], name='test_volume') collection = volumes_model.get_data_collection() for i in range(0, 30): collection.insert({ 'server_id': self.server_id, 'device_id': volume['_id'], 't': i, 'total': 12, 'used': 2, 'percent': 60.0 }) result = system_model.get_device_data_after(timestamp=10, enddate=20, server=self.server, check='disk', device_id=volume['_id']) total = result[1]['data'] eq_(len(total), 11) for entry in total: assert entry['x'] >= 10 assert entry['x'] <= 20 assert entry['y'] == 12 assert type(entry['y']) is float used = result[0]['data'] eq_(len(used), 11) for entry in used: assert entry['x'] >= 10 assert entry['x'] <= 20 assert entry['y'] == 2 # Test global data - used in the dashboards all_servers = server_model.get_all() result = system_model.get_global_device_data_after( timestamp=10, enddate=20, filtered_servers=all_servers, check='disk', key='percent') used_percent = result[0]['data'] eq_(len(used_percent), 11) assert result[0]['name'] == "{0}.test_volume".format( self.server['name']) for entry in used_percent: assert entry['x'] >= 10 assert entry['x'] <= 20 assert entry['y'] == 60.0 assert type(entry['y']) is float self._cleanup()