def zabbix_collect_used_disk(task):
# TODO: Write tests for this method
status = TaskHistory.STATUS_SUCCESS
threshold_disk_resize = Configuration.get_by_name_as_int(
"threshold_disk_resize", default=80.0
)
collected = 0
resizes = 0
problems = 0
integration = CredentialType.objects.get(type=CredentialType.ZABBIX)
for environment in Environment.objects.all():
task.add_detail(
'Execution for environment: {}'.format(environment.name)
)
try:
credentials = Credential.get_credentials(
environment=environment, integration=integration
)
except IndexError:
task.update_status_for(
TaskHistory.STATUS_ERROR,
'There is no Zabbix credential for {} environment.'.format(
environment
)
)
return
for database in Database.objects.filter(environment=environment):
database_resized = False
task.add_detail(
message='Database: {}'.format(database.name), level=1
)
zabbix_provider = factory_for(
databaseinfra=database.databaseinfra, credentials=credentials
)
metrics = ZabbixMetrics(
zabbix_provider.api, zabbix_provider.main_clientgroup
)
driver = database.databaseinfra.get_driver()
non_database_instances = driver.get_non_database_instances()
for host in zabbix_provider.hosts:
instance = database.databaseinfra.instances.filter(address=host.address).first()
if instance in non_database_instances:
continue
collected += 1
task.add_detail(
message='Host: {} ({})'.format(
host.hostname, host.address
), level=2
)
try:
zabbix_size = metrics.get_current_disk_data_size(host)
zabbix_used = metrics.get_current_disk_data_used(host)
zabbix_percentage = (zabbix_used * 100)/zabbix_size
except ZabbixMetricsError as error:
problems += 1
task.add_detail(message='Error: {}'.format(error), level=3)
status = TaskHistory.STATUS_WARNING
continue
task.add_detail(
message='Zabbix /data: {}% ({}kb/{}kb)'.format(
zabbix_percentage, zabbix_used, zabbix_size
), level=3
)
current_percentage = zabbix_percentage
current_used = zabbix_used
current_size = zabbix_size
if zabbix_percentage >= threshold_disk_resize:
current_percentage, current_used, current_size = host_mount_data_percentage(
address=host.address, task=task
)
if zabbix_percentage > current_percentage:
problems += 1
status = TaskHistory.STATUS_WARNING
task.add_detail(
message='Error: Zabbix metrics not updated',
level=4
)
size_metadata = database.databaseinfra.disk_offering.size_kb
if has_difference_between(size_metadata, current_size):
problems += 1
task.add_detail(
message='Error: Disk size different: {}kb'.format(
size_metadata
),
level=4
)
status = TaskHistory.STATUS_WARNING
elif current_percentage >= threshold_disk_resize and \
database.disk_auto_resize and \
not database_resized:
try:
task_resize = disk_auto_resize(
database=database,
current_size=current_size,
usage_percentage=current_percentage
)
database_resized = True
except Exception as e:
problems += 1
status = TaskHistory.STATUS_WARNING
task.add_detail(
message='Error: Could not do resize. {}'.format(e),
level=4
)
else:
resizes += 1
task.add_detail(
message='Executing Resize... Task: {}'.format(
task_resize.id
), level=4
)
if not update_disk(
database=database, address=host.address, task=task,
total_size=current_size, used_size=current_used
):
problems += 1
status = TaskHistory.STATUS_WARNING
zabbix_provider.logout()
details = 'Collected: {} | Resize: {} | Problems: {}'.format(
collected, resizes, problems
)
task.update_status_for(status=status, details=details)
def zabbix_collect_used_disk(task):
# TODO: Write tests for this method
status = TaskHistory.STATUS_SUCCESS
threshold_disk_resize = Configuration.get_by_name_as_int(
"threshold_disk_resize", default=80.0)
collected = 0
resizes = 0
problems = 0
integration = CredentialType.objects.get(type=CredentialType.ZABBIX)
for environment in Environment.objects.all():
task.add_detail("Execution for environment: {}".format(
environment.name))
try:
credentials = Credential.get_credentials(environment=environment,
integration=integration)
except IndexError:
task.update_status_for(
TaskHistory.STATUS_ERROR,
"There is no Zabbix credential for {} environment.".format(
environment),
)
return
for database in Database.objects.filter(environment=environment):
database_resized = False
task.add_detail(message="Database: {}".format(database.name),
level=1)
if database.is_locked:
message = ("Skip updating disk used size for database {}. "
"It is used by another task.").format(database)
task.add_detail(message, level=2)
continue
zabbix_provider = factory_for(databaseinfra=database.databaseinfra,
credentials=credentials)
metrics = ZabbixMetrics(zabbix_provider.api,
zabbix_provider.main_clientgroup)
driver = database.databaseinfra.get_driver()
non_database_instances = driver.get_non_database_instances()
for host in zabbix_provider.hosts:
instance = database.databaseinfra.instances.filter(
address=host.address).first()
if instance in non_database_instances:
continue
collected += 1
task.add_detail(message="Host: {} ({})".format(
host.hostname, host.address),
level=2)
try:
zabbix_size = metrics.get_current_disk_data_size(host)
zabbix_used = metrics.get_current_disk_data_used(host)
zabbix_percentage = (zabbix_used * 100) / zabbix_size
except ZabbixMetricsError as error:
ret = host_mount_data_percentage(host, task)
if ret != (None, None, None):
zabbix_percentage, zabbix_used, zabbix_size = ret
else:
problems += 1
task.add_detail(message="Error: {}".format(error),
level=3)
status = TaskHistory.STATUS_WARNING
continue
task.add_detail(
message="Zabbix /data: {}% ({}kb/{}kb)".format(
zabbix_percentage, zabbix_used, zabbix_size),
level=3,
)
current_percentage = zabbix_percentage
current_used = zabbix_used
current_size = zabbix_size
if zabbix_percentage >= threshold_disk_resize:
(
current_percentage,
current_used,
current_size,
) = host_mount_data_percentage(host=host, task=task)
if zabbix_percentage > current_percentage:
problems += 1
status = TaskHistory.STATUS_WARNING
task.add_detail(
message="Error: Zabbix metrics not updated",
level=4)
size_metadata = database.databaseinfra.disk_offering.size_kb
if has_difference_between(size_metadata, current_size):
problems += 1
task.add_detail(
message="Error: Disk size different: {}kb".format(
size_metadata),
level=4,
)
status = TaskHistory.STATUS_WARNING
elif (current_percentage >= threshold_disk_resize
and database.disk_auto_resize and not database_resized):
try:
task_resize = disk_auto_resize(
database=database,
current_size=size_metadata,
usage_percentage=current_percentage,
)
database_resized = True
except Exception as e:
problems += 1
status = TaskHistory.STATUS_WARNING
task.add_detail(
message="Error: Could not do resize. {}".format(e),
level=4)
else:
resizes += 1
task.add_detail(
message="Executing Resize... Task: {}".format(
task_resize.id),
level=4,
)
if not update_disk(
database=database,
address=host.address,
task=task,
total_size=current_size,
used_size=current_used,
):
problems += 1
status = TaskHistory.STATUS_WARNING
zabbix_provider.logout()
details = "Collected: {} | Resize: {} | Problems: {}".format(
collected, resizes, problems)
task.update_status_for(status=status, details=details)
def zabbix_collect_used_disk(task):
# TODO: Write tests for this method
status = TaskHistory.STATUS_SUCCESS
threshold_disk_resize = Configuration.get_by_name_as_int(
"threshold_disk_resize", default=80.0)
collected = 0
resizes = 0
problems = 0
integration = CredentialType.objects.get(type=CredentialType.ZABBIX)
for environment in Environment.objects.all():
task.add_detail('Execution for environment: {}'.format(
environment.name))
credentials = Credential.get_credentials(environment=environment,
integration=integration)
for database in Database.objects.filter(environment=environment):
database_resized = False
task.add_detail(message='Database: {}'.format(database.name),
level=1)
zabbix_provider = factory_for(databaseinfra=database.databaseinfra,
credentials=credentials)
metrics = ZabbixMetrics(zabbix_provider.api,
zabbix_provider.main_clientgroup)
driver = database.databaseinfra.get_driver()
non_database_instances = driver.get_non_database_instances()
for host in zabbix_provider.hosts:
instance = database.databaseinfra.instances.filter(
address=host.address).first()
if instance in non_database_instances:
continue
collected += 1
task.add_detail(message='Host: {} ({})'.format(
host.hostname, host.address),
level=2)
try:
zabbix_size = metrics.get_current_disk_data_size(host)
zabbix_used = metrics.get_current_disk_data_used(host)
zabbix_percentage = (zabbix_used * 100) / zabbix_size
except ZabbixMetricsError as error:
problems += 1
task.add_detail(message='Error: {}'.format(error), level=3)
status = TaskHistory.STATUS_WARNING
continue
task.add_detail(message='Zabbix /data: {}% ({}kb/{}kb)'.format(
zabbix_percentage, zabbix_used, zabbix_size),
level=3)
current_percentage = zabbix_percentage
current_used = zabbix_used
current_size = zabbix_size
if zabbix_percentage >= threshold_disk_resize:
current_percentage, current_used, current_size = host_mount_data_percentage(
address=host.address, task=task)
if zabbix_percentage > current_percentage:
problems += 1
status = TaskHistory.STATUS_WARNING
task.add_detail(message='Zabbix metrics not updated',
level=4)
size_metadata = database.databaseinfra.disk_offering.size_kb
if has_difference_between(size_metadata, current_size):
problems += 1
task.add_detail(
message='Disk size different in metadata: {}kb'.format(
size_metadata),
level=4)
status = TaskHistory.STATUS_WARNING
elif current_percentage >= threshold_disk_resize and \
database.disk_auto_resize and \
not database_resized:
try:
task_resize = disk_auto_resize(
database=database,
current_size=current_size,
usage_percentage=current_percentage)
database_resized = True
except Exception as e:
problems += 1
status = TaskHistory.STATUS_WARNING
task.add_detail(
message='Could not do resize. {}'.format(e),
level=4)
else:
resizes += 1
task.add_detail(
message='Executing Resize... Task: {}'.format(
task_resize.id),
level=4)
if not update_disk(database=database,
address=host.address,
task=task,
total_size=current_size,
used_size=current_used):
problems += 1
status = TaskHistory.STATUS_WARNING
zabbix_provider.logout()
details = 'Collected: {} | Resize: {} | Problems: {}'.format(
collected, resizes, problems)
task.update_status_for(status=status, details=details)
def setUp(self):
self.api = FakeApi()
self.group = 2
self.host = FakeHost(name='fake_host')
self.metrics = ZabbixMetrics(zappix_api=self.api, group=self.group)
class TestZabbixMetricsApi(unittest.TestCase):
def setUp(self):
self.api = FakeApi()
self.group = 2
self.host = FakeHost(name='fake_host')
self.metrics = ZabbixMetrics(zappix_api=self.api, group=self.group)
def test_can_get_items(self):
self.api.item.add_disk_data_size_item(item_id=123,
host=self.host,
group=self.group)
self.api.item.add_disk_data_used_item(item_id=456,
host=self.host,
group=self.group)
items = self.metrics.get_items(key=KEY_DISK_SIZE_DATA,
host_name=self.host.hostname)
self.assertEqual(1, len(items))
self.assertEqual(2, len(items[0]))
self.assertIn('itemid', items[0])
self.assertIn('value_type', items[0])
items = self.metrics.get_items(key=KEY_DISK_USED_DATA,
host_name=self.host.hostname)
self.assertEqual(1, len(items))
self.assertEqual(2, len(items[0]))
self.assertIn('itemid', items[0])
self.assertIn('value_type', items[0])
def test_empty_item_list(self):
items = self.metrics.get_items(key=KEY_DISK_USED_DATA,
host_name=self.host.hostname)
self.assertEqual(0, len(items))
def test_check_item_filters(self):
self.metrics.get_items(key=KEY_DISK_SIZE_DATA,
host_name=self.host.hostname)
last_filter = self.api.item.last_filter
self.assertEqual(5, len(last_filter))
self.assertIn('key_', last_filter)
self.assertEqual(last_filter['key_'], KEY_DISK_SIZE_DATA)
self.assertIn('group', last_filter)
self.assertEqual(last_filter['group'], self.group)
self.assertIn('host', last_filter)
self.assertEqual(last_filter['host'], self.host.hostname)
self.assertIn('status', last_filter)
self.assertEqual(last_filter['status'], 0)
self.assertIn('state', last_filter)
self.assertEqual(last_filter['state'], 0)
def test_can_get_history(self):
self.api.history.add_history(123, 3, 550, 200)
self.api.history.add_history(123, 3, 600, 210)
self.api.history.add_history(123, 3, 650, 220)
self.api.history.add_history(456, 3, 1950, 1200)
self.api.history.add_history(456, 3, 2200, 1300)
histories = self.metrics.get_history(3, [123, 456], 217, 1217)
self.assertEqual(2, len(histories))
self.assertEqual(3, len(histories[0]))
self.assertIn('itemid', histories[0])
self.assertIn('value', histories[0])
self.assertIn('clock', histories[0])
self.assertEqual(123, histories[0]['itemid'])
self.assertEqual(650, histories[0]['value'])
self.assertEqual(220, histories[0]['clock'])
self.assertEqual(3, len(histories[1]))
self.assertIn('itemid', histories[1])
self.assertIn('value', histories[1])
self.assertIn('clock', histories[1])
self.assertEqual(456, histories[1]['itemid'])
self.assertEqual(1950, histories[1]['value'])
self.assertEqual(1200, histories[1]['clock'])
def test_empty_history_list(self):
histories = self.metrics.get_history(3, [123, 456], 217, 1217)
self.assertEqual(0, len(histories))
def test_check_history_request(self):
history = 3
items = [123, 456]
time_from = 217
time_till = 1217
self.metrics.get_history(history, items, time_from, time_till)
request = self.api.history.last_request
self.assertEqual(5, len(request))
self.assertIn('output', request)
self.assertEqual(request['output'], 'extend')
self.assertIn('history', request)
self.assertEqual(request['history'], history)
self.assertIn('itemids', request)
self.assertEqual(request['itemids'], items)
self.assertIn('time_from', request)
self.assertEqual(request['time_from'], time_from)
self.assertIn('time_till', request)
self.assertEqual(request['time_till'], time_till)
def test_can_get_metrics(self):
self.api.item.add_disk_data_size_item(item_id=123,
host=self.host,
group=self.group)
self.api.item.add_disk_data_used_item(item_id=456,
host=self.host,
group=self.group)
self.api.history.add_history(123, 3, 550, 200)
self.api.history.add_history(123, 3, 600, 210)
self.api.history.add_history(123, 3, 650, 220)
self.api.history.add_history(456, 3, 1950, 1200)
self.api.history.add_history(456, 3, 2200, 1300)
metrics = self.metrics.get_metrics(217, 1217, KEYS, self.host)
self.assertEqual(len(KEYS), len(metrics))
for key in KEYS:
self.assertEqual(1, len(metrics[key]))
def test_cannot_get_metrics_without_items(self):
self.assertRaises(ZabbixApiKeyNotFoundError, self.metrics.get_metrics,
217, 1217, KEYS, self.host)
def test_cannot_get_metrics_without_history(self):
self.api.item.add_disk_data_used_item(item_id=456,
host=self.host,
group=self.group)
self.api.item.add_disk_data_size_item(item_id=456,
host=self.host,
group=self.group)
self.assertRaises(ZabbixApiNoDataBetweenTimeError,
self.metrics.get_metrics, 217, 1217, KEYS, self.host)
def setUp(self):
self.api = FakeApi()
self.group = 2
self.host = FakeHost(name='fake_host')
self.metrics = ZabbixMetrics(zappix_api=self.api, group=self.group)
class TestZabbixMetricsApi(unittest.TestCase):
def setUp(self):
self.api = FakeApi()
self.group = 2
self.host = FakeHost(name='fake_host')
self.metrics = ZabbixMetrics(zappix_api=self.api, group=self.group)
def test_can_get_items(self):
self.api.item.add_disk_data_size_item(
item_id=123, host=self.host, group=self.group
)
self.api.item.add_disk_data_used_item(
item_id=456, host=self.host, group=self.group
)
items = self.metrics.get_items(
key=KEY_DISK_SIZE_DATA, host_name=self.host.hostname
)
self.assertEqual(1, len(items))
self.assertEqual(2, len(items[0]))
self.assertIn('itemid', items[0])
self.assertIn('value_type', items[0])
items = self.metrics.get_items(
key=KEY_DISK_USED_DATA, host_name=self.host.hostname
)
self.assertEqual(1, len(items))
self.assertEqual(2, len(items[0]))
self.assertIn('itemid', items[0])
self.assertIn('value_type', items[0])
def test_empty_item_list(self):
items = self.metrics.get_items(
key=KEY_DISK_USED_DATA, host_name=self.host.hostname
)
self.assertEqual(0, len(items))
def test_check_item_filters(self):
self.metrics.get_items(
key=KEY_DISK_SIZE_DATA, host_name=self.host.hostname
)
last_filter = self.api.item.last_filter
self.assertEqual(5, len(last_filter))
self.assertIn('key_', last_filter)
self.assertEqual(last_filter['key_'], KEY_DISK_SIZE_DATA)
self.assertIn('group', last_filter)
self.assertEqual(last_filter['group'], self.group)
self.assertIn('host', last_filter)
self.assertEqual(last_filter['host'], self.host.hostname)
self.assertIn('status', last_filter)
self.assertEqual(last_filter['status'], 0)
self.assertIn('state', last_filter)
self.assertEqual(last_filter['state'], 0)
def test_can_get_history(self):
self.api.history.add_history(123, 3, 550, 200)
self.api.history.add_history(123, 3, 600, 210)
self.api.history.add_history(123, 3, 650, 220)
self.api.history.add_history(456, 3, 1950, 1200)
self.api.history.add_history(456, 3, 2200, 1300)
histories = self.metrics.get_history(3, [123, 456], 217, 1217)
self.assertEqual(2, len(histories))
self.assertEqual(3, len(histories[0]))
self.assertIn('itemid', histories[0])
self.assertIn('value', histories[0])
self.assertIn('clock', histories[0])
self.assertEqual(123, histories[0]['itemid'])
self.assertEqual(650, histories[0]['value'])
self.assertEqual(220, histories[0]['clock'])
self.assertEqual(3, len(histories[1]))
self.assertIn('itemid', histories[1])
self.assertIn('value', histories[1])
self.assertIn('clock', histories[1])
self.assertEqual(456, histories[1]['itemid'])
self.assertEqual(1950, histories[1]['value'])
self.assertEqual(1200, histories[1]['clock'])
def test_empty_history_list(self):
histories = self.metrics.get_history(3, [123, 456], 217, 1217)
self.assertEqual(0, len(histories))
def test_check_history_request(self):
history = 3
items = [123, 456]
time_from = 217
time_till = 1217
self.metrics.get_history(history, items, time_from, time_till)
request = self.api.history.last_request
self.assertEqual(5, len(request))
self.assertIn('output', request)
self.assertEqual(request['output'], 'extend')
self.assertIn('history', request)
self.assertEqual(request['history'], history)
self.assertIn('itemids', request)
self.assertEqual(request['itemids'], items)
self.assertIn('time_from', request)
self.assertEqual(request['time_from'], time_from)
self.assertIn('time_till', request)
self.assertEqual(request['time_till'], time_till)
def test_can_get_metrics(self):
self.api.item.add_disk_data_size_item(
item_id=123, host=self.host, group=self.group
)
self.api.item.add_disk_data_used_item(
item_id=456, host=self.host, group=self.group
)
self.api.history.add_history(123, 3, 550, 200)
self.api.history.add_history(123, 3, 600, 210)
self.api.history.add_history(123, 3, 650, 220)
self.api.history.add_history(456, 3, 1950, 1200)
self.api.history.add_history(456, 3, 2200, 1300)
metrics = self.metrics.get_metrics(217, 1217, KEYS, self.host)
self.assertEqual(len(KEYS), len(metrics))
for key in KEYS:
self.assertEqual(1, len(metrics[key]))
def test_cannot_get_metrics_without_items(self):
self.assertRaises(
ZabbixApiKeyNotFoundError,
self.metrics.get_metrics, 217, 1217, KEYS, self.host
)
def test_cannot_get_metrics_without_history(self):
self.api.item.add_disk_data_used_item(
item_id=456, host=self.host, group=self.group
)
self.api.item.add_disk_data_size_item(
item_id=456, host=self.host, group=self.group
)
self.assertRaises(
ZabbixApiNoDataBetweenTimeError,
self.metrics.get_metrics, 217, 1217, KEYS, self.host
)