def collect(resource_type):
try:
operational_clusters = ClusterCollection.filter_by(
iterable=None, status=consts.CLUSTER_STATUSES.operational).all()
error_clusters = ClusterCollection.filter_by(
iterable=None, status=consts.CLUSTER_STATUSES.error).all()
all_envs_last_recs = \
OpenStackWorkloadStatsCollection.get_last_by_resource_type(
resource_type)
ready_or_error_ids = set([c.id for c in operational_clusters] +
[c.id for c in error_clusters])
envs_ids_to_clear = set(r.cluster_id for r in all_envs_last_recs) - \
ready_or_error_ids
# Clear current resource data for unavailable clusters.
# Current OSWL data is cleared for those clusters which status is not
# 'operational' nor 'error' or when cluster was removed. Data is
# cleared for cluster only if it was updated recently (today or
# yesterday). While this collector is running with interval much
# smaller than one day it should not miss any unavailable cluster.
for id in envs_ids_to_clear:
oswl_statistics_save(id, resource_type, [])
# Collect current OSWL data and update data in DB
for cluster in operational_clusters:
try:
client_provider = helpers.ClientProvider(cluster)
proxy_for_os_api = utils.get_proxy_for_cluster(cluster)
version_info = utils.get_version_info(cluster)
with utils.set_proxy(proxy_for_os_api):
data = helpers.get_info_from_os_resource_manager(
client_provider, resource_type)
oswl_statistics_save(cluster.id, resource_type, data,
version_info=version_info)
except errors.StatsException as e:
logger.error("Cannot collect OSWL resource {0} for cluster "
"with id {1}. Details: {2}."
.format(resource_type,
cluster.id,
six.text_type(e))
)
except Exception as e:
logger.exception("Error while collecting OSWL resource {0} "
"for cluster with id {1}. Details: {2}."
.format(resource_type,
cluster.id,
six.text_type(e))
)
db.commit()
except Exception as e:
logger.exception("Exception while collecting OS workloads "
"for resource name {0}. Details: {1}"
.format(resource_type, six.text_type(e)))
finally:
db.remove()
def test_oswl_statistics_save_version_info(self):
cluster = self.env.create()
# Without version info
oswl_statistics_save(cluster.id, consts.OSWL_RESOURCE_TYPES.vm, [])
oswl = OpenStackWorkloadStats.get_last_by(
cluster.id, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual({}, oswl.version_info)
# With version info
oswl_statistics_save(
cluster.id, consts.OSWL_RESOURCE_TYPES.vm, [{'id': 1}],
version_info=utils.get_version_info(cluster)
)
oswl = OpenStackWorkloadStats.get_last_by(
cluster.id, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual(utils.get_version_info(cluster), oswl.version_info)
def test_oswl_statistics_save_version_info(self):
cluster = self.env.create()
# Without version info
oswl_statistics_save(cluster.id, consts.OSWL_RESOURCE_TYPES.vm, [])
oswl = OpenStackWorkloadStats.get_last_by(
cluster.id, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual({}, oswl.version_info)
# With version info
oswl_statistics_save(cluster.id,
consts.OSWL_RESOURCE_TYPES.vm, [{
'id': 1
}],
version_info=utils.get_version_info(cluster))
oswl = OpenStackWorkloadStats.get_last_by(
cluster.id, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual(utils.get_version_info(cluster), oswl.version_info)
def test_oswl_is_sent_restored_on_changes(self):
cluster_id = 1
vm_info = {
"id": 1,
"power_state": 1,
}
oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm,
[vm_info])
last = OpenStackWorkloadStats.get_last_by(
cluster_id, consts.OSWL_RESOURCE_TYPES.vm)
# Setting is_sent to True
OpenStackWorkloadStats.update(last, {'is_sent': True})
self.assertEqual(True, last.is_sent)
# Checking is_sent is not changed if data is not changed
oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm,
[vm_info])
last_no_change = OpenStackWorkloadStats.get_last_by(
cluster_id, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual(True, last_no_change.is_sent)
# Checking is_sent is changed if data is changed
vm_info["power_state"] += 1
oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm,
[vm_info])
last_changed = OpenStackWorkloadStats.get_last_by(
cluster_id, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual(False, last_changed.is_sent)
def test_oswl_is_sent_restored_on_changes(self):
cluster_id = 1
vm_info = {
"id": 1,
"power_state": 1,
}
oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm,
[vm_info])
last = OpenStackWorkloadStats.get_last_by(
cluster_id, consts.OSWL_RESOURCE_TYPES.vm)
# Setting is_sent to True
OpenStackWorkloadStats.update(last, {'is_sent': True})
self.assertEqual(True, last.is_sent)
# Checking is_sent is not changed if data is not changed
oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm,
[vm_info])
last_no_change = OpenStackWorkloadStats.get_last_by(
cluster_id, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual(True, last_no_change.is_sent)
# Checking is_sent is changed if data is changed
vm_info["power_state"] += 1
oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm,
[vm_info])
last_changed = OpenStackWorkloadStats.get_last_by(
cluster_id, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual(False, last_changed.is_sent)
def test_add_row_per_day(self):
# VM is added
last = self.save_data_and_check_record([self.vms_info])
date_cur = last.created_date
time_update = last.updated_time
date_1st_rec = date_cur - datetime.timedelta(days=1)
# make existing record one day older
OpenStackWorkloadStats.update(last,
{'created_date': date_1st_rec})
# pass the same data
# no new record was created and existing one remains unchanged
self.assertEqual(last,
self.save_data_and_check_record([self.vms_info]))
# VM is removed
oswl_statistics_save(1, consts.OSWL_RESOURCE_TYPES.vm, [])
saved = self.check_overall_rec_count(2)
last = OpenStackWorkloadStats.get_last_by(
1, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual(last.created_date, date_cur)
for rec in saved:
if rec.created_date == date_cur:
self.assertEqual(rec, last)
# last record contains 'removed' and empty 'added'
data = self.empty_data
removed = dict(self.vms_info)
removed['time'] = last.updated_time.isoformat()
data['resource_data']['removed'] = [removed]
self.check_data_vs_rec(data, rec)
elif rec.created_date == date_1st_rec:
# first record contains 'added' and empty 'removed'
data = self.data_w_default_vm_info(time_update)
data['created_date'] = date_1st_rec
self.check_data_vs_rec(data, rec)
def test_add_row_per_day(self):
# VM is added
last = self.save_data_and_check_record([self.vms_info])
date_cur = last.created_date
time_update = last.updated_time
date_1st_rec = date_cur - datetime.timedelta(days=1)
# make existing record one day older
OpenStackWorkloadStats.update(last, {'created_date': date_1st_rec})
# pass the same data
# no new record was created and existing one remains unchanged
self.assertEqual(last,
self.save_data_and_check_record([self.vms_info]))
# VM is removed
oswl_statistics_save(1, consts.OSWL_RESOURCE_TYPES.vm, [])
saved = self.check_overall_rec_count(2)
last = OpenStackWorkloadStats.get_last_by(
1, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual(last.created_date, date_cur)
for rec in saved:
if rec.created_date == date_cur:
self.assertEqual(rec, last)
# last record contains 'removed' and empty 'added'
data = self.empty_data
removed = dict(self.vms_info)
removed['time'] = last.updated_time.isoformat()
data['resource_data']['removed'] = [removed]
self.check_data_vs_rec(data, rec)
elif rec.created_date == date_1st_rec:
# first record contains 'added' and empty 'removed'
data = self.data_w_default_vm_info(time_update)
data['created_date'] = date_1st_rec
self.check_data_vs_rec(data, rec)
def save_data_and_check_record(self, data):
oswl_statistics_save(1, consts.OSWL_RESOURCE_TYPES.vm, data)
last = OpenStackWorkloadStats.get_last_by(
1, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual(last, self.check_overall_rec_count(1).first())
return last
def save_data_and_check_record(self, data):
oswl_statistics_save(1, consts.OSWL_RESOURCE_TYPES.vm, data)
last = OpenStackWorkloadStats.get_last_by(
1, consts.OSWL_RESOURCE_TYPES.vm)
self.assertEqual(last, self.check_overall_rec_count(1).first())
return last
