def collect(resource_type): try: operational_clusters = ClusterCollection.filter_by( iterable=None, status=consts.CLUSTER_STATUSES.operational).all() error_clusters = ClusterCollection.filter_by( iterable=None, status=consts.CLUSTER_STATUSES.error).all() all_envs_last_recs = \ OpenStackWorkloadStatsCollection.get_last_by_resource_type( resource_type) ready_or_error_ids = set([c.id for c in operational_clusters] + [c.id for c in error_clusters]) envs_ids_to_clear = set(r.cluster_id for r in all_envs_last_recs) - \ ready_or_error_ids # Clear current resource data for unavailable clusters. # Current OSWL data is cleared for those clusters which status is not # 'operational' nor 'error' or when cluster was removed. Data is # cleared for cluster only if it was updated recently (today or # yesterday). While this collector is running with interval much # smaller than one day it should not miss any unavailable cluster. for id in envs_ids_to_clear: oswl_statistics_save(id, resource_type, []) # Collect current OSWL data and update data in DB for cluster in operational_clusters: try: client_provider = helpers.ClientProvider(cluster) proxy_for_os_api = utils.get_proxy_for_cluster(cluster) version_info = utils.get_version_info(cluster) with utils.set_proxy(proxy_for_os_api): data = helpers.get_info_from_os_resource_manager( client_provider, resource_type) oswl_statistics_save(cluster.id, resource_type, data, version_info=version_info) except errors.StatsException as e: logger.error("Cannot collect OSWL resource {0} for cluster " "with id {1}. Details: {2}." .format(resource_type, cluster.id, six.text_type(e)) ) except Exception as e: logger.exception("Error while collecting OSWL resource {0} " "for cluster with id {1}. Details: {2}." .format(resource_type, cluster.id, six.text_type(e)) ) db.commit() except Exception as e: logger.exception("Exception while collecting OS workloads " "for resource name {0}. Details: {1}" .format(resource_type, six.text_type(e))) finally: db.remove()
def test_oswl_statistics_save_version_info(self): cluster = self.env.create() # Without version info oswl_statistics_save(cluster.id, consts.OSWL_RESOURCE_TYPES.vm, []) oswl = OpenStackWorkloadStats.get_last_by( cluster.id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual({}, oswl.version_info) # With version info oswl_statistics_save( cluster.id, consts.OSWL_RESOURCE_TYPES.vm, [{'id': 1}], version_info=utils.get_version_info(cluster) ) oswl = OpenStackWorkloadStats.get_last_by( cluster.id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(utils.get_version_info(cluster), oswl.version_info)
def test_oswl_statistics_save_version_info(self): cluster = self.env.create() # Without version info oswl_statistics_save(cluster.id, consts.OSWL_RESOURCE_TYPES.vm, []) oswl = OpenStackWorkloadStats.get_last_by( cluster.id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual({}, oswl.version_info) # With version info oswl_statistics_save(cluster.id, consts.OSWL_RESOURCE_TYPES.vm, [{ 'id': 1 }], version_info=utils.get_version_info(cluster)) oswl = OpenStackWorkloadStats.get_last_by( cluster.id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(utils.get_version_info(cluster), oswl.version_info)
def test_oswl_is_sent_restored_on_changes(self): cluster_id = 1 vm_info = { "id": 1, "power_state": 1, } oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm, [vm_info]) last = OpenStackWorkloadStats.get_last_by( cluster_id, consts.OSWL_RESOURCE_TYPES.vm) # Setting is_sent to True OpenStackWorkloadStats.update(last, {'is_sent': True}) self.assertEqual(True, last.is_sent) # Checking is_sent is not changed if data is not changed oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm, [vm_info]) last_no_change = OpenStackWorkloadStats.get_last_by( cluster_id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(True, last_no_change.is_sent) # Checking is_sent is changed if data is changed vm_info["power_state"] += 1 oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm, [vm_info]) last_changed = OpenStackWorkloadStats.get_last_by( cluster_id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(False, last_changed.is_sent)
def test_add_row_per_day(self): # VM is added last = self.save_data_and_check_record([self.vms_info]) date_cur = last.created_date time_update = last.updated_time date_1st_rec = date_cur - datetime.timedelta(days=1) # make existing record one day older OpenStackWorkloadStats.update(last, {'created_date': date_1st_rec}) # pass the same data # no new record was created and existing one remains unchanged self.assertEqual(last, self.save_data_and_check_record([self.vms_info])) # VM is removed oswl_statistics_save(1, consts.OSWL_RESOURCE_TYPES.vm, []) saved = self.check_overall_rec_count(2) last = OpenStackWorkloadStats.get_last_by( 1, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(last.created_date, date_cur) for rec in saved: if rec.created_date == date_cur: self.assertEqual(rec, last) # last record contains 'removed' and empty 'added' data = self.empty_data removed = dict(self.vms_info) removed['time'] = last.updated_time.isoformat() data['resource_data']['removed'] = [removed] self.check_data_vs_rec(data, rec) elif rec.created_date == date_1st_rec: # first record contains 'added' and empty 'removed' data = self.data_w_default_vm_info(time_update) data['created_date'] = date_1st_rec self.check_data_vs_rec(data, rec)
def save_data_and_check_record(self, data): oswl_statistics_save(1, consts.OSWL_RESOURCE_TYPES.vm, data) last = OpenStackWorkloadStats.get_last_by( 1, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(last, self.check_overall_rec_count(1).first()) return last