def test_oswl_is_sent_restored_on_changes(self): cluster_id = 1 vm_info = { "id": 1, "power_state": 1, } oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm, [vm_info]) last = OpenStackWorkloadStats.get_last_by( cluster_id, consts.OSWL_RESOURCE_TYPES.vm) # Setting is_sent to True OpenStackWorkloadStats.update(last, {'is_sent': True}) self.assertEqual(True, last.is_sent) # Checking is_sent is not changed if data is not changed oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm, [vm_info]) last_no_change = OpenStackWorkloadStats.get_last_by( cluster_id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(True, last_no_change.is_sent) # Checking is_sent is changed if data is changed vm_info["power_state"] += 1 oswl_statistics_save(cluster_id, consts.OSWL_RESOURCE_TYPES.vm, [vm_info]) last_changed = OpenStackWorkloadStats.get_last_by( cluster_id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(False, last_changed.is_sent)
def test_skip_collection_for_errorful_cluster(self, get_info_mock, *_): error_cluster = self.env.create( api=False, nodes_kwargs=[{"roles": ["controller"], "online": False}], cluster_kwargs={"name": "error", "status": consts.CLUSTER_STATUSES.operational} ) normal_cluster = self.env.create( api=False, nodes_kwargs=[{"roles": ["controller"], "online": True}], cluster_kwargs={"name": "normal", "status": consts.CLUSTER_STATUSES.operational} ) get_info_mock.return_value = self.vms_info oswl_collect_once(consts.OSWL_RESOURCE_TYPES.vm) last_for_error_clsr = OpenStackWorkloadStats.get_last_by( error_cluster["id"], consts.OSWL_RESOURCE_TYPES.vm) self.assertIsNone(last_for_error_clsr) last_for_normal_clsr = OpenStackWorkloadStats.get_last_by( normal_cluster["id"], consts.OSWL_RESOURCE_TYPES.vm) self.assertIsNotNone(last_for_normal_clsr) upd_time = last_for_normal_clsr.updated_time res_data = { 'added': [{'time': upd_time.isoformat(), 'id': 1}], 'removed': [], 'modified': [], 'current': self.vms_info} self.assertEqual(last_for_normal_clsr.resource_data, res_data)
def test_version_info_serialized(self): version_info = {'release': '9.0'} dt = datetime.datetime.utcnow() obj = OpenStackWorkloadStats.create( { 'cluster_id': 1, 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, 'created_date': dt.date(), 'updated_time': dt.time(), 'resource_checksum': "", 'version_info': version_info } ) self.assertEqual( version_info, OpenStackWorkloadStats.to_dict(obj)['version_info'] )
def test_oswl_statistics_save_version_info(self): cluster = self.env.create() # Without version info oswl_statistics_save(cluster.id, consts.OSWL_RESOURCE_TYPES.vm, []) oswl = OpenStackWorkloadStats.get_last_by( cluster.id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual({}, oswl.version_info) # With version info oswl_statistics_save( cluster.id, consts.OSWL_RESOURCE_TYPES.vm, [{'id': 1}], version_info=utils.get_version_info(cluster) ) oswl = OpenStackWorkloadStats.get_last_by( cluster.id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(utils.get_version_info(cluster), oswl.version_info)
def test_oswl_send_todays_record(self, send_data_to_url): dt = datetime.datetime.utcnow() obj_data = { 'cluster_id': 1, 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, 'created_date': dt.date(), 'updated_time': dt.time(), 'resource_checksum': "" } obj = OpenStackWorkloadStats.create(obj_data) self.assertEqual( OpenStackWorkloadStats.get_last_by(1, consts.OSWL_RESOURCE_TYPES.vm), obj) StatsSender().send_oswl_info() self.assertEqual(send_data_to_url.call_count, 1)
def test_skip_collection_for_errorful_cluster(self, get_info_mock, *_): error_cluster = self.env.create(api=False, nodes_kwargs=[{ "roles": ["controller"], "online": False }], cluster_kwargs={ "name": "error", "status": consts.CLUSTER_STATUSES.operational }) normal_cluster = self.env.create( api=False, nodes_kwargs=[{ "roles": ["controller"], "online": True }], cluster_kwargs={ "name": "normal", "status": consts.CLUSTER_STATUSES.operational }) get_info_mock.return_value = self.vms_info oswl_collect_once(consts.OSWL_RESOURCE_TYPES.vm) last_for_error_clsr = OpenStackWorkloadStats.get_last_by( error_cluster["id"], consts.OSWL_RESOURCE_TYPES.vm) self.assertIsNone(last_for_error_clsr) last_for_normal_clsr = OpenStackWorkloadStats.get_last_by( normal_cluster["id"], consts.OSWL_RESOURCE_TYPES.vm) self.assertIsNotNone(last_for_normal_clsr) upd_time = last_for_normal_clsr.updated_time res_data = { 'added': [{ 'time': upd_time.isoformat(), 'id': 1 }], 'removed': [], 'modified': [], 'current': self.vms_info } self.assertEqual(last_for_normal_clsr.resource_data, res_data)
def test_oswl_get_last_by_cluster_id_resource_type(self): cluster = self.env.create(nodes_kwargs=[{ 'roles': ['compute'] }, { 'roles': ['compute'] }, { 'roles': ['controller'] }]) dt = datetime.datetime.utcnow() obj_data = { 'cluster_id': cluster['id'], 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, 'created_date': dt.date(), 'updated_time': dt.time(), 'resource_checksum': "" } obj = OpenStackWorkloadStats.create(obj_data) self.assertEqual( OpenStackWorkloadStats.get_last_by(cluster['id'], consts.OSWL_RESOURCE_TYPES.vm), obj) self.assertIsNone( OpenStackWorkloadStats.get_last_by(0, consts.OSWL_RESOURCE_TYPES.vm)) self.assertIsNone( OpenStackWorkloadStats.get_last_by( cluster['id'], consts.OSWL_RESOURCE_TYPES.tenant)) OpenStackWorkloadStats.delete(obj) self.assertIsNone( OpenStackWorkloadStats.get_last_by(cluster['id'], consts.OSWL_RESOURCE_TYPES.vm))
def test_oswl_get_last_by_cluster_id_resource_type(self): cluster_id = 1 dt = datetime.datetime.utcnow() obj_data = { 'cluster_id': cluster_id, 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, 'created_date': dt.date(), 'updated_time': dt.time(), 'resource_checksum': "" } obj = OpenStackWorkloadStats.create(obj_data) self.assertEqual( OpenStackWorkloadStats.get_last_by( cluster_id, consts.OSWL_RESOURCE_TYPES.vm), obj ) self.assertIsNone( OpenStackWorkloadStats.get_last_by( 0, consts.OSWL_RESOURCE_TYPES.vm) ) self.assertIsNone( OpenStackWorkloadStats.get_last_by( cluster_id, consts.OSWL_RESOURCE_TYPES.tenant) ) OpenStackWorkloadStats.delete(obj) self.assertIsNone( OpenStackWorkloadStats.get_last_by( cluster_id, consts.OSWL_RESOURCE_TYPES.vm) )
def test_oswl_get_last_by_cluster_id_resource_type(self): cluster_id = 1 dt = datetime.datetime.utcnow() obj_data = { 'cluster_id': cluster_id, 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, 'created_date': dt.date(), 'updated_time': dt.time(), 'resource_checksum': "" } obj = OpenStackWorkloadStats.create(obj_data) self.assertEqual( OpenStackWorkloadStats.get_last_by(cluster_id, consts.OSWL_RESOURCE_TYPES.vm), obj) self.assertIsNone( OpenStackWorkloadStats.get_last_by(0, consts.OSWL_RESOURCE_TYPES.vm)) self.assertIsNone( OpenStackWorkloadStats.get_last_by( cluster_id, consts.OSWL_RESOURCE_TYPES.tenant)) OpenStackWorkloadStats.delete(obj) self.assertIsNone( OpenStackWorkloadStats.get_last_by(cluster_id, consts.OSWL_RESOURCE_TYPES.vm))
def test_oswl_statistics_save_version_info(self): cluster = self.env.create() # Without version info oswl_statistics_save(cluster.id, consts.OSWL_RESOURCE_TYPES.vm, []) oswl = OpenStackWorkloadStats.get_last_by( cluster.id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual({}, oswl.version_info) # With version info oswl_statistics_save(cluster.id, consts.OSWL_RESOURCE_TYPES.vm, [{ 'id': 1 }], version_info=utils.get_version_info(cluster)) oswl = OpenStackWorkloadStats.get_last_by( cluster.id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(utils.get_version_info(cluster), oswl.version_info)
def test_oswl_send_todays_record(self, send_data_to_url): dt = datetime.datetime.utcnow() obj_data = { 'cluster_id': 1, 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, 'created_date': dt.date(), 'updated_time': dt.time(), 'resource_checksum': "" } obj = OpenStackWorkloadStats.create(obj_data) self.assertEqual( OpenStackWorkloadStats.get_last_by( 1, consts.OSWL_RESOURCE_TYPES.vm), obj ) StatsSender().send_oswl_info() self.assertEqual(send_data_to_url.call_count, 1)
def test_oswl_nothing_to_send(self, send_data_to_url): dt = datetime.datetime.utcnow() obj_data = { 'cluster_id': 1, 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, 'created_date': dt.date(), 'updated_time': dt.time(), 'resource_checksum': "" } obj = OpenStackWorkloadStats.create(obj_data) self.assertEqual( OpenStackWorkloadStats.get_last_by(1, consts.OSWL_RESOURCE_TYPES.vm), obj) StatsSender().send_oswl_info() # Nothing to send as it doesn't send today's records. Today's are not # sent as they are not complete and can be updated during the day. self.assertEqual(send_data_to_url.call_count, 0)
def test_oswl_nothing_to_send(self, send_data_to_url): dt = datetime.datetime.utcnow() obj_data = { 'cluster_id': 1, 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, 'created_date': dt.date(), 'updated_time': dt.time(), 'resource_checksum': "" } obj = OpenStackWorkloadStats.create(obj_data) self.assertEqual( OpenStackWorkloadStats.get_last_by( 1, consts.OSWL_RESOURCE_TYPES.vm), obj ) StatsSender().send_oswl_info() # Nothing to send as it doesn't send today's records. Today's are not # sent as they are not complete and can be updated during the day. self.assertEqual(send_data_to_url.call_count, 0)
def test_removed_several_times(self, get_info_mock, *_): cls_id, res_data = self.collect_for_operational_cluster(get_info_mock) last = OpenStackWorkloadStats.get_last_by( cls_id, consts.OSWL_RESOURCE_TYPES.vm) self.assertItemsEqual(self.vms_info, last.resource_data['current']) # reset cluster get_info_mock.return_value = [] oswl_collect_once(consts.OSWL_RESOURCE_TYPES.vm) last = OpenStackWorkloadStats.get_last_by( cls_id, consts.OSWL_RESOURCE_TYPES.vm) removed = dict(self.vms_info[0]) removed['time'] = last.updated_time.isoformat() removed_data = [removed] # check data is not duplicated in removed on several collects for _ in xrange(10): oswl_collect_once(consts.OSWL_RESOURCE_TYPES.vm) last = OpenStackWorkloadStats.get_last_by( cls_id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(removed_data, last.resource_data['removed']) # cluster is operational # checking 'removed' is don't changed get_info_mock.return_value = self.vms_info oswl_collect_once(consts.OSWL_RESOURCE_TYPES.vm) last = OpenStackWorkloadStats.get_last_by( cls_id, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(removed_data, last.resource_data['removed']) # reset cluster again # checking only id and time added to 'removed' get_info_mock.return_value = [] oswl_collect_once(consts.OSWL_RESOURCE_TYPES.vm) last = OpenStackWorkloadStats.get_last_by( cls_id, consts.OSWL_RESOURCE_TYPES.vm) removed_data.append({ 'id': removed_data[0]['id'], 'time': last.updated_time.isoformat() }) self.assertListEqual(removed_data, last.resource_data['removed'])
def test_clear_data_for_removed_cluster(self, get_info_mock, *_): cls_id, res_data = self.collect_for_operational_cluster(get_info_mock) cls = Cluster.get_by_uid(cls_id) Cluster.delete(cls) oswl_collect_once(consts.OSWL_RESOURCE_TYPES.vm) last = OpenStackWorkloadStats.get_last_by( cls_id, consts.OSWL_RESOURCE_TYPES.vm) removed = dict(self.vms_info[0]) removed['time'] = last.updated_time.isoformat() res_data.update({'removed': [removed], 'current': []}) # current data is cleared when cluster is deleted self.assertEqual(last.resource_data, res_data)
def test_add_row_per_day(self): # VM is added last = self.save_data_and_check_record([self.vms_info]) date_cur = last.created_date time_update = last.updated_time date_1st_rec = date_cur - datetime.timedelta(days=1) # make existing record one day older OpenStackWorkloadStats.update(last, {'created_date': date_1st_rec}) # pass the same data # no new record was created and existing one remains unchanged self.assertEqual(last, self.save_data_and_check_record([self.vms_info])) # VM is removed oswl_statistics_save(1, consts.OSWL_RESOURCE_TYPES.vm, []) saved = self.check_overall_rec_count(2) last = OpenStackWorkloadStats.get_last_by( 1, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(last.created_date, date_cur) for rec in saved: if rec.created_date == date_cur: self.assertEqual(rec, last) # last record contains 'removed' and empty 'added' data = self.empty_data removed = dict(self.vms_info) removed['time'] = last.updated_time.isoformat() data['resource_data']['removed'] = [removed] self.check_data_vs_rec(data, rec) elif rec.created_date == date_1st_rec: # first record contains 'added' and empty 'removed' data = self.data_w_default_vm_info(time_update) data['created_date'] = date_1st_rec self.check_data_vs_rec(data, rec)
def test_clear_data_for_removed_cluster(self, get_info_mock, *_): cls_id, res_data = self.collect_for_operational_cluster(get_info_mock) cls = Cluster.get_by_uid(cls_id) Cluster.delete(cls) oswl_collect_once(consts.OSWL_RESOURCE_TYPES.vm) last = OpenStackWorkloadStats.get_last_by( cls_id, consts.OSWL_RESOURCE_TYPES.vm) removed = dict(self.vms_info[0]) removed['time'] = last.updated_time.isoformat() res_data.update({ 'removed': [removed], 'current': []}) # current data is cleared when cluster is deleted self.assertEqual(last.resource_data, res_data)
def collect_for_operational_cluster(self, get_info_mock): cluster = self.env.create_cluster( api=False, status=consts.CLUSTER_STATUSES.operational ) cls_id = cluster.id get_info_mock.return_value = self.vms_info oswl_collect_once(consts.OSWL_RESOURCE_TYPES.vm) last = OpenStackWorkloadStats.get_last_by( cls_id, consts.OSWL_RESOURCE_TYPES.vm) upd_time = last.updated_time res_data = { 'added': [{'time': upd_time.isoformat(), 'id': 1}], 'removed': [], 'modified': [], 'current': self.vms_info} self.assertEqual(last.resource_data, res_data) return cls_id, res_data
def collect_for_operational_cluster(self, get_info_mock): cluster = self.env.create_cluster( api=False, status=consts.CLUSTER_STATUSES.operational) cls_id = cluster.id get_info_mock.return_value = self.vms_info oswl_collect_once(consts.OSWL_RESOURCE_TYPES.vm) last = OpenStackWorkloadStats.get_last_by( cls_id, consts.OSWL_RESOURCE_TYPES.vm) upd_time = last.updated_time res_data = { 'added': [{ 'time': upd_time.isoformat(), 'id': 1 }], 'removed': [], 'modified': [], 'current': self.vms_info } self.assertEqual(last.resource_data, res_data) return cls_id, res_data
def test_clean_expired_entries(self): dt_now = datetime.datetime.utcnow() t_delta = datetime.timedelta(days=settings.OSWL_STORING_PERIOD) entries_to_del_cluster_ids = (1, 2) for cluster_id in entries_to_del_cluster_ids: obj_kwargs = { "cluster_id": cluster_id, "resource_type": consts.OSWL_RESOURCE_TYPES.volume, "updated_time": dt_now.time(), "created_date": dt_now.date() - t_delta, "resource_checksum": "" } OpenStackWorkloadStats.create(obj_kwargs) untouched_obj_kwargs = { "cluster_id": 3, "resource_type": consts.OSWL_RESOURCE_TYPES.vm, "updated_time": dt_now.time(), "created_date": dt_now.date(), "resource_checksum": "" } OpenStackWorkloadStats.create(untouched_obj_kwargs) OpenStackWorkloadStatsCollection.clean_expired_entries() self.db.commit() for cluster_id in entries_to_del_cluster_ids: instance = \ OpenStackWorkloadStats.get_last_by( cluster_id, consts.OSWL_RESOURCE_TYPES.volume ) self.assertIsNone(instance) untouched_obj = OpenStackWorkloadStats.get_last_by( untouched_obj_kwargs["cluster_id"], consts.OSWL_RESOURCE_TYPES.vm ) self.assertIsNotNone(untouched_obj)
def update_cluster_status_and_oswl_data(self, cls_id, status): cls = Cluster.get_by_uid(cls_id) Cluster.update(cls, {'status': status}) oswl_collect_once(consts.OSWL_RESOURCE_TYPES.vm) return OpenStackWorkloadStats.get_last_by( cls_id, consts.OSWL_RESOURCE_TYPES.vm)
def save_data_and_check_record(self, data): oswl_statistics_save(1, consts.OSWL_RESOURCE_TYPES.vm, data) last = OpenStackWorkloadStats.get_last_by( 1, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(last, self.check_overall_rec_count(1).first()) return last
def check_oswl_data_send_result(self, send_data_to_url, status, is_sent): # make yesterdays record (today's will not be sent) dt = datetime.datetime.utcnow() - datetime.timedelta(days=1) obj_data = { 'cluster_id': 1, 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, 'created_date': dt.date(), 'updated_time': dt.time(), 'resource_checksum': "" } obj = OpenStackWorkloadStats.create(obj_data) self.assertEqual( OpenStackWorkloadStats.get_last_by( 1, consts.OSWL_RESOURCE_TYPES.vm), obj ) rec_id = obj.id self.assertEqual(obj.is_sent, False) # emulate the answer from requests.post() class response(object): status_code = 200 data = { "status": "ok", "text": "ok", "oswl_stats": [{ "master_node_uid": "", "id": rec_id, "status": status }] } def __getitem__(self, key): return self.data[key] @classmethod def json(cls): return cls.data send_data_to_url.return_value = response sender = StatsSender() sender.send_oswl_info() obj_data_sent = {'oswl_stats': [{ 'id': rec_id, 'cluster_id': 1, 'created_date': dt.date().isoformat(), 'updated_time': dt.time().isoformat(), 'resource_type': 'vm', 'resource_checksum': '', 'master_node_uid': None, 'resource_data': None, }]} send_data_to_url.assert_called_once_with( url=sender.build_collector_url("COLLECTOR_OSWL_INFO_URL"), data=obj_data_sent) obj = OpenStackWorkloadStats.get_last_by( 1, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(obj.is_sent, is_sent) OpenStackWorkloadStats.delete(obj) send_data_to_url.reset_mock()
def check_oswl_data_send_result(self, send_data_to_url, status, is_sent): # make yesterdays record (today's will not be sent) dt = datetime.datetime.utcnow() - datetime.timedelta(days=1) obj_data = { 'cluster_id': 1, 'resource_type': consts.OSWL_RESOURCE_TYPES.vm, 'created_date': dt.date(), 'updated_time': dt.time(), 'resource_checksum': "" } obj = OpenStackWorkloadStats.create(obj_data) self.assertEqual( OpenStackWorkloadStats.get_last_by(1, consts.OSWL_RESOURCE_TYPES.vm), obj) rec_id = obj.id self.assertEqual(obj.is_sent, False) # emulate the answer from requests.post() class response(object): status_code = 200 data = { "status": "ok", "text": "ok", "oswl_stats": [{ "master_node_uid": "", "id": rec_id, "status": status }] } def __getitem__(self, key): return self.data[key] @classmethod def json(cls): return cls.data send_data_to_url.return_value = response sender = StatsSender() sender.send_oswl_info() obj_data_sent = { 'oswl_stats': [{ 'id': rec_id, 'cluster_id': 1, 'created_date': dt.date().isoformat(), 'updated_time': dt.time().isoformat(), 'resource_type': 'vm', 'resource_checksum': '', 'master_node_uid': None, 'resource_data': None, }] } send_data_to_url.assert_called_once_with( url=sender.build_collector_url("COLLECTOR_OSWL_INFO_URL"), data=obj_data_sent) obj = OpenStackWorkloadStats.get_last_by(1, consts.OSWL_RESOURCE_TYPES.vm) self.assertEqual(obj.is_sent, is_sent) OpenStackWorkloadStats.delete(obj) send_data_to_url.reset_mock()