def test_get_health_with_ignore_tenants(self, mock_get_projects):
self.override_config('collector', ignore_tenants=['project_2'])
mock_get_projects.return_value = [
{
'id': '111',
'name': 'project_1',
'description': ''
},
{
'id': '222',
'name': 'project_2',
'description': ''
},
]
# Insert projects in the database.
project_1_collect = datetime.utcnow() - timedelta(days=2)
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
}, project_1_collect)
project_2_collect = datetime.utcnow() - timedelta(hours=25)
db_api.project_add(
{
'id': '222',
'name': 'project_2',
'description': '',
}, project_2_collect)
ret = health.get_health()
self.assertEqual('FAIL', ret['usage_collection'].get('status'))
self.assertIn('1', ret['usage_collection'].get('msg'))
def test_get_health_ok(self, mock_get_projects):
mock_get_projects.return_value = [
{
'id': '111',
'name': 'project_1',
'description': ''
},
{
'id': '222',
'name': 'project_2',
'description': ''
},
]
# Insert projects in the database.
project_1_collect = datetime.utcnow() - timedelta(hours=1)
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
}, project_1_collect)
project_2_collect = datetime.utcnow() - timedelta(hours=2)
db_api.project_add(
{
'id': '222',
'name': 'project_2',
'description': '',
}, project_2_collect)
ret = health.get_health()
self.assertEqual('OK', ret['usage_collection'].get('status'))
def test_project_order_random(self, mock_get_lock, mock_get_projects,
mock_cclient):
self.override_config('collector', project_order='random')
mock_get_projects.return_value = [
{'id': '111', 'name': 'project_1', 'description': ''},
{'id': '222', 'name': 'project_2', 'description': ''},
{'id': '333', 'name': 'project_3', 'description': ''},
{'id': '444', 'name': 'project_4', 'description': ''},
]
# Insert a project in the database in order to get last_collect time.
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
},
datetime.utcnow() - timedelta(hours=2)
)
svc = collector.CollectorService()
svc.collector = mock.Mock()
svc.collect_usage()
unexpected_list = ['111', '222', '333', '444']
actual_list = [call_args[0][0]
for call_args in mock_get_lock.call_args_list]
self.assertNotEqual(unexpected_list, actual_list)
def test_collect_with_end_time(self, mock_get_projects, mock_cclient,
mock_kill):
end_time = datetime.utcnow() + timedelta(hours=0.5)
end_time_str = end_time.strftime(constants.iso_time)
self.override_config(collect_end_time=end_time_str)
mock_get_projects.return_value = [{
'id': '111',
'name': 'project_1',
'description': 'description'
}]
# Insert the project info in the database.
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
}, datetime.utcnow())
srv = collector.CollectorService()
srv.thread_grp = mock.Mock()
srv.collect_usage()
self.assertEqual(1, srv.thread_grp.stop.call_count)
self.assertEqual(1, mock_kill.call_count)
def test_collect_with_end_time(self, mock_get_projects, mock_cclient,
mock_kill):
end_time = datetime.utcnow() + timedelta(hours=0.5)
end_time_str = end_time.strftime(constants.iso_time)
self.override_config(collect_end_time=end_time_str)
mock_get_projects.return_value = [
{
'id': '111',
'name': 'project_1',
'description': 'description'
}
]
# Insert the project info in the database.
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
},
datetime.utcnow()
)
srv = collector.CollectorService()
srv.thread_grp = mock.Mock()
srv.collect_usage()
self.assertEqual(1, srv.thread_grp.stop.call_count)
self.assertEqual(1, mock_kill.call_count)
def test_last_collect_ignore_project(self, mock_get_projects, mock_cclient,
mock_collect_usage):
self.override_config('collector', ignore_tenants=['project_2'])
mock_get_projects.return_value = [
{'id': '111', 'name': 'project_1', 'description': ''},
{'id': '222', 'name': 'project_2', 'description': ''},
]
project1_time = datetime(2017, 5, 17, 20)
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
},
project1_time
)
project2_time = datetime(2017, 5, 17, 19)
db_api.project_add(
{
'id': '222',
'name': 'project_2',
'description': '',
},
project2_time
)
svc = collector.CollectorService()
svc.collect_usage()
mock_collect_usage.assert_called_once_with(
{'id': '111', 'name': 'project_1', 'description': ''},
[(project1_time, project1_time + timedelta(hours=1))]
)
def test_get_health_ok(self, mock_get_projects):
mock_get_projects.return_value = [
{'id': '111', 'name': 'project_1', 'description': ''},
{'id': '222', 'name': 'project_2', 'description': ''},
]
# Insert projects in the database.
project_1_collect = datetime.utcnow() - timedelta(hours=1)
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
},
project_1_collect
)
project_2_collect = datetime.utcnow() - timedelta(hours=2)
db_api.project_add(
{
'id': '222',
'name': 'project_2',
'description': '',
},
project_2_collect
)
ret = health.get_health()
self.assertEqual('OK', ret['usage_collection'].get('status'))
def test_get_health_with_ignore_tenants(self, mock_get_projects):
self.override_config('collector', ignore_tenants=['project_2'])
mock_get_projects.return_value = [
{'id': '111', 'name': 'project_1', 'description': ''},
{'id': '222', 'name': 'project_2', 'description': ''},
]
# Insert projects in the database.
project_1_collect = datetime.utcnow() - timedelta(days=2)
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
},
project_1_collect
)
project_2_collect = datetime.utcnow() - timedelta(hours=25)
db_api.project_add(
{
'id': '222',
'name': 'project_2',
'description': '',
},
project_2_collect
)
ret = health.get_health()
self.assertEqual('FAIL', ret['usage_collection'].get('status'))
self.assertIn('1', ret['usage_collection'].get('msg'))
def test_collect_swift_resource_id(self, mock_get_meter):
project_id = 'fake_project_id'
project_name = 'fake_project'
project = {'id': project_id, 'name': project_name}
start_time = datetime.strptime(
'2017-02-27 00:00:00',
"%Y-%m-%d %H:%M:%S"
)
end_time = datetime.strptime(
'2017-02-27 01:00:00',
"%Y-%m-%d %H:%M:%S"
)
# Add project to db in order to satisfy the foreign key constraint of
# UsageEntry
db_api.project_add(
{
'id': project_id,
'name': 'fake_project',
'description': 'project for test'
}
)
container_name = 'my_container'
resource_id = '%s/%s' % (project_id, container_name)
resource_id_hash = hashlib.md5(resource_id.encode('utf-8')).hexdigest()
mock_get_meter.return_value = [
{
'resource_id': resource_id,
'source': 'openstack',
'volume': 1024
}
]
collector = collector_base.BaseCollector()
collector.collect_usage(project, [(start_time, end_time)])
resources = db_api.resource_get_by_ids(project_id, [resource_id_hash])
res_info = json.loads(resources[0].info)
self.assertEqual(1, len(resources))
self.assertEqual(container_name, res_info['name'])
entries = db_api.usage_get(project_id, start_time, end_time)
self.assertEqual(1, len(entries))
self.assertEqual(resource_id_hash, entries[0].resource_id)
def test_project_order_random(self, mock_get_lock, mock_get_projects,
mock_cclient):
self.override_config('collector', project_order='random')
mock_get_projects.return_value = [
{
'id': '111',
'name': 'project_1',
'description': ''
},
{
'id': '222',
'name': 'project_2',
'description': ''
},
{
'id': '333',
'name': 'project_3',
'description': ''
},
{
'id': '444',
'name': 'project_4',
'description': ''
},
]
# Insert a project in the database in order to get last_collect time.
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
},
datetime.utcnow() - timedelta(hours=2))
svc = collector.CollectorService()
svc.collector = mock.Mock()
svc.collect_usage()
unexpected_list = ['111', '222', '333', '444']
actual_list = [
call_args[0][0] for call_args in mock_get_lock.call_args_list
]
self.assertNotEqual(unexpected_list, actual_list)
def test_get_health_fail(self, mock_get_projects):
mock_get_projects.return_value = [
{
'id': '111',
'name': 'project_1',
'description': ''
},
{
'id': '222',
'name': 'project_2',
'description': ''
},
]
# Insert projects in the database.
project_1_collect = datetime.utcnow() - timedelta(days=2)
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
}, project_1_collect)
project_2_collect = datetime.utcnow() - timedelta(hours=25)
db_api.project_add(
{
'id': '222',
'name': 'project_2',
'description': '',
}, project_2_collect)
ret = health.get_health()
projects = ret['usage_collection'].get('failed_projects')
self.assertIsNotNone(projects)
self.assertEqual(2, len(projects))
self.assertEqual('FAIL', ret['usage_collection'].get('status'))
self.assertIn('2', ret['usage_collection'].get('msg'))
p_names = [p['name'] for p in projects]
p_ids = [p['id'] for p in projects]
self.assertEqual(["project_1", "project_2"], p_names)
self.assertEqual(["111", "222"], p_ids)
def test_last_collect_new_project(self, mock_get_projects, mock_cclient,
mock_collect_usage):
# Assume project_2 is a new project that doesn't exist in distil db.
mock_get_projects.return_value = [
{
'id': '111',
'name': 'project_1',
'description': ''
},
{
'id': '222',
'name': 'project_2',
'description': ''
},
]
# Insert project_0 and project_1 in the database, project_0 is not in
# keystone anymore.
project_0_collect = datetime(2017, 5, 17, 19)
db_api.project_add(
{
'id': '000',
'name': 'project_0',
'description': 'deleted',
}, project_0_collect)
project_1_collect = datetime(2017, 5, 17, 20)
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
}, project_1_collect)
svc = collector.CollectorService()
svc.collect_usage()
self.assertEqual(2, mock_collect_usage.call_count)
mock_collect_usage.assert_called_with(
{
'id': '222',
'name': 'project_2',
'description': ''
}, [(project_1_collect, project_1_collect + timedelta(hours=1))])
def test_last_collect_ignore_project(self, mock_get_projects, mock_cclient,
mock_collect_usage):
self.override_config('collector', ignore_tenants=['project_2'])
mock_get_projects.return_value = [
{
'id': '111',
'name': 'project_1',
'description': ''
},
{
'id': '222',
'name': 'project_2',
'description': ''
},
]
project1_time = datetime(2017, 5, 17, 20)
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
}, project1_time)
project2_time = datetime(2017, 5, 17, 19)
db_api.project_add(
{
'id': '222',
'name': 'project_2',
'description': '',
}, project2_time)
svc = collector.CollectorService()
svc.collect_usage()
mock_collect_usage.assert_called_once_with(
{
'id': '111',
'name': 'project_1',
'description': ''
}, [(project1_time, project1_time + timedelta(hours=1))])
def test_collect_swift_resource_id(self, mock_get_meter):
project_id = 'fake_project_id'
project_name = 'fake_project'
project = {'id': project_id, 'name': project_name}
start_time = datetime.strptime('2017-02-27 00:00:00',
"%Y-%m-%d %H:%M:%S")
end_time = datetime.strptime('2017-02-27 01:00:00',
"%Y-%m-%d %H:%M:%S")
# Add project to db in order to satisfy the foreign key constraint of
# UsageEntry
db_api.project_add({
'id': project_id,
'name': 'fake_project',
'description': 'project for test'
})
container_name = 'my_container'
resource_id = '%s/%s' % (project_id, container_name)
resource_id_hash = hashlib.md5(resource_id.encode('utf-8')).hexdigest()
mock_get_meter.return_value = [{
'resource_id': resource_id,
'source': 'openstack',
'volume': 1024
}]
collector = collector_base.BaseCollector()
collector.collect_usage(project, [(start_time, end_time)])
resources = db_api.resource_get_by_ids(project_id, [resource_id_hash])
res_info = json.loads(resources[0].info)
self.assertEqual(1, len(resources))
self.assertEqual(container_name, res_info['name'])
entries = db_api.usage_get(project_id, start_time, end_time)
self.assertEqual(1, len(entries))
self.assertEqual(resource_id_hash, entries[0].resource_id)
def test_get_health_fail(self, mock_get_projects):
mock_get_projects.return_value = [
{'id': '111', 'name': 'project_1', 'description': ''},
{'id': '222', 'name': 'project_2', 'description': ''},
]
# Insert projects in the database.
project_1_collect = datetime.utcnow() - timedelta(days=2)
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
},
project_1_collect
)
project_2_collect = datetime.utcnow() - timedelta(hours=25)
db_api.project_add(
{
'id': '222',
'name': 'project_2',
'description': '',
},
project_2_collect
)
ret = health.get_health()
projects = ret['usage_collection'].get('failed_projects')
self.assertIsNotNone(projects)
self.assertEqual(2, len(projects))
self.assertEqual('FAIL', ret['usage_collection'].get('status'))
self.assertIn('2', ret['usage_collection'].get('msg'))
p_names = [p['name'] for p in projects]
p_ids = [p['id'] for p in projects]
self.assertEqual(["project_1", "project_2"], p_names)
self.assertEqual(["111", "222"], p_ids)
def test_last_collect_new_project(self, mock_get_projects, mock_cclient,
mock_collect_usage):
# Assume project_2 is a new project that doesn't exist in distil db.
mock_get_projects.return_value = [
{'id': '111', 'name': 'project_1', 'description': ''},
{'id': '222', 'name': 'project_2', 'description': ''},
]
# Insert project_0 and project_1 in the database, project_0 is not in
# keystone anymore.
project_0_collect = datetime(2017, 5, 17, 19)
db_api.project_add(
{
'id': '000',
'name': 'project_0',
'description': 'deleted',
},
project_0_collect
)
project_1_collect = datetime(2017, 5, 17, 20)
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
},
project_1_collect
)
svc = collector.CollectorService()
svc.collect_usage()
self.assertEqual(2, mock_collect_usage.call_count)
mock_collect_usage.assert_called_with(
{'id': '222', 'name': 'project_2', 'description': ''},
[(project_1_collect, project_1_collect + timedelta(hours=1))]
)