def setUp(self):
"""Create fake pickle objects to be used as cache"""
self.tmpdir = tempfile.mkdtemp()
os.mkdir(self.tmpdir + os.path.sep + 'keystone')
os.mkdir(self.tmpdir + os.path.sep + 'region1')
os.mkdir(self.tmpdir + os.path.sep + 'region2')
sample_dict = {id: 'value1'}
for resource in self.keystone_objects:
with open(self.tmpdir + '/keystone/' + resource + '.pickle',
'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
for resource in OpenStackMap.resources_region:
with open(self.tmpdir + '/region1/' + resource + '.pickle',
'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
with open(self.tmpdir + '/region2/vms.pickle', 'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
self.map = OpenStackMap(
self.tmpdir,
region='region1',
auto_load=False,
objects_strategy=OpenStackMap.USE_CACHE_OBJECTS_ONLY)
def test_load_nova2(self):
"""test_load_nova check that we could build a map from nova resources using Direct_objects directive."""
openstackmap = OpenStackMap(
auto_load=False, objects_strategy=OpenStackMap.DIRECT_OBJECTS)
openstackmap.load_nova()
self.assertIsNotNone(openstackmap)
def test_load_cinder(self):
"""test_load_cinder check that we could build a map from cinder resources using Direct_objects directive."""
environ.setdefault('KEYSTONE_ADMIN_ENDPOINT', self.OS_AUTH_URL)
openstackmap = OpenStackMap(auto_load=False, objects_strategy=OpenStackMap.DIRECT_OBJECTS)
openstackmap.load_cinder()
self.assertIsNotNone(openstackmap)
def __init__(self, cache_dir, regions=None, offline_mode=False):
"""Constructor
It also build the sets about users and tenants
:param cache_dir: the directory where the data is cached.
:param regions: a list with the regions whose maps are preload. If None
only the current region.
:param offline_mode: if True, never connect with servers, use only the
cached data.
"""
self.logger = logging.getLogger(__name__)
if offline_mode:
strategy = OpenStackMap.USE_CACHE_OBJECTS_ONLY
else:
strategy = OpenStackMap.USE_CACHE_OBJECTS
if regions:
self.map = OpenStackMap(cache_dir, objects_strategy=strategy, auto_load=False)
self.map.preload_regions(regions)
else:
self.map = OpenStackMap(cache_dir, objects_strategy=strategy)
# This groups should be disjoint
self.admin_users = set()
self.trial_users = set()
self.community_users = set()
self.basic_users = set()
self.other_users = set()
self.not_found_users = set()
# Broken users can be of any of the other types (not found, basic...)
self.broken_users = set()
self.user_cloud_projects = set()
self.user_default_projects = set()
self.comtrialadmin_cloud_projects = set()
self.basic_cloud_projects = set()
self.community_cloud_projects = set()
self.trial_cloud_projects = set()
self.admin_cloud_projects = set()
self.other_users_cloud_projects = set()
self.not_found_cloud_projects = set()
self.broken_users_projects = set()
self._process_users()
# Read all the filters
self.filter_by_region = dict()
self.empty_filter = None
for filter in self.map.filters.values():
filter_cond = filter['filters']
if not filter_cond:
self.empty_filter = filter.id
elif 'region_id' in filter_cond:
self.filter_by_region[filter_cond['region_id']] = filter.id
if regions:
self.regions = regions
else:
self.regions = [self.map.osclients.region]
def test_load_cinder(self):
"""test_load_cinder check that we could build a map from cinder resources using Direct_objects directive."""
environ.setdefault('KEYSTONE_ADMIN_ENDPOINT', self.OS_AUTH_URL)
openstackmap = OpenStackMap(
auto_load=False, objects_strategy=OpenStackMap.DIRECT_OBJECTS)
openstackmap.load_cinder()
self.assertIsNotNone(openstackmap)
def __init__(self):
"""constructor"""
self.logger = logging.getLogger(__name__)
OpenStackMap.load_filters = False
osmap = OpenStackMap(objects_strategy=OpenStackMap.NO_CACHE_OBJECTS,
auto_load=False)
osmap.load_keystone()
osmap.load_neutron()
self.map = osmap
self.neutron = self.map.osclients.get_neutronclient()
def get_email_osclient(username, password, region):
"""
Get the list of user of one region taking into account a bottom-up analysis.
:param username: The name of the admin user that launch the request.
:param password: The password of the admin user.
:param region: The region in which we want to obtain the data.
:return: The emaillist.
"""
print("Making analysis bottom-up...")
# Set environment variables
os.environ['OS_USERNAME'] = username
os.environ['OS_PASSWORD'] = password
os.environ['OS_TENANT_NAME'] = 'admin'
os.environ['OS_REGION_NAME'] = region
os.environ['KEYSTONE_ADMIN_ENDPOINT'] = 'http://cloud.lab.fiware.org:4730/'
os.environ['OS_AUTH_URL'] = 'http://cloud.lab.fiware.org:4730/v2.0'
# load data from servers
map = OpenStackMap('tmp_cache', auto_load=False)
map.load_keystone()
# Get region filters and empty filter
regions_filters = dict()
empty_filter = None
for filter in map.filters.values():
if 'region_id' in filter['filters']:
regions_filters[filter['filters']['region_id']] = filter['id']
elif not filter['filters']:
empty_filter = filter['id']
useremail = dict()
# Get users. Genuine FIWARE Users should have cloud_project_id. Be aware that
# there are users without this field (administrators and also other users)
for user in map.users.values():
if 'cloud_project_id' not in user:
continue
project_id = user['cloud_project_id']
found = False
if project_id not in map.filters_by_project:
found = False
else:
for filter in map.filters_by_project[project_id]:
if filter == regions_filters[region] or filter == empty_filter:
found = True
break
if found:
useremail[user.id] = user.name
return useremail
def test_load_nova(self, mock_exists, mock_os):
"""test_load_nova check that we could build an empty map from nova resources."""
environ.setdefault('KEYSTONE_ADMIN_ENDPOINT', self.OS_AUTH_URL)
mock_exists.return_value = False
mock_os.return_value = True
openstackmap = OpenStackMap(auto_load=False, objects_strategy=OpenStackMap.NO_CACHE_OBJECTS)
openstackmap.load_nova()
self.assertTrue(mock_os.called)
self.assertIsNotNone(openstackmap)
def test_load_nova(self, mock_exists, mock_os):
"""test_load_nova check that we could build an empty map from nova resources."""
environ.setdefault('KEYSTONE_ADMIN_ENDPOINT', self.OS_AUTH_URL)
mock_exists.return_value = False
mock_os.return_value = True
openstackmap = OpenStackMap(
auto_load=False, objects_strategy=OpenStackMap.NO_CACHE_OBJECTS)
openstackmap.load_nova()
self.assertTrue(mock_os.called)
self.assertIsNotNone(openstackmap)
def test_implement_openstackmap_without_region_name(self):
"""test_implement_openstackmap_without_region_name check that we could not build an empty map without providing
a region Name."""
del os.environ['OS_REGION_NAME']
with self.assertRaises(Exception):
OpenStackMap(auth_url=self.OS_AUTH_URL, auto_load=False)
def test_implement_openstackmap(self, mock_exists, mock_os):
"""test_implement_openstackmap check that we could build an empty map from the resources (VMs, networks, images,
volumes, users, tenants, roles...) in an OpenStack infrastructure."""
mock_exists.return_value = False
mock_os.return_value = True
openstackmap = OpenStackMap(region=self.OS_REGION_NAME,
auto_load=False)
self.assertTrue(mock_os.called)
self.assertIsNotNone(openstackmap)
def test_implement_openstackmap_with_keystone_admin_endpoint(
self, mock_exists, mock_os):
"""test_implement_openstackmap_with_keystone_admin_endpoint check that we could build an empty map from the
resources providing a keystone admin endpoint environment variable."""
environ.setdefault('KEYSTONE_ADMIN_ENDPOINT', self.OS_AUTH_URL)
mock_exists.return_value = False
mock_os.return_value = True
openstackmap = OpenStackMap(auto_load=False)
self.assertTrue(mock_os.called)
self.assertIsNotNone(openstackmap)
def __init__(self):
"""constructor"""
self.logger = logging.getLogger(__name__)
OpenStackMap.load_filters = False
osmap = OpenStackMap(objects_strategy=OpenStackMap.NO_CACHE_OBJECTS,
auto_load=False)
osmap.load_keystone()
osmap.load_neutron()
self.map = osmap
self.neutron = self.map.osclients.get_neutronclient()
def setUp(self):
"""Create fake pickle objects to be used as cache"""
self.tmpdir = tempfile.mkdtemp()
os.mkdir(self.tmpdir + os.path.sep + 'keystone')
os.mkdir(self.tmpdir + os.path.sep + 'region1')
os.mkdir(self.tmpdir + os.path.sep + 'region2')
sample_dict = {id: 'value1'}
for resource in self.keystone_objects:
with open(self.tmpdir + '/keystone/' + resource + '.pickle', 'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
for resource in OpenStackMap.resources_region:
with open(self.tmpdir + '/region1/' + resource + '.pickle', 'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
with open(self.tmpdir + '/region2/vms.pickle', 'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
self.map = OpenStackMap(
self.tmpdir, region='region1', auto_load=False, objects_strategy=OpenStackMap.USE_CACHE_OBJECTS_ONLY)
class TestOpenstackMapCacheOnly(TestCase):
keystone_objects = ['users', 'users_by_name', 'tenants', 'tenants_by_name', 'roles_a',
'filters', 'filters_by_project', 'roles', 'roles_by_user', 'roles_by_project']
def setUp(self):
"""Create fake pickle objects to be used as cache"""
self.tmpdir = tempfile.mkdtemp()
os.mkdir(self.tmpdir + os.path.sep + 'keystone')
os.mkdir(self.tmpdir + os.path.sep + 'region1')
os.mkdir(self.tmpdir + os.path.sep + 'region2')
sample_dict = {id: 'value1'}
for resource in self.keystone_objects:
with open(self.tmpdir + '/keystone/' + resource + '.pickle', 'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
for resource in OpenStackMap.resources_region:
with open(self.tmpdir + '/region1/' + resource + '.pickle', 'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
with open(self.tmpdir + '/region2/vms.pickle', 'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
self.map = OpenStackMap(
self.tmpdir, region='region1', auto_load=False, objects_strategy=OpenStackMap.USE_CACHE_OBJECTS_ONLY)
def tearDown(self):
for dir in os.listdir(self.tmpdir):
for name in os.listdir(self.tmpdir + os.path.sep + dir):
print self.tmpdir + os.path.sep + dir + os.path.sep + name
os.unlink(self.tmpdir + os.path.sep + dir + os.path.sep + name)
os.rmdir(self.tmpdir + '/keystone')
os.rmdir(self.tmpdir + '/region1')
os.rmdir(self.tmpdir + '/region2')
os.rmdir(self.tmpdir)
def test_load_all(self):
"""test the load_all method"""
self.map.load_all()
for resource in OpenStackMap.resources_region:
data = getattr(self.map, resource)
self.assertTrue(data)
for resource in self.keystone_objects:
data = getattr(self.map, resource)
self.assertTrue(data)
def test_load_nova_region2(self):
"""test the load_nova in region2"""
self.map.change_region('region2', False)
self.map.load_nova()
self.assertTrue(self.map.vms)
def test_load_all_region2(self):
"""test the load_all in region2 (is invoked by change_region)"""
self.map.change_region('region2')
self.assertTrue(self.map.vms)
# region2 has vms, but not other resources as networks
self.assertFalse(self.map.networks)
def test_load_neutron_region2_failed(self):
"""test the load_neutron in region2: it must fail"""
self.map.change_region('region2', False)
with self.assertRaises(Exception):
self.map.load_neutron()
def test_preload_regions(self):
"""test the preload_regions method"""
self.map.preload_regions()
self.assertEquals(len(self.map.region_map), 2)
self.assertTrue(self.map.region_map['region2']['vms'])
class ClassifyResources(object):
"""Class to analyse the resources by owner, for discovering resources
belonging to users that should not have resources, or resources without
an owner"""
def __init__(self, cache_dir, regions=None, offline_mode=False):
"""Constructor
It also build the sets about users and tenants
:param cache_dir: the directory where the data is cached.
:param regions: a list with the regions whose maps are preload. If None
only the current region.
:param offline_mode: if True, never connect with servers, use only the
cached data.
"""
self.logger = logging.getLogger(__name__)
if offline_mode:
strategy = OpenStackMap.USE_CACHE_OBJECTS_ONLY
else:
strategy = OpenStackMap.USE_CACHE_OBJECTS
if regions:
self.map = OpenStackMap(cache_dir, objects_strategy=strategy, auto_load=False)
self.map.preload_regions(regions)
else:
self.map = OpenStackMap(cache_dir, objects_strategy=strategy)
# This groups should be disjoint
self.admin_users = set()
self.trial_users = set()
self.community_users = set()
self.basic_users = set()
self.other_users = set()
self.not_found_users = set()
# Broken users can be of any of the other types (not found, basic...)
self.broken_users = set()
self.user_cloud_projects = set()
self.user_default_projects = set()
self.comtrialadmin_cloud_projects = set()
self.basic_cloud_projects = set()
self.community_cloud_projects = set()
self.trial_cloud_projects = set()
self.admin_cloud_projects = set()
self.other_users_cloud_projects = set()
self.not_found_cloud_projects = set()
self.broken_users_projects = set()
self._process_users()
# Read all the filters
self.filter_by_region = dict()
self.empty_filter = None
for filter in self.map.filters.values():
filter_cond = filter['filters']
if not filter_cond:
self.empty_filter = filter.id
elif 'region_id' in filter_cond:
self.filter_by_region[filter_cond['region_id']] = filter.id
if regions:
self.regions = regions
else:
self.regions = [self.map.osclients.region]
def _process_users(self):
"""get information about all the users. This information is used
by the other methods, as print_users_summary"""
projects = set(self.map.tenants.keys())
for user in self.map.users.values():
if 'cloud_project_id' in user:
cloud_project_id = user.cloud_project_id
self.user_cloud_projects.add(cloud_project_id)
if user.cloud_project_id not in projects:
self.broken_users.add(user.id)
self.broken_users_projects.add(cloud_project_id)
else:
cloud_project_id = None
if 'default_project_id' in user:
self.user_default_projects.add(user.default_project_id)
if user.id not in self.map.roles_by_user:
self.not_found_users.add(user.id)
if cloud_project_id:
self.not_found_cloud_projects.add(cloud_project_id)
continue
roles = self.map.roles_by_user[user.id]
user_has_type = False
for rol in roles:
if rol[0] == BASIC_ROLE_ID:
user_has_type = True
self.basic_users.add(user.id)
if cloud_project_id:
self.basic_cloud_projects.add(cloud_project_id)
elif rol[0] == COMMUNITY_ROLE_ID:
user_has_type = True
self.community_users.add(user.id)
if cloud_project_id:
self.community_cloud_projects.add(cloud_project_id)
elif rol[0] == TRIAL_ROLE_ID:
user_has_type = True
self.trial_users.add(user.id)
if cloud_project_id:
self.trial_cloud_projects.add(cloud_project_id)
elif rol[0] == ADMIN_ROLE_ID:
user_has_type = True
self.admin_users.add(user.id)
# use default_project_id with admin users, because
# cloud_project_id does not exist.
if hasattr(user, 'default_project_id'):
self.admin_cloud_projects.add(user.default_project_id)
self.user_cloud_projects.add(cloud_project_id)
if not user_has_type:
self.other_users.add(user.id)
if cloud_project_id:
self.other_users_cloud_projects.add(cloud_project_id)
self.comtrialadmin_cloud_projects.update(self.community_cloud_projects)
self.comtrialadmin_cloud_projects.update(self.trial_cloud_projects)
self.comtrialadmin_cloud_projects.update(self.admin_cloud_projects)
if self.basic_users.intersection(self.admin_users):
self.logger.error('There are users both in admin and basic')
if self.basic_users.intersection(self.community_users):
msg = 'There are users both in community and basic'
self.logger.error(msg)
if self.basic_users.intersection(self.trial_users):
self.logger.error('There are users both in trial and basic')
if self.community_users.intersection(self.trial_users):
msg = 'There are users both in community and trial'
self.logger.error(msg)
if self.community_users.intersection(self.admin_users):
msg = 'There are users both in community and admin'
self.logger.error(msg)
if self.trial_users.intersection(self.admin_users):
self.logger.error('There are users both in admin and trial')
def print_users_summary(self):
"""print a summary of users by their type. Also includes a summary
by region of the number of community /trial users"""
print('\nTotal users: {}'.format(len(self.map.users)))
print('---- Users by type:')
print('Basic users: {}'.format(len(self.basic_users)))
print('Community users: {}'.format(len(self.community_users)))
print('Trial users: {}'.format(len(self.trial_users)))
print('Admin users: {}'.format(len(self.admin_users)))
print('Other type users: {}'.format(len(self.other_users)))
print('Users without type: {}'.format(len(self.not_found_users)))
print('----')
print('Users with a project-id that does not exist: {}\n'.format(
len(self.broken_users)))
for region in self.regions:
print('---Region: ' + region)
print('Community: {0}'.format(len(self.filter_projects(
self.community_cloud_projects, region))))
print('Trial: {0}'.format(len(self.filter_projects(
self.trial_cloud_projects, region))))
def classify_resource(self, member, region=None):
"""
This is a wrapper to classify_resource_raw. It locates the
resources by name and region, change the owner tenant_id of the
attribute when it has a different name and then call
classify_resource_raw. See the documentation of classify_resource_raw
:param member: the member. It can be: subnets, routers, networks, vms,
volume_backups, volumes, images, volume_snapshots,
ports, security_groups, floatingips
:param region: the region
:return: a tuple with three lists, see classify_resource_raw.
"""
# Resources where the tenant-id attribute has a different name
special_cases = {'images': 'owner',
'volumes': 'os-vol-tenant-attr:tenant_id',
'volume_snapshots':
'os-extended-snapshot-attributes:project_id',
'volume_backups':
'os-extended-snapshot-attributes:project_id'}
if region and self.map.osclients.region != region:
if region in self.map.region_map:
elements = self.map.region_map[region][member]
else:
self.map.change_region(region)
elements = getattr(self.map, member)
else:
elements = getattr(self.map, member)
region = self.map.osclients.region
if member in special_cases:
attr = special_cases[member]
for element in elements.values():
element['tenant_id'] = element[attr]
print('==Resources: {0}. Region: {1} '.format(member, region))
print('Total: {}'.format(len(elements)))
return self.classify_resource_raw(elements, region)
def classify_resource_raw(self, element_list, region_name):
"""
This method receives a dictionary of resources (e.g. vms, volumes...)
and prints a summary about who owned them:
*resources owned by legal users: community / admin / trial users
*resources with an unexpected owner: basic / other / unknown
*resources not using cloud_project_id / using default_project_id
*resources with a project id that does not exists.
It also returns a tuple with four list of elements:
*resources owned by community / admin / trial users but in the wrong
region (i.e. the project id is not authorised in that region)
*resources that are not owned by any community / admin / trial users
*resources that are not owned by any know user
*resources that whose tenant_id (project_id) does not exist.
The elements of the resource list must use tenant-id attribute to
identify the owner of the resource.
:param element_list: resources list to classify.
:param region_name: region of the resources. It is used for checking if
the project-id is authorised in that region.
:return: a tuple with four list of resources with some anomaly.
"""
owned = list()
unkown_owner = list()
unkown_tenant = list()
forbidden_region = list()
ok = 0
community = 0
trial = 0
admin = 0
basic = 0
other_type = 0
unknown_type = 0
default_project_id = 0
bad_region = 0
filtered = self.filter_projects(
self.comtrialadmin_cloud_projects, region_name)
for element in element_list.values():
tenant_id = element['tenant_id']
if tenant_id in self.comtrialadmin_cloud_projects:
if tenant_id in filtered:
ok += 1
if tenant_id in self.community_cloud_projects:
community += 1
elif tenant_id in self.trial_cloud_projects:
trial += 1
else:
admin += 1
else:
bad_region += 1
forbidden_region.append(element)
elif tenant_id in self.user_cloud_projects:
owned.append(element)
if tenant_id in self.basic_cloud_projects:
basic += 1
elif tenant_id in self.other_users_cloud_projects:
other_type += 1
else:
unknown_type += 1
elif tenant_id in self.map.tenants:
if tenant_id in self.user_default_projects:
default_project_id += 1
unkown_owner.append(element)
else:
unkown_tenant.append(element)
msg = 'All OK. Owned by users community/trial/admin: {0} ({1}/{2}/{3})'
print(msg.format(ok, community, trial, admin))
msg = 'Owned by users communtiy/trial/admin but bad region: {0}'
print(msg.format(bad_region))
msg = 'Owned by users basic/other type/unknown type: {0} ({1}/{2}/{3})'
print(msg.format(len(owned), basic, other_type, unknown_type))
m = 'Owned by another projects id: {0} (using default_project_id {1})'
print(m.format(len(unkown_owner), default_project_id))
print('Project id does not exist: {0}'.format(len(unkown_tenant)))
return forbidden_region, owned, unkown_owner, unkown_tenant
def filter_projects(self, projects, region):
"""Filter the list of projects: only are selected the projects which
are authorised in the specified region.
:param projects: a list of project ids
:param region: a region name
:return: a filtered set of project ids: only the project authorised to
use the region resources are included.
"""
projects_in_region = set()
fil_reg = self.filter_by_region[region]
for project_id in projects:
if project_id not in self.map.filters_by_project:
projects_in_region.add(project_id)
else:
for filter_id in self.map.filters_by_project[project_id]:
if filter_id == self.empty_filter or filter_id == fil_reg:
projects_in_region.add(project_id)
break
return projects_in_region
def __init__(self, cache_dir, regions=None, offline_mode=False):
"""Constructor
It also build the sets about users and tenants
:param cache_dir: the directory where the data is cached.
:param regions: a list with the regions whose maps are preload. If None
only the current region.
:param offline_mode: if True, never connect with servers, use only the
cached data.
"""
self.logger = logging.getLogger(__name__)
if offline_mode:
strategy = OpenStackMap.USE_CACHE_OBJECTS_ONLY
else:
strategy = OpenStackMap.USE_CACHE_OBJECTS
if regions:
self.map = OpenStackMap(cache_dir,
objects_strategy=strategy,
auto_load=False)
self.map.preload_regions(regions)
else:
self.map = OpenStackMap(cache_dir, objects_strategy=strategy)
# This groups should be disjoint
self.admin_users = set()
self.trial_users = set()
self.community_users = set()
self.basic_users = set()
self.other_users = set()
self.not_found_users = set()
# Broken users can be of any of the other types (not found, basic...)
self.broken_users = set()
self.user_cloud_projects = set()
self.user_default_projects = set()
self.comtrialadmin_cloud_projects = set()
self.basic_cloud_projects = set()
self.community_cloud_projects = set()
self.trial_cloud_projects = set()
self.admin_cloud_projects = set()
self.other_users_cloud_projects = set()
self.not_found_cloud_projects = set()
self.broken_users_projects = set()
self._process_users()
# Read all the filters
self.filter_by_region = dict()
self.empty_filter = None
for filter in self.map.filters.values():
filter_cond = filter['filters']
if not filter_cond:
self.empty_filter = filter.id
elif 'region_id' in filter_cond:
self.filter_by_region[filter_cond['region_id']] = filter.id
if regions:
self.regions = regions
else:
self.regions = [self.map.osclients.region]
class ClassifyResources(object):
"""Class to analyse the resources by owner, for discovering resources
belonging to users that should not have resources, or resources without
an owner"""
def __init__(self, cache_dir, regions=None, offline_mode=False):
"""Constructor
It also build the sets about users and tenants
:param cache_dir: the directory where the data is cached.
:param regions: a list with the regions whose maps are preload. If None
only the current region.
:param offline_mode: if True, never connect with servers, use only the
cached data.
"""
self.logger = logging.getLogger(__name__)
if offline_mode:
strategy = OpenStackMap.USE_CACHE_OBJECTS_ONLY
else:
strategy = OpenStackMap.USE_CACHE_OBJECTS
if regions:
self.map = OpenStackMap(cache_dir,
objects_strategy=strategy,
auto_load=False)
self.map.preload_regions(regions)
else:
self.map = OpenStackMap(cache_dir, objects_strategy=strategy)
# This groups should be disjoint
self.admin_users = set()
self.trial_users = set()
self.community_users = set()
self.basic_users = set()
self.other_users = set()
self.not_found_users = set()
# Broken users can be of any of the other types (not found, basic...)
self.broken_users = set()
self.user_cloud_projects = set()
self.user_default_projects = set()
self.comtrialadmin_cloud_projects = set()
self.basic_cloud_projects = set()
self.community_cloud_projects = set()
self.trial_cloud_projects = set()
self.admin_cloud_projects = set()
self.other_users_cloud_projects = set()
self.not_found_cloud_projects = set()
self.broken_users_projects = set()
self._process_users()
# Read all the filters
self.filter_by_region = dict()
self.empty_filter = None
for filter in self.map.filters.values():
filter_cond = filter['filters']
if not filter_cond:
self.empty_filter = filter.id
elif 'region_id' in filter_cond:
self.filter_by_region[filter_cond['region_id']] = filter.id
if regions:
self.regions = regions
else:
self.regions = [self.map.osclients.region]
def _process_users(self):
"""get information about all the users. This information is used
by the other methods, as print_users_summary"""
projects = set(self.map.tenants.keys())
for user in self.map.users.values():
if 'cloud_project_id' in user:
cloud_project_id = user.cloud_project_id
self.user_cloud_projects.add(cloud_project_id)
if user.cloud_project_id not in projects:
self.broken_users.add(user.id)
self.broken_users_projects.add(cloud_project_id)
else:
cloud_project_id = None
if 'default_project_id' in user:
self.user_default_projects.add(user.default_project_id)
if user.id not in self.map.roles_by_user:
self.not_found_users.add(user.id)
if cloud_project_id:
self.not_found_cloud_projects.add(cloud_project_id)
continue
roles = self.map.roles_by_user[user.id]
user_has_type = False
for rol in roles:
if rol[0] == BASIC_ROLE_ID:
user_has_type = True
self.basic_users.add(user.id)
if cloud_project_id:
self.basic_cloud_projects.add(cloud_project_id)
elif rol[0] == COMMUNITY_ROLE_ID:
user_has_type = True
self.community_users.add(user.id)
if cloud_project_id:
self.community_cloud_projects.add(cloud_project_id)
elif rol[0] == TRIAL_ROLE_ID:
user_has_type = True
self.trial_users.add(user.id)
if cloud_project_id:
self.trial_cloud_projects.add(cloud_project_id)
elif rol[0] == ADMIN_ROLE_ID:
user_has_type = True
self.admin_users.add(user.id)
# use default_project_id with admin users, because
# cloud_project_id does not exist.
if hasattr(user, 'default_project_id'):
self.admin_cloud_projects.add(user.default_project_id)
self.user_cloud_projects.add(cloud_project_id)
if not user_has_type:
self.other_users.add(user.id)
if cloud_project_id:
self.other_users_cloud_projects.add(cloud_project_id)
self.comtrialadmin_cloud_projects.update(self.community_cloud_projects)
self.comtrialadmin_cloud_projects.update(self.trial_cloud_projects)
self.comtrialadmin_cloud_projects.update(self.admin_cloud_projects)
if self.basic_users.intersection(self.admin_users):
self.logger.error('There are users both in admin and basic')
if self.basic_users.intersection(self.community_users):
msg = 'There are users both in community and basic'
self.logger.error(msg)
if self.basic_users.intersection(self.trial_users):
self.logger.error('There are users both in trial and basic')
if self.community_users.intersection(self.trial_users):
msg = 'There are users both in community and trial'
self.logger.error(msg)
if self.community_users.intersection(self.admin_users):
msg = 'There are users both in community and admin'
self.logger.error(msg)
if self.trial_users.intersection(self.admin_users):
self.logger.error('There are users both in admin and trial')
def print_users_summary(self):
"""print a summary of users by their type. Also includes a summary
by region of the number of community /trial users"""
print('\nTotal users: {}'.format(len(self.map.users)))
print('---- Users by type:')
print('Basic users: {}'.format(len(self.basic_users)))
print('Community users: {}'.format(len(self.community_users)))
print('Trial users: {}'.format(len(self.trial_users)))
print('Admin users: {}'.format(len(self.admin_users)))
print('Other type users: {}'.format(len(self.other_users)))
print('Users without type: {}'.format(len(self.not_found_users)))
print('----')
print('Users with a project-id that does not exist: {}\n'.format(
len(self.broken_users)))
for region in self.regions:
print('---Region: ' + region)
print('Community: {0}'.format(
len(self.filter_projects(self.community_cloud_projects,
region))))
print('Trial: {0}'.format(
len(self.filter_projects(self.trial_cloud_projects, region))))
def classify_resource(self, member, region=None):
"""
This is a wrapper to classify_resource_raw. It locates the
resources by name and region, change the owner tenant_id of the
attribute when it has a different name and then call
classify_resource_raw. See the documentation of classify_resource_raw
:param member: the member. It can be: subnets, routers, networks, vms,
volume_backups, volumes, images, volume_snapshots,
ports, security_groups, floatingips
:param region: the region
:return: a tuple with three lists, see classify_resource_raw.
"""
# Resources where the tenant-id attribute has a different name
special_cases = {
'images': 'owner',
'volumes': 'os-vol-tenant-attr:tenant_id',
'volume_snapshots': 'os-extended-snapshot-attributes:project_id',
'volume_backups': 'os-extended-snapshot-attributes:project_id'
}
if region and self.map.osclients.region != region:
if region in self.map.region_map:
elements = self.map.region_map[region][member]
else:
self.map.change_region(region)
elements = getattr(self.map, member)
else:
elements = getattr(self.map, member)
region = self.map.osclients.region
if member in special_cases:
attr = special_cases[member]
for element in elements.values():
element['tenant_id'] = element[attr]
print('==Resources: {0}. Region: {1} '.format(member, region))
print('Total: {}'.format(len(elements)))
return self.classify_resource_raw(elements, region)
def classify_resource_raw(self, element_list, region_name):
"""
This method receives a dictionary of resources (e.g. vms, volumes...)
and prints a summary about who owned them:
*resources owned by legal users: community / admin / trial users
*resources with an unexpected owner: basic / other / unknown
*resources not using cloud_project_id / using default_project_id
*resources with a project id that does not exists.
It also returns a tuple with four list of elements:
*resources owned by community / admin / trial users but in the wrong
region (i.e. the project id is not authorised in that region)
*resources that are not owned by any community / admin / trial users
*resources that are not owned by any know user
*resources that whose tenant_id (project_id) does not exist.
The elements of the resource list must use tenant-id attribute to
identify the owner of the resource.
:param element_list: resources list to classify.
:param region_name: region of the resources. It is used for checking if
the project-id is authorised in that region.
:return: a tuple with four list of resources with some anomaly.
"""
owned = list()
unkown_owner = list()
unkown_tenant = list()
forbidden_region = list()
ok = 0
community = 0
trial = 0
admin = 0
basic = 0
other_type = 0
unknown_type = 0
default_project_id = 0
bad_region = 0
filtered = self.filter_projects(self.comtrialadmin_cloud_projects,
region_name)
for element in element_list.values():
tenant_id = element['tenant_id']
if tenant_id in self.comtrialadmin_cloud_projects:
if tenant_id in filtered:
ok += 1
if tenant_id in self.community_cloud_projects:
community += 1
elif tenant_id in self.trial_cloud_projects:
trial += 1
else:
admin += 1
else:
bad_region += 1
forbidden_region.append(element)
elif tenant_id in self.user_cloud_projects:
owned.append(element)
if tenant_id in self.basic_cloud_projects:
basic += 1
elif tenant_id in self.other_users_cloud_projects:
other_type += 1
else:
unknown_type += 1
elif tenant_id in self.map.tenants:
if tenant_id in self.user_default_projects:
default_project_id += 1
unkown_owner.append(element)
else:
unkown_tenant.append(element)
msg = 'All OK. Owned by users community/trial/admin: {0} ({1}/{2}/{3})'
print(msg.format(ok, community, trial, admin))
msg = 'Owned by users communtiy/trial/admin but bad region: {0}'
print(msg.format(bad_region))
msg = 'Owned by users basic/other type/unknown type: {0} ({1}/{2}/{3})'
print(msg.format(len(owned), basic, other_type, unknown_type))
m = 'Owned by another projects id: {0} (using default_project_id {1})'
print(m.format(len(unkown_owner), default_project_id))
print('Project id does not exist: {0}'.format(len(unkown_tenant)))
return forbidden_region, owned, unkown_owner, unkown_tenant
def filter_projects(self, projects, region):
"""Filter the list of projects: only are selected the projects which
are authorised in the specified region.
:param projects: a list of project ids
:param region: a region name
:return: a filtered set of project ids: only the project authorised to
use the region resources are included.
"""
projects_in_region = set()
fil_reg = self.filter_by_region[region]
for project_id in projects:
if project_id not in self.map.filters_by_project:
projects_in_region.add(project_id)
else:
for filter_id in self.map.filters_by_project[project_id]:
if filter_id == self.empty_filter or filter_id == fil_reg:
projects_in_region.add(project_id)
break
return projects_in_region
class TestOpenstackMapCacheOnly(TestCase):
keystone_objects = [
'users', 'users_by_name', 'tenants', 'tenants_by_name', 'roles_a',
'filters', 'filters_by_project', 'roles', 'roles_by_user',
'roles_by_project'
]
def setUp(self):
"""Create fake pickle objects to be used as cache"""
self.tmpdir = tempfile.mkdtemp()
os.mkdir(self.tmpdir + os.path.sep + 'keystone')
os.mkdir(self.tmpdir + os.path.sep + 'region1')
os.mkdir(self.tmpdir + os.path.sep + 'region2')
sample_dict = {id: 'value1'}
for resource in self.keystone_objects:
with open(self.tmpdir + '/keystone/' + resource + '.pickle',
'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
for resource in OpenStackMap.resources_region:
with open(self.tmpdir + '/region1/' + resource + '.pickle',
'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
with open(self.tmpdir + '/region2/vms.pickle', 'wb') as f:
pickle.dump(sample_dict, f, protocol=-1)
self.map = OpenStackMap(
self.tmpdir,
region='region1',
auto_load=False,
objects_strategy=OpenStackMap.USE_CACHE_OBJECTS_ONLY)
def tearDown(self):
for dir in os.listdir(self.tmpdir):
for name in os.listdir(self.tmpdir + os.path.sep + dir):
print self.tmpdir + os.path.sep + dir + os.path.sep + name
os.unlink(self.tmpdir + os.path.sep + dir + os.path.sep + name)
os.rmdir(self.tmpdir + '/keystone')
os.rmdir(self.tmpdir + '/region1')
os.rmdir(self.tmpdir + '/region2')
os.rmdir(self.tmpdir)
def test_load_all(self):
"""test the load_all method"""
self.map.load_all()
for resource in OpenStackMap.resources_region:
data = getattr(self.map, resource)
self.assertTrue(data)
for resource in self.keystone_objects:
data = getattr(self.map, resource)
self.assertTrue(data)
def test_load_nova_region2(self):
"""test the load_nova in region2"""
self.map.change_region('region2', False)
self.map.load_nova()
self.assertTrue(self.map.vms)
def test_load_all_region2(self):
"""test the load_all in region2 (is invoked by change_region)"""
self.map.change_region('region2')
self.assertTrue(self.map.vms)
# region2 has vms, but not other resources as networks
self.assertFalse(self.map.networks)
def test_load_neutron_region2_failed(self):
"""test the load_neutron in region2: it must fail"""
self.map.change_region('region2', False)
with self.assertRaises(Exception):
self.map.load_neutron()
def test_preload_regions(self):
"""test the preload_regions method"""
self.map.preload_regions()
self.assertEquals(len(self.map.region_map), 2)
self.assertTrue(self.map.region_map['region2']['vms'])
def test_load_nova2(self):
"""test_load_nova check that we could build a map from nova resources using Direct_objects directive."""
openstackmap = OpenStackMap(auto_load=False, objects_strategy=OpenStackMap.DIRECT_OBJECTS)
openstackmap.load_nova()
self.assertIsNotNone(openstackmap)
