def __init__(self, username=None, password=None, tenantid=None, baseurl=None):
        """Set instance authentication constants.

        Args:
            username (str): OpenStack administrator username.
            password (str): OpenStack administrator password.
            tenantid (str): OpenStack tenant for the administrator account.
            baseurl  (str): OpenStack environment URI.
        """
        self._AUTH_USERNAME = username
        self._AUTH_PASSWORD = password
        self._AUTH_TENANTID = tenantid
        self._BASE_URL = baseurl
        keystone = keystoneClient.Client(username=self._AUTH_USERNAME,
                                         password=self._AUTH_PASSWORD,
                                         project_name=self._AUTH_TENANTID,
                                         auth_url='%s:5001/v3' % self._BASE_URL)
        self._roleManager = RoleManager(keystone)
        self._groupManager = GroupManager(keystone)
        self._domainManager = DomainManager(keystone)
        self._projectManager = ProjectManager(keystone)
        self._roleAssignmentManager = RoleAssignmentManager(keystone)
Beispiel #2
0
def generate_test_data():
    ''' Builds a set of test_data data as returned by Keystone V2. '''
    test_data = TestDataContainer()

    keystone_service = {
        'type':
        'identity',
        'id':
        uuid.uuid4().hex,
        'endpoints': [{
            'url': 'http://admin.localhost:35357/v3',
            'region': 'RegionOne',
            'interface': 'admin',
            'id': uuid.uuid4().hex,
        }, {
            'url': 'http://internal.localhost:5000/v3',
            'region': 'RegionOne',
            'interface': 'internal',
            'id': uuid.uuid4().hex
        }, {
            'url': 'http://public.localhost:5000/v3',
            'region': 'RegionOne',
            'interface': 'public',
            'id': uuid.uuid4().hex
        }]
    }

    # Domains
    domain_dict = {
        'id': uuid.uuid4().hex,
        'name': 'domain',
        'description': '',
        'enabled': True
    }
    test_data.domain = Domain(DomainManager(None), domain_dict, loaded=True)

    # Users
    user_dict = {
        'id': uuid.uuid4().hex,
        'name': 'gabriel',
        'email': '*****@*****.**',
        'password': '******',
        'domain_id': domain_dict['id'],
        'token': '',
        'enabled': True
    }
    test_data.user = User(UserManager(None), user_dict, loaded=True)

    # Projects
    project_dict_1 = {
        'id': uuid.uuid4().hex,
        'name': 'tenant_one',
        'description': '',
        'domain_id': domain_dict['id'],
        'enabled': True
    }
    project_dict_2 = {
        'id': uuid.uuid4().hex,
        'name': '',
        'description': '',
        'domain_id': domain_dict['id'],
        'enabled': False
    }
    test_data.project_one = Project(ProjectManager(None),
                                    project_dict_1,
                                    loaded=True)
    test_data.project_two = Project(ProjectManager(None),
                                    project_dict_2,
                                    loaded=True)

    # Roles
    role_dict = {'id': uuid.uuid4().hex, 'name': 'Member'}
    test_data.role = Role(RoleManager, role_dict)

    nova_service = {
        'type': 'compute',
        'id': uuid.uuid4().hex,
        'endpoints': [
            {
                'url': 'http://nova-admin.localhost:8774/v2.0/%s' \
                       % (project_dict_1['id']),
                'region': 'RegionOne',
                'interface': 'admin',
                'id': uuid.uuid4().hex,
            },
            {
                'url': 'http://nova-internal.localhost:8774/v2.0/%s' \
                       % (project_dict_1['id']),
                'region': 'RegionOne',
                'interface': 'internal',
                'id': uuid.uuid4().hex
            },
            {
                'url':'http://nova-public.localhost:8774/v2.0/%s' \
                       % (project_dict_1['id']),
                'region':'RegionOne',
                'interface': 'public',
                 'id': uuid.uuid4().hex
            },
            {
                'url': 'http://nova2-admin.localhost:8774/v2.0/%s' \
                       % (project_dict_1['id']),
                'region': 'RegionTwo',
                'interface': 'admin',
                'id': uuid.uuid4().hex,
            },
            {
                'url': 'http://nova2-internal.localhost:8774/v2.0/%s' \
                       % (project_dict_1['id']),
                'region': 'RegionTwo',
                'interface': 'internal',
                'id': uuid.uuid4().hex
            },
            {
                'url':'http://nova2-public.localhost:8774/v2.0/%s' \
                       % (project_dict_1['id']),
                'region':'RegionTwo',
                'interface': 'public',
                 'id': uuid.uuid4().hex
            }
        ]
    }

    # Tokens
    tomorrow = datetime_safe.datetime.now() + timedelta(days=1)
    expiration = datetime_safe.datetime.isoformat(tomorrow)
    auth_token = uuid.uuid4().hex
    auth_response_headers = {'X-Subject-Token': auth_token}

    auth_response = TestResponse({"headers": auth_response_headers})

    scoped_token_dict = {
        'token': {
            'methods': ['password'],
            'expires_at': expiration,
            'project': {
                'id': project_dict_1['id'],
                'name': project_dict_1['name'],
                'domain': {
                    'id': domain_dict['id'],
                    'name': domain_dict['name']
                }
            },
            'user': {
                'id': user_dict['id'],
                'name': user_dict['name'],
                'domain': {
                    'id': domain_dict['id'],
                    'name': domain_dict['name']
                }
            },
            'roles': [role_dict],
            'catalog': [keystone_service, nova_service]
        }
    }

    test_data.scoped_access_info = AccessInfo.factory(resp=auth_response,
                                                      body=scoped_token_dict)

    unscoped_token_dict = {
        'token': {
            'methods': ['password'],
            'expires_at': expiration,
            'user': {
                'id': user_dict['id'],
                'name': user_dict['name'],
                'domain': {
                    'id': domain_dict['id'],
                    'name': domain_dict['name']
                }
            },
            'roles': [role_dict],
            'catalog': [keystone_service]
        }
    }

    test_data.unscoped_access_info = AccessInfo.factory(
        resp=auth_response, body=unscoped_token_dict)

    # Service Catalog
    test_data.service_catalog = ServiceCatalog.factory(
        {
            'methods': ['password'],
            'user': {},
            'catalog': [keystone_service, nova_service],
        },
        token=auth_token)

    return test_data
class QuotaChecker:

    """Check and enforce OpenStack tenant quota.

    Verifies that a given tenant does have its correct allocated quota.

    Attributes:
        DEFAULT_QUOTA (dict): The default quota for a service developer.
        PARTNER_QUOTA (dict): The default quota for a partner with CRA.
        BIGDATA_QUOTA (dict): The quota for big data enabled projects.
    """

    _DEFAULT_QUOTA_NAME = 'Default CRA quota'
    _BIGDATA_QUOTA_NAME = 'Bigdata CRA quota'

    DEFAULT_QUOTA = {
        'instances': 16,
        'cores': 16,
        'ram': 32 * 1024,
        'floating_ips': 5,
        'cinder_GB': 1024,
        'swift_bytes': 1024 * 1024 * 1024 * 1024,
        'flavors': ['m1.tiny', 'm1.small', 'm1.medium', 'm1.large', 'm1.x-large']
    }

    PARTNER_QUOTA = {
        'instances': 1,
        'cores': 1,
        'ram': 1024,
        'floating_ips': 1,
        'cinder_GB': 40,
        'swift_bytes': 40 * 1024 * 1024 * 1024,
        'flavors': ['m1.tiny']
    }

    BIGDATA_QUOTA = {
        'instances': 16,
        'cores': 46,
        'ram': 400 * 1024,
        'floating_ips': 15,
        'cinder_GB': 1024,
        'swift_bytes': 1024 * 1024 * 1024 * 1024,
        'flavors': ['m1.tiny', 'm1.small', 'hadoop.small', 'hadoop.medium', 'hadoop.large']
    }

    def __init__(self, username=None, password=None, tenantid=None, baseurl=None):
        """Set instance authentication constants.

        Args:
            username (str): OpenStack administrator username.
            password (str): OpenStack administrator password.
            tenantid (str): OpenStack tenant for the administrator account.
            baseurl  (str): OpenStack environment URI.
        """
        self._AUTH_USERNAME = username
        self._AUTH_PASSWORD = password
        self._AUTH_TENANTID = tenantid
        self._BASE_URL = baseurl
        keystone = keystoneClient.Client(username=self._AUTH_USERNAME,
                                         password=self._AUTH_PASSWORD,
                                         project_name=self._AUTH_TENANTID,
                                         auth_url='%s:5001/v3' % self._BASE_URL)
        self._roleManager = RoleManager(keystone)
        self._groupManager = GroupManager(keystone)
        self._domainManager = DomainManager(keystone)
        self._projectManager = ProjectManager(keystone)
        self._roleAssignmentManager = RoleAssignmentManager(keystone)

    def _getOpenstackGroup(self, group):
        try:
            os_group = self._groupManager.find(name=group)
        except:
            return None
        return os_group

    def _getTenantId(self, tenant):
        projectMap = dict(map(lambda assignment: (assignment.group['id'], assignment.scope['project']['id']),
                              filter(lambda a: 'group' in a._info.keys(), self._roleAssignmentManager.list())))

        return projectMap[tenant].strip() if tenant in projectMap.keys() else None

    def _ensureTenantNetwork(self, tenant):
        neutron = neutronClient.Client(username=self._AUTH_USERNAME,
                                       password=self._AUTH_PASSWORD,
                                       tenant_id=self._AUTH_TENANTID,
                                       auth_url='%s:5001/v2.0' % self._BASE_URL)

        if not filter(lambda network: network['tenant_id'] == tenant, neutron.list_networks()['networks']):
            network = neutron.create_network({'network': {'name': 'default', 'tenant_id': tenant}})['network']
            while not neutron.list_networks(id=network['id'])['networks']:
                sleep(1)

            allocated_cidrs = map(lambda chunk: (int(chunk[0]), int(chunk[1])),
                                  map(lambda cidr: cidr['cidr'].split('/')[0].split('.')[-2:],
                                      filter(lambda subnet: subnet['cidr'].endswith('/27'),
                                             neutron.list_subnets()['subnets'])))

            if (192, 0) in allocated_cidrs:
                allocated_cidrs.remove((192, 0))

            if allocated_cidrs:
                max_bigchunk = max(map(lambda chunk: chunk[0], allocated_cidrs))
                max_smlchunk = max(map(lambda chunk: chunk[1], filter(lambda c: c[0] == max_bigchunk, allocated_cidrs)))

                if max_bigchunk == 191 and max_smlchunk == 224:
                    max_bigchunk = 192
                    max_smlchunk = 0

                if max_smlchunk == 224:
                    cidr = '.'.join([str(chunk) for chunk in [192, 168, max_bigchunk + 1, 0]]) + '/27'
                else:
                    cidr = '.'.join([str(chunk) for chunk in [192, 168, max_bigchunk, max_smlchunk + 32]]) + '/27'
            else:
                cidr = '192.168.0.0/27'
            subnet = neutron.create_subnet({'subnet': {'name': 'default-subnet',
                                                       'cidr': cidr,
                                                       'dns_nameservers': ['193.166.4.24', '193.166.4.25'],
                                                       'tenant_id': tenant,
                                                       'network_id': network['id'],
                                                       'ip_version': '4'}})['subnet']
            while not neutron.list_subnets(id=subnet['id'])['subnets']:
                sleep(1)

            router = neutron.create_router({'router': {'tenant_id': tenant,
                                                       'name': 'default-router'}})['router']
            while not neutron.list_routers(id=router['id'])['routers']:
                sleep(1)
            public_net_id = filter(lambda n: n['router:external'],
                                   neutron.list_networks(name='public')['networks'])[0]['id']
            neutron.add_gateway_router(router['id'], {'network_id': public_net_id})
            neutron.add_interface_router(router['id'], {'subnet_id': subnet['id']})

    def _getTenantQuota(self, tenant, tenantType):
        quota = None
        statedQuota = map(lambda q: q['FIELD_VALUE'], filter(lambda f: f['CUSTOM_FIELD_ID'] == 'PROJECT_FIELD_1',
                                                             tenant['CUSTOMFIELDS']))[0]
        if statedQuota == self._BIGDATA_QUOTA_NAME:
            quota = self.BIGDATA_QUOTA
        else:
            if statedQuota == self._DEFAULT_QUOTA_NAME:
                if tenantType == LDAPUpdater.FPA_CRA:
                    quota = self.PARTNER_QUOTA
                if tenantType == LDAPUpdater.SDA:
                    quota = self.DEFAULT_QUOTA

        return quota

    def _grantAccess(self, client, flavor, tenant):
        try:
            client.flavor_access.add_tenant_access(flavor, tenant)
        except Conflict:
            pass

    def _revokeAccess(self, client, flavor, tenant):
        try:
            client.flavor_access.remove_tenant_access(flavor, tenant)
        except NotFound:
            pass

    def _enforceQuota(self, ldap_tenant, quotaDefinition, ldap_conn=None):
        openstackGroup = self._getOpenstackGroup(ldap_tenant)
        if openstackGroup:
            tenant = self._getTenantId(ldap_tenant)
            if not tenant:
                # Create or map tenant in openstack
                project = self._projectManager.list(name=ldap_tenant)
                if not project:
                    project = self._projectManager.create(ldap_tenant, self._domainManager.find(id='default'))
                self._roleManager.grant(self._roleManager.find(name='member').id,
                                        group=openstackGroup.id,
                                        project=project.id)
                tenant = project.id

            if ldap_conn and ldap_tenant in map(lambda t: t[0].split(',')[0].split('=')[1],
                                                ldap_conn.ldap_search('cn=digile.platform,ou=projects,\
                                                                       dc=forgeservicelab,dc=fi',
                                                                      SCOPE_SUBORDINATE, attrsonly=1)):
                with novaClient.Client(username=self._AUTH_USERNAME,
                                       api_key=self._AUTH_PASSWORD,
                                       tenant_id=tenant,
                                       auth_url='%s:5001/v2.0' % self._BASE_URL) as nova:
                    try:
                        nova.security_group_rules.create(nova.security_groups.find(name='default').id,
                                                         ip_protocol='tcp',
                                                         from_port=22,
                                                         to_port=22,
                                                         cidr='86.50.27.230/32')
                    except Unauthorized:
                        # butler.service not yet part of the tenant, wait for next round.
                        pass
                    except BadRequest:
                        # Rule already exists, that's OK.
                        pass

            self._ensureTenantNetwork(tenant)

            if quotaDefinition:
                service_opts = {
                    'meta': ['quota-bytes:%s' % quotaDefinition['swift_bytes']],
                    'os_username': self._AUTH_USERNAME,
                    'os_password': self._AUTH_PASSWORD,
                    'os_auth_url': '%s:5001/v2.0' % self._BASE_URL,
                    'os_storage_url': '%s:8081/v1/AUTH_%s' % (self._BASE_URL, self._projectManager.get(tenant).name),
                    'os_tenant_name': self._AUTH_TENANTID
                }

                swift = swiftService.SwiftService(options=service_opts)
                swift.post()
                del swift

                cinder = cinderClient.Client(username=self._AUTH_USERNAME,
                                             api_key=self._AUTH_PASSWORD,
                                             tenant_id=self._AUTH_TENANTID,
                                             auth_url=service_opts['os_auth_url'])
                cinder.quotas.update(tenant, gigabytes=quotaDefinition['cinder_GB'])
                del cinder

                with novaClient.Client(username=self._AUTH_USERNAME,
                                       api_key=self._AUTH_PASSWORD,
                                       tenant_id=self._AUTH_TENANTID,
                                       auth_url=service_opts['os_auth_url']) as nova:
                    nova.quotas.update(tenant,
                                       instances=quotaDefinition['instances'],
                                       cores=quotaDefinition['cores'],
                                       ram=quotaDefinition['ram'],
                                       floating_ips=quotaDefinition['floating_ips'])
                    allFlavors = nova.flavors.findall(is_public=None)
                    map(lambda f: self._grantAccess(nova, f, tenant),
                        filter(lambda f: f.name.encode() in quotaDefinition['flavors'], allFlavors))
                    map(lambda f: self._revokeAccess(nova, f, tenant),
                        filter(lambda f: f.name.encode() not in quotaDefinition['flavors'], allFlavors))

                neutron = neutronClient.Client(username=self._AUTH_USERNAME,
                                               password=self._AUTH_PASSWORD,
                                               tenant_id=self._AUTH_TENANTID,
                                               auth_url=service_opts['os_auth_url'])
                neutron.update_quota(tenant, {'quota': {'floatingip': quotaDefinition['floating_ips']}})
                del neutron

            with novaClient.Client(username=self._AUTH_USERNAME,
                                   api_key=self._AUTH_PASSWORD,
                                   tenant_id=self._AUTH_TENANTID,
                                   auth_url='%s:5001/v2.0' % self._BASE_URL) as nova:
                self._grantAccess(nova, nova.flavors.find(name='m1.tiny', is_public=None), tenant)

    def enforceQuotas(self, tenantList, tenantsType, ldap_conn=None):
        """Enforce the quota for each tenant on the list.

        Args:
            tenantList (List): A list of tenants as JSON from Insightly.
            tenantsType (str): A description of the type of tenant, one of 'SDA', 'FPA' or 'FPA (CRA)'.
        """
        map(lambda t: self._enforceQuota(sanitize(t['PROJECT_NAME']), self._getTenantQuota(t, tenantsType),
                                         ldap_conn), tenantList)