def generate_tests(self):
"""
Generate and return a suite of tests for all provider and test class
combinations
"""
factory = CloudProviderFactory()
use_mock_drivers = parse_bool(
os.environ.get("CB_USE_MOCK_PROVIDERS", True))
provider_name = os.environ.get("CB_TEST_PROVIDER", None)
if provider_name:
provider_classes = [
factory.get_provider_class(
provider_name,
get_mock=use_mock_drivers)]
if not provider_classes[0]:
raise ValueError(
"Could not find specified test provider %s" %
provider_name)
else:
provider_classes = factory.get_all_provider_classes(
get_mock=use_mock_drivers)
suite = unittest.TestSuite()
suites = [
self.generate_test_suite_for_provider(p) for p in provider_classes]
for s in suites:
suite.addTest(s)
return suite
def get_cloud_provider(cloud, cred_dict):
"""
Returns a provider for a cloud given a cloud model and a dictionary
containing the relevant credentials.
:type cloud: Cloud
:param cloud: The cloud to create a provider for
:rtype: ``object`` of :class:`.dict`
:return: A dict containing the necessary credentials for the cloud.
"""
# In case a base class instance is sent in, attempt to retrieve the actual
# subclass.
if type(cloud) is models.Cloud:
cloud = models.Cloud.objects.get_subclass(slug=cloud.slug)
if isinstance(cloud, models.OpenStack):
config = {
'os_auth_url': cloud.auth_url,
'os_region_name': cloud.region_name
}
config.update(cred_dict)
return CloudProviderFactory().create_provider(ProviderList.OPENSTACK,
config)
elif isinstance(cloud, models.AWS):
config = {
'ec2_is_secure': cloud.compute.ec2_is_secure,
'ec2_region_name': cloud.compute.ec2_region_name,
'ec2_region_endpoint': cloud.compute.ec2_region_endpoint,
'ec2_port': cloud.compute.ec2_port,
'ec2_conn_path': cloud.compute.ec2_conn_path,
's3_host': cloud.object_store.s3_host,
's3_port': cloud.object_store.s3_port,
's3_conn_path': cloud.object_store.s3_conn_path,
}
config.update(cred_dict)
return CloudProviderFactory().create_provider(ProviderList.AWS, config)
elif isinstance(cloud, models.Azure):
config = {
'azure_region_name': cloud.region_name,
'azure_resource_group': cloud.resource_group,
'azure_storage_account': cloud.storage_account,
'azure_vm_default_user_name': cloud.vm_default_user_name
}
config.update(cred_dict)
return CloudProviderFactory().create_provider(ProviderList.AZURE,
config)
elif isinstance(cloud, models.GCE):
config = {
'gce_service_creds_dict': cred_dict,
'gce_default_zone': cloud.zone_name,
'gce_region_name': cloud.region_name
}
return CloudProviderFactory().create_provider(ProviderList.GCE, config)
else:
raise Exception("Unrecognised cloud provider: %s" % cloud)
def create_provider_instance(self):
provider_name = os.environ.get("CB_TEST_PROVIDER", "aws")
use_mock_drivers = parse_bool(
os.environ.get("CB_USE_MOCK_PROVIDERS", "True"))
factory = CloudProviderFactory()
provider_class = factory.get_provider_class(provider_name,
get_mock=use_mock_drivers)
config = {'default_wait_interval':
self.get_provider_wait_interval(provider_class),
'default_result_limit': 1}
return provider_class(config)
def test_find_provider_mock_valid(self):
# Searching for a provider with a known mock driver should return
# an implementation implementing helpers.TestMockHelperMixin
mock = CloudProviderFactory().get_provider_class(
factory.ProviderList.AWS, get_mock=True)
self.assertTrue(
issubclass(mock, helpers.TestMockHelperMixin),
"Expected mock for AWS but class does not implement mock provider")
for cls in CloudProviderFactory().get_all_provider_classes(
get_mock=False):
self.assertTrue(
not issubclass(cls, TestMockHelperMixin),
"Did not expect mock but %s implements mock provider" % cls)
def upload(self, source, upload_info):
container, location = upload_info["path"].split("/", 1)
provider = CloudProviderFactory().create_provider(
ProviderList.OPENSTACK,
{'os_storage_url': upload_info["swiftFileUrl"],
'os_auth_token': upload_info["token"]})
bucket = provider.object_store.get(container)
obj = bucket.create_object(location)
obj.upload_from_file(source)
def upload(self, source, upload_info):
creds = upload_info["amazonCredentials"]
provider = CloudProviderFactory().create_provider(
ProviderList.AWS,
{'aws_access_key': creds["accessKey"],
'aws_secret_key': creds["secretKey"],
'aws_session_token': creds["sessionToken"]})
bucket = provider.object_store.get(upload_info['s3BucketName'])
obj = bucket.create_object(upload_info['s3ObjectKey'])
obj.upload_from_file(source)
def test_authenticate_failure(self):
if isinstance(self.provider, TestMockHelperMixin):
raise unittest.SkipTest(
"Mock providers are not expected to"
" authenticate correctly")
# Mock up test by clearing credentials on a per provider basis
cloned_config = self.provider.config.copy()
if self.provider.PROVIDER_ID == 'aws':
cloned_config['aws_access_key'] = "dummy_a_key"
cloned_config['aws_secret_key'] = "dummy_s_key"
elif self.provider.PROVIDER_ID == 'openstack':
cloned_config['os_username'] = "******"
cloned_config['os_password'] = "******"
with self.assertRaises(ProviderConnectionException):
cloned_provider = CloudProviderFactory().create_provider(
self.provider.PROVIDER_ID, cloned_config)
cloned_provider.authenticate()
def test_register_mock_provider_class_double(self):
# Attempting to register the same mock provider twice should register
# only the second instance
class DummyClass(CloudProvider, TestMockHelperMixin):
PROVIDER_ID = 'aws'
factory = CloudProviderFactory()
factory.list_providers()
factory.register_provider_class(DummyClass)
self.assertTrue(DummyClass in factory.get_all_provider_classes(
get_mock=True))
self.assertTrue(
MockAWSCloudProvider not in factory.get_all_provider_classes(
get_mock=True))
def __init__(self):
"""Initialize target properties and credentials."""
# ~/.cloudbridge file with access configs is required
self.provider = CloudProviderFactory().create_provider(
ProviderList.OPENSTACK, {})
# Configs come from slurmscale.ini config file
self.image_id = ss.config.get_config_value(
'image_id', '736e206d-9c2c-4369-88db-8c3293bd2ad7')
self.instance_type = ss.config.get_config_value(
'instance_type', 'm1.large')
self.subnet_id = ss.config.get_config_value(
'subnet_id', '04d0dbf2-c5c2-4c23-b9a9-16f9038f8dac')
self.key_pair = ss.config.get_config_value('key_pair', 'elasticity_kp')
self.security_groups = ss.config.get_config_value(
'security_groups', ['gxy-workers-sg'])
if not isinstance(self.security_groups, list):
self.security_groups = self.security_groups.split(',')
def __init__(cls, cloud_name, cloud_details):
"""
Initialize the AWS Provider
:param cloud_name:
:param cloud_details:
"""
creds_config = cloud_details['credentials']
compute_config = cloud_details['compute']
# Initialize the params
cls.cloud = cloud_name
cls.image = compute_config['ec2_image_id']
cls.flavor = compute_config['ec2_flavor_name']
cls.secgroup = compute_config['ec2_secgroup_name']
cls.keypair = compute_config['ec2_keypair_name']
# Initialize the CloudBridge provider
cls.provider = CloudProviderFactory().create_provider(
ProviderList.AWS, config=creds_config)
def _configure_provider(provider, credentials):
"""
Given a provider name and required credentials, it configures and returns a cloudbridge
connection to the provider.
:type provider: string
:param provider: the name of cloud-based resource provided. A list of supported providers is given in
`SUPPORTED_PROVIDERS` variable.
:type credentials: dict
:param credentials: a dictionary containing all the credentials required to authenticated to the
specified provider.
:rtype: provider specific, e.g., `cloudbridge.cloud.providers.aws.provider.AWSCloudProvider` for AWS.
:return: a cloudbridge connection to the specified provider.
"""
missing_credentials = []
if provider == 'aws':
access = credentials.get('access_key', None)
if access is None:
missing_credentials.append('access_key')
secret = credentials.get('secret_key', None)
if secret is None:
missing_credentials.append('secret_key')
if len(missing_credentials) > 0:
raise RequestParameterMissingException("The following required key(s) are missing from the provided "
"credentials object: {}".format(missing_credentials))
config = {'aws_access_key': access,
'aws_secret_key': secret}
connection = CloudProviderFactory().create_provider(ProviderList.AWS, config)
elif provider == "azure":
subscription = credentials.get('subscription_id', None)
if subscription is None:
missing_credentials.append('subscription_id')
client = credentials.get('client_id', None)
if client is None:
missing_credentials.append('client_id')
secret = credentials.get('secret', None)
if secret is None:
missing_credentials.append('secret')
tenant = credentials.get('tenant', None)
if tenant is None:
missing_credentials.append('tenant')
if len(missing_credentials) > 0:
raise RequestParameterMissingException("The following required key(s) are missing from the provided "
"credentials object: {}".format(missing_credentials))
config = {'azure_subscription_id': subscription,
'azure_client_id': client,
'azure_secret': secret,
'azure_tenant': tenant}
connection = CloudProviderFactory().create_provider(ProviderList.AZURE, config)
elif provider == "openstack":
username = credentials.get('username', None)
if username is None:
missing_credentials.append('username')
password = credentials.get('password', None)
if password is None:
missing_credentials.append('password')
auth_url = credentials.get('auth_url', None)
if auth_url is None:
missing_credentials.append('auth_url')
prj_name = credentials.get('project_name', None)
if prj_name is None:
missing_credentials.append('project_name')
prj_domain_name = credentials.get('project_domain_name', None)
if prj_domain_name is None:
missing_credentials.append('project_domain_name')
user_domain_name = credentials.get('user_domain_name', None)
if user_domain_name is None:
missing_credentials.append('user_domain_name')
if len(missing_credentials) > 0:
raise RequestParameterMissingException("The following required key(s) are missing from the provided "
"credentials object: {}".format(missing_credentials))
config = {'os_username': username,
'os_password': password,
'os_auth_url': auth_url,
'os_project_name': prj_name,
'os_project_domain_name': prj_domain_name,
'os_user_domain_name': user_domain_name}
connection = CloudProviderFactory().create_provider(ProviderList.OPENSTACK, config)
else:
raise RequestParameterInvalidException("Unrecognized provider '{}'; the following are the supported "
"providers: {}.".format(provider, SUPPORTED_PROVIDERS))
try:
if connection.authenticate():
return connection
except ProviderConnectionException as e:
raise AuthenticationFailed("Could not authenticate to the '{}' provider. {}".format(provider, e))
