def _get_connection(provider, credentials):
log.debug(f"Configuring `{provider}` Connection")
if provider == "aws":
config = {
"aws_access_key": credentials["access_key"],
"aws_secret_key": credentials["secret_key"]
}
connection = CloudProviderFactory().create_provider(
ProviderList.AWS, config)
elif provider == "azure":
config = {
"azure_subscription_id": credentials["subscription_id"],
"azure_client_id": credentials["client_id"],
"azure_secret": credentials["secret"],
"azure_tenant": credentials["tenant"]
}
connection = CloudProviderFactory().create_provider(
ProviderList.AZURE, config)
elif provider == "google":
config = {
"gcp_service_creds_file": credentials["credentials_file"]
}
connection = CloudProviderFactory().create_provider(
ProviderList.GCP, config)
else:
raise Exception(f"Unsupported provider `{provider}`.")
# Ideally it would be better to assert if the connection is
# authorized to perform operations required by ObjectStore
# before returning it (and initializing ObjectStore); hence
# any related issues can be handled properly here, and ObjectStore
# can "trust" the connection is established.
#
# However, the mechanism implemented in Cloudbridge to assert if
# a user/service is authorized to perform an operation, assumes
# the user/service is granted with an elevated privileges, such
# as admin/owner-level access to all resources. For a detailed
# discussion see:
#
# https://github.com/CloudVE/cloudbridge/issues/135
#
# Hence, if a resource owner wants to only authorize Galaxy to r/w
# a bucket/container on the provider, but does not allow it to access
# other resources, Cloudbridge may fail asserting credentials.
# For instance, to r/w an Amazon S3 bucket, the resource owner
# also needs to authorize full access to Amazon EC2, because Cloudbridge
# leverages EC2-specific functions to assert the credentials.
#
# Therefore, to adhere with principle of least privilege, we do not
# assert credentials; instead, we handle exceptions raised as a
# result of signing API calls to cloud provider (e.g., GCP) using
# incorrect, invalid, or unauthorized credentials.
return connection
def create_provider_instance(self):
provider_name = cb_helpers.get_env("CB_TEST_PROVIDER", "aws")
zone_cfg_key = get_provider_test_data(provider_name,
'placement_cfg_key')
factory = CloudProviderFactory()
provider_class = factory.get_provider_class(provider_name)
config = {
'default_wait_interval': self.get_provider_wait_interval(
provider_class),
'default_result_limit': 5,
zone_cfg_key: get_provider_test_data(provider_name, 'placement')
}
return provider_class(config)
def test_create_provider_valid(self):
# Creating a provider with a known name should return
# a valid implementation
self.assertIsInstance(
CloudProviderFactory().create_provider(factory.ProviderList.AWS,
{}),
interfaces.CloudProvider,
"create_provider did not return a valid VM type")
def test_register_provider_class_invalid(self):
# Attempting to register an invalid test class should be ignored
class DummyClass(object):
PROVIDER_ID = 'aws'
factory = CloudProviderFactory()
factory.register_provider_class(DummyClass)
self.assertTrue(DummyClass not in factory.get_all_provider_classes())
def test_register_provider_class_without_id(self):
# Attempting to register a class without a PROVIDER_ID attribute
# should be ignored.
class DummyClass(CloudProvider):
pass
factory = CloudProviderFactory()
factory.register_provider_class(DummyClass)
self.assertTrue(DummyClass not in factory.get_all_provider_classes())
def upload(self, source, upload_info):
container, location = upload_info["path"].split("/", 1)
provider = CloudProviderFactory().create_provider(
ProviderList.OPENSTACK, {
'os_storage_url': upload_info["swiftFileUrl"],
'os_auth_token': upload_info["token"]
})
bucket = provider.storage.buckets.get(container)
obj = bucket.objects.create(location)
obj.upload_from_file(source)
def upload(self, source, upload_info):
creds = upload_info["amazonCredentials"]
provider = CloudProviderFactory().create_provider(
ProviderList.AWS, {
'aws_access_key': creds["accessKey"],
'aws_secret_key': creds["secretKey"],
'aws_session_token': creds["sessionToken"]
})
bucket = provider.storage.buckets.get(upload_info['s3BucketName'])
obj = bucket.objects.create(upload_info['s3ObjectKey'])
obj.upload_from_file(source)
def test_register_provider_class_double(self):
# Attempting to register the same class twice should register second
# instance
class DummyClass(CloudProvider):
PROVIDER_ID = 'aws'
factory = CloudProviderFactory()
factory.list_providers()
factory.register_provider_class(DummyClass)
self.assertTrue(DummyClass in factory.get_all_provider_classes())
self.assertTrue(
AWSCloudProvider not in factory.get_all_provider_classes())
def test_provider_always_has_zone(self):
cloned_config = self.provider.config.copy()
# Just a simpler way set zone to null for any provider
# instead of doing it individually for each provider
cloned_config['aws_zone_name'] = None
cloned_config['azure_zone_name'] = None
cloned_config['gcp_zone_name'] = None
cloned_config['os_zone_name'] = None
cloned_provider = CloudProviderFactory().create_provider(
self.provider.PROVIDER_ID, cloned_config)
# FIXME: GCP always requires a zone, so skip for now
if self.provider.PROVIDER_ID != 'gcp':
self.assertIsNotNone(cloned_provider.zone_name)
def test_authenticate_failure(self):
if isinstance(self.provider, TestMockHelperMixin):
raise unittest.SkipTest("Mock providers are not expected to"
" authenticate correctly")
# Mock up test by clearing credentials on a per provider basis
cloned_config = self.provider.config.copy()
if self.provider.PROVIDER_ID == 'aws':
cloned_config['aws_access_key'] = "dummy_a_key"
cloned_config['aws_secret_key'] = "dummy_s_key"
elif self.provider.PROVIDER_ID == 'openstack':
cloned_config['os_username'] = "******"
cloned_config['os_password'] = "******"
elif self.provider.PROVIDER_ID == 'azure':
cloned_config['azure_subscription_id'] = "cb_dummy"
elif self.provider.PROVIDER_ID == 'gcp':
cloned_config['gcp_service_creds_dict'] = {'dummy': 'dict'}
with self.assertRaises(ProviderConnectionException):
cloned_provider = CloudProviderFactory().create_provider(
self.provider.PROVIDER_ID, cloned_config)
cloned_provider.authenticate()
def main(cluster_id):
cloudlaunch_data = get_bootstrap_data()
provider_config = get_provider_config(cloudlaunch_data)
provider = CloudProviderFactory().create_provider(ProviderList.AWS,
provider_config)
inst_id = cloudlaunch_data.get('host_config', {}).get('instance_id')
this_inst = provider.compute.instances.get(inst_id)
if this_inst:
# pylint:disable=protected-access
this_inst._ec2_instance.create_tags(Tags=[{
'Key': f'kubernetes.io/cluster/{cluster_id}',
'Value': "owned"
}])
def test_provider_zone_in_region(self):
cloned_config = self.provider.config.copy()
# Just a simpler way set zone to null for any provider
# instead of doing it individually for each provider
cloned_config['aws_zone_name'] = None
cloned_config['azure_zone_name'] = None
cloned_config['gcp_zone_name'] = None
cloned_config['os_zone_name'] = None
cloned_provider = CloudProviderFactory().create_provider(
self.provider.PROVIDER_ID, cloned_config)
region = cloned_provider.compute.regions.get(
cloned_provider.region_name)
matches = [
zone.name for zone in region.zones
if zone.name == cloned_provider.zone_name
]
# FIXME: GCP always requires a zone, so skip for now
if self.provider.PROVIDER_ID != 'gcp':
self.assertListEqual([cloned_provider.zone_name], matches)
import json
import requests
import sys
import cachetools
from cloudbridge.factory import CloudProviderFactory
from cloudbridge.factory import ProviderList
aws = CloudProviderFactory().create_provider(ProviderList.AWS, {})
info_json_url = 'https://cdn.rawgit.com/powdahound/ec2instances' \
'.info/master/www/instances.json'
vmtypes_dirpath = 'vmtypes/'
@cachetools.cached(cachetools.TTLCache(maxsize=1, ttl=24 * 3600))
def get_all_info_from_json_file():
print("Getting all VM Types")
r = requests.get(info_json_url)
print("Successfully reached: " + info_json_url)
data = r.json()
# keeping entire dict to not invalidate "extra_data" property in
# CloudBridge
vm_types = {}
for entry in data:
vm_types[entry['instance_type']] = entry
return vm_types
def configure_provider(provider, credentials):
"""
Given a provider name and required credentials, it configures and returns a cloudbridge
connection to the provider.
:type provider: string
:param provider: the name of cloud-based resource provided. A list of supported providers is given in
`SUPPORTED_PROVIDERS` variable.
:type credentials: dict
:param credentials: a dictionary containing all the credentials required to authenticated to the
specified provider.
:rtype: provider specific, e.g., `cloudbridge.cloud.providers.aws.provider.AWSCloudProvider` for AWS.
:return: a cloudbridge connection to the specified provider.
"""
missing_credentials = []
if provider == 'aws':
access = credentials.get('access_key', None)
if access is None:
access = credentials.get("AccessKeyId", None)
if access is None:
missing_credentials.append('access_key')
secret = credentials.get('secret_key', None)
if secret is None:
secret = credentials.get("SecretAccessKey", None)
if secret is None:
missing_credentials.append('secret_key')
if len(missing_credentials) > 0:
raise RequestParameterMissingException(
"The following required key(s) are missing from the provided "
"credentials object: {}".format(missing_credentials))
session_token = credentials.get("SessionToken")
config = {
'aws_access_key': access,
'aws_secret_key': secret,
"aws_session_token": session_token
}
connection = CloudProviderFactory().create_provider(
ProviderList.AWS, config)
elif provider == "azure":
subscription = credentials.get('subscription_id', None)
if subscription is None:
missing_credentials.append('subscription_id')
client = credentials.get('client_id', None)
if client is None:
missing_credentials.append('client_id')
secret = credentials.get('secret', None)
if secret is None:
missing_credentials.append('secret')
tenant = credentials.get('tenant', None)
if tenant is None:
missing_credentials.append('tenant')
if len(missing_credentials) > 0:
raise RequestParameterMissingException(
"The following required key(s) are missing from the provided "
"credentials object: {}".format(missing_credentials))
config = {
'azure_subscription_id': subscription,
'azure_client_id': client,
'azure_secret': secret,
'azure_tenant': tenant
}
storage_account = credentials.get("storage_account")
if storage_account:
config["azure_storage_account"] = storage_account
resource_group = credentials.get("resource_group")
if resource_group:
config["azure_resource_group"] = resource_group
connection = CloudProviderFactory().create_provider(
ProviderList.AZURE, config)
elif provider == "openstack":
username = credentials.get('username', None)
if username is None:
missing_credentials.append('username')
password = credentials.get('password', None)
if password is None:
missing_credentials.append('password')
auth_url = credentials.get('auth_url', None)
if auth_url is None:
missing_credentials.append('auth_url')
prj_name = credentials.get('project_name', None)
if prj_name is None:
missing_credentials.append('project_name')
prj_domain_name = credentials.get('project_domain_name', None)
if prj_domain_name is None:
missing_credentials.append('project_domain_name')
user_domain_name = credentials.get('user_domain_name', None)
if user_domain_name is None:
missing_credentials.append('user_domain_name')
if len(missing_credentials) > 0:
raise RequestParameterMissingException(
"The following required key(s) are missing from the provided "
"credentials object: {}".format(missing_credentials))
config = {
'os_username': username,
'os_password': password,
'os_auth_url': auth_url,
'os_project_name': prj_name,
'os_project_domain_name': prj_domain_name,
'os_user_domain_name': user_domain_name
}
connection = CloudProviderFactory().create_provider(
ProviderList.OPENSTACK, config)
elif provider == "gcp":
config = {"gcp_service_creds_dict": credentials}
connection = CloudProviderFactory().create_provider(
ProviderList.GCP, config)
else:
raise RequestParameterInvalidException(
"Unrecognized provider '{}'; the following are the supported "
"providers: {}.".format(provider, SUPPORTED_PROVIDERS.keys()))
# The authorization-assertion mechanism of Cloudbridge assumes a user has an elevated privileges,
# such as Admin-level access to all resources (see https://github.com/CloudVE/cloudbridge/issues/135).
# As a result, a user who wants to authorize Galaxy to read/write an Amazon S3 bucket, need to
# also authorize Galaxy with full permission to Amazon EC2 (because Cloudbridge leverages EC2-specific
# operation to assert credentials). While the EC2 authorization is not required by Galaxy to
# read/write a S3 bucket, it can cause this exception.
#
# Until Cloudbridge implements an authorization-specific credentials assertion, we are not asserting
# the authorization/validity of the credentials, in order to avoid asking users to grant Galaxy with an
# elevated, yet unnecessary, privileges.
#
# Note, if user's credentials are invalid/expired to perform the authorized action, that can cause
# exceptions which we capture separately in related read/write attempts.
return connection
def test_create_provider_invalid(self):
# Creating a provider with an invalid name should raise a
# NotImplementedError
with self.assertRaises(NotImplementedError):
CloudProviderFactory().create_provider("ec23", {})
def test_get_provider_class_valid(self):
# Searching for a provider class with a known name should return a
# valid class
self.assertEqual(
CloudProviderFactory().get_provider_class(
factory.ProviderList.AWS), AWSCloudProvider)
def _init_provider(config, provider):
# Connecting to provider and generating keypair for all instances
prov = CloudProviderFactory().create_provider(provider, config)
return prov
def test_get_provider_class_invalid(self):
# Searching for a provider class with an invalid name should
# return None
self.assertIsNone(CloudProviderFactory().get_provider_class("aws1"))
def test_find_provider_include_mocks(self):
self.assertTrue(
any(cls
for cls in CloudProviderFactory().get_all_provider_classes()
if issubclass(cls, TestMockHelperMixin)),
"expected to find at least one mock provider")
def test_find_provider_exclude_mocks(self):
for cls in CloudProviderFactory().get_all_provider_classes(
ignore_mocks=True):
self.assertTrue(
not issubclass(cls, TestMockHelperMixin),
"Did not expect mock but %s implements mock provider" % cls)