def get_fogbow_cloud(): cloud_info = CloudInfo() cloud_info.type = "FogBow" cloud_info.server = "server.com" cloud_info.port = 8182 cloud = FogBowCloudConnector(cloud_info) return cloud
def get_one_cloud(): cloud_info = CloudInfo() cloud_info.type = "OpenNebula" cloud_info.server = "server.com" cloud_info.port = 2633 one_cloud = OpenNebulaCloudConnector(cloud_info) return one_cloud
def get_kube_cloud(): cloud_info = CloudInfo() cloud_info.type = "Kubernetes" cloud_info.protocol = "http" cloud_info.server = "server.com" cloud_info.port = 8080 cloud = KubernetesCloudConnector(cloud_info) return cloud
def get_docker_cloud(): cloud_info = CloudInfo() cloud_info.type = "Kubernetes" cloud_info.protocol = "http" cloud_info.server = "server.com" cloud_info.port = 2375 cloud = DockerCloudConnector(cloud_info) return cloud
def get_ost_cloud(): cloud_info = CloudInfo() cloud_info.type = "OpenStack" cloud_info.protocol = "https" cloud_info.server = "server.com" cloud_info.port = 5000 one_cloud = OpenStackCloudConnector(cloud_info) return one_cloud
def get_occi_cloud(): cloud_info = CloudInfo() cloud_info.type = "OCCI" cloud_info.protocol = "https" cloud_info.server = "server.com" cloud_info.port = 11443 cloud = OCCICloudConnector(cloud_info) return cloud
def gen_launch_res(self, inf, radl, requested_radl, num_vm, auth_data): res = [] for _ in range(num_vm): cloud = CloudInfo() cloud.type = "Dummy" vm = VirtualMachine(inf, "1234", cloud, radl, requested_radl) vm.get_ssh = Mock(side_effect=self.get_dummy_ssh) vm.state = VirtualMachine.RUNNING res.append((True, vm)) return res
def gen_launch_res(self, inf, radl, requested_radl, num_vm, auth_data): res = [] for i in range(num_vm): cloud = CloudInfo() cloud.type = "DeployedNode" vm = VirtualMachine(inf, "1234", cloud, radl, requested_radl) # create the mock for the vm finalize function vm.finalize = Mock(return_value=(True, vm)) res.append((True, vm)) return res
def get_occi_cloud(): cloud_info = CloudInfo() cloud_info.type = "OCCI" cloud_info.protocol = "https" cloud_info.server = "server.com" cloud_info.port = 11443 inf = MagicMock() inf.id = "1" cloud = OCCICloudConnector(cloud_info, inf) return cloud
def get_ost_cloud(): cloud_info = CloudInfo() cloud_info.type = "OpenStack" cloud_info.protocol = "https" cloud_info.server = "server.com" cloud_info.port = 5000 inf = MagicMock() inf.id = "1" one_cloud = OpenStackCloudConnector(cloud_info, inf) return one_cloud
def get_kube_cloud(): cloud_info = CloudInfo() cloud_info.type = "Kubernetes" cloud_info.protocol = "http" cloud_info.server = "server.com" cloud_info.port = 8080 inf = MagicMock() inf.id = "1" cloud = KubernetesCloudConnector(cloud_info, inf) return cloud
def get_docker_cloud(): cloud_info = CloudInfo() cloud_info.type = "Docker" cloud_info.protocol = "http" cloud_info.server = "server.com" cloud_info.port = 2375 inf = MagicMock() inf.id = "1" cloud = DockerCloudConnector(cloud_info, inf) return cloud
import logging.config sys.path.append("..") from IM.CloudInfo import CloudInfo from IM.auth import Authentication from radl import radl_parse from IM.VirtualMachine import VirtualMachine from IM.VMRC import VMRC from IM.InfrastructureInfo import InfrastructureInfo TESTS_PATH = '/home/micafer/codigo/git_im/im/test' AUTH_FILE = TESTS_PATH + '/auth.dat' auth = Authentication(Authentication.read_auth_data(AUTH_FILE)) cloud_list = dict([(c.id, c.getCloudConnector()) for c in CloudInfo.get_cloud_list(auth)]) class TestConnectors(unittest.TestCase): """ Class to test the IM connectors """ vm_list = [] """ List of VMs launched in the test """ # connectors_to_test = "all" connectors_to_test = ["fogbow"] """ Specify the connectors to test: "all": All the connectors specified in the auth file or a list with the IDs""" @classmethod def setUpClass(cls):
def get_azure_cloud(): cloud_info = CloudInfo() cloud_info.type = "Azure" cloud = AzureCloudConnector(cloud_info) return cloud
def get_lib_cloud(): cloud_info = CloudInfo() cloud_info.type = "LibCloud" cloud = LibCloudCloudConnector(cloud_info) return cloud
def get_azure_cloud(): cloud_info = CloudInfo() cloud_info.type = "AzureClassic" cloud = AzureClassicCloudConnector(cloud_info) return cloud
def get_ec2_cloud(): cloud_info = CloudInfo() cloud_info.type = "EC2" cloud = EC2CloudConnector(cloud_info) return cloud
def AddResource(inf_id, radl_data, auth, context=True, failed_clouds=[]): """ Add the resources in the RADL to the infrastructure. Args: - inf_id(str): infrastructure id. - radl(str): RADL description. - auth(Authentication): parsed authentication tokens. - context(bool): Flag to specify if the ctxt step will be made - failed_clouds(list of CloudInfo): A list of failed Cloud providers to avoid launching the VMs in them. Return(list of int): ids of the new virtual machine created. """ auth = InfrastructureManager.check_auth_data(auth) InfrastructureManager.logger.info( "Adding resources to inf: " + str(inf_id)) if isinstance(radl_data, RADL): radl = radl_data else: radl = radl_parse.parse_radl(radl_data) InfrastructureManager.logger.debug(radl) radl.check() sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) # Update infrastructure RADL with this new RADL sel_inf.complete_radl(radl) # If any deploy is defined, only update definitions. if not radl.deploys: sel_inf.update_radl(radl, []) InfrastructureManager.logger.warn( "Infrastructure without any deploy. Exiting.") return [] for system in radl.systems: # Add apps requirements to the RADL apps_to_install = system.getApplications() for app_to_install in apps_to_install: for app_avail, _, _, _, requirements in Recipe.getInstallableApps(): if requirements and app_avail.isNewerThan(app_to_install): # This app must be installed and it has special # requirements try: requirements_radl = radl_parse.parse_radl( requirements).systems[0] system.applyFeatures( requirements_radl, conflict="other", missing="other") except Exception: InfrastructureManager.logger.exception( "Error in the requirements of the app: " + app_to_install.getValue("name") + ". Ignore them.") InfrastructureManager.logger.debug(requirements) break # Get VMRC credentials vmrc_list = [] for vmrc_elem in auth.getAuthInfo('VMRC'): if ('host' in vmrc_elem and 'username' in vmrc_elem and 'password' in vmrc_elem): vmrc_list.append(VMRC(vmrc_elem['host'], vmrc_elem['username'], vmrc_elem['password'])) # Concrete systems using VMRC # NOTE: consider not-fake deploys (vm_number > 0) systems_with_vmrc = {} for system_id in set([d.id for d in radl.deploys if d.vm_number > 0]): s = radl.get_system_by_name(system_id) if not s.getValue("disk.0.image.url") and len(vmrc_list) == 0: raise Exception( "No correct VMRC auth data provided nor image URL") # Remove the requested apps from the system s_without_apps = radl.get_system_by_name(system_id).clone() s_without_apps.delValue("disk.0.applications") # Set the default values for cpu, memory defaults = (Feature("cpu.count", ">=", Config.DEFAULT_VM_CPUS), Feature("memory.size", ">=", Config.DEFAULT_VM_MEMORY, Config.DEFAULT_VM_MEMORY_UNIT), Feature("cpu.arch", "=", Config.DEFAULT_VM_CPU_ARCH)) for f in defaults: if not s_without_apps.hasFeature(f.prop, check_softs=True): s_without_apps.addFeature(f) vmrc_res = [s0 for vmrc in vmrc_list for s0 in vmrc.search_vm(s)] # Check that now the image URL is in the RADL if not s.getValue("disk.0.image.url") and not vmrc_res: raise Exception( "No VMI obtained from VMRC to system: " + system_id) n = [s_without_apps.clone().applyFeatures(s0, conflict="other", missing="other") for s0 in vmrc_res] systems_with_vmrc[system_id] = n if n else [s_without_apps] # Concrete systems with cloud providers and select systems with the greatest score # in every cloud cloud_list = dict([(c.id, c.getCloudConnector()) for c in CloudInfo.get_cloud_list(auth) if c not in failed_clouds]) concrete_systems = {} for cloud_id, cloud in cloud_list.items(): for system_id, systems in systems_with_vmrc.items(): s1 = [InfrastructureManager._compute_score(s.clone().applyFeatures(s0, conflict="other", missing="other").concrete(), radl.get_system_by_name(system_id)) for s in systems for s0 in cloud.concreteSystem(s, auth)] # Store the concrete system with largest score concrete_systems.setdefault(cloud_id, {})[system_id] = ( max(s1, key=lambda x: x[1]) if s1 else (None, -1e9)) # Group virtual machines to deploy by network dependencies deploy_groups = InfrastructureManager._compute_deploy_groups(radl) InfrastructureManager.logger.debug("Groups of VMs with dependencies") InfrastructureManager.logger.debug(deploy_groups) # Sort by score the cloud providers # NOTE: consider fake deploys (vm_number == 0) deploys_group_cloud_list = {} for deploy_group in deploy_groups: suggested_cloud_ids = list( set([d.cloud_id for d in deploy_group if d.cloud_id])) if len(suggested_cloud_ids) > 1: raise Exception("Two deployments that have to be launched in the same cloud provider " "are asked to be deployed in different cloud providers: %s" % deploy_group) elif len(suggested_cloud_ids) == 1: if suggested_cloud_ids[0] not in cloud_list: InfrastructureManager.logger.debug("Cloud Provider list:") InfrastructureManager.logger.debug(cloud_list) raise Exception("No auth data for cloud with ID: %s" % suggested_cloud_ids[0]) else: cloud_list0 = [ (suggested_cloud_ids[0], cloud_list[suggested_cloud_ids[0]])] else: cloud_list0 = cloud_list.items() scored_clouds = [] for cloud_id, _ in cloud_list0: total = 0 for d in deploy_group: if d.vm_number: total += d.vm_number * concrete_systems[cloud_id][d.id][1] else: total += 1 scored_clouds.append((cloud_id, total)) ordered_cloud_list = [c.id for c in CloudInfo.get_cloud_list(auth)] # reverse the list to use the reverse order in the sort function ordered_cloud_list.reverse() # Order the clouds first by the score and then using the cloud # order in the auth data sorted_scored_clouds = sorted(scored_clouds, key=lambda x: ( x[1], ordered_cloud_list.index(x[0])), reverse=True) deploys_group_cloud_list[id(deploy_group)] = [ c[0] for c in sorted_scored_clouds] # Launch every group in the same cloud provider deployed_vm = {} cancel_deployment = [] try: if Config.MAX_SIMULTANEOUS_LAUNCHES > 1: pool = ThreadPool(processes=Config.MAX_SIMULTANEOUS_LAUNCHES) pool.map( lambda ds: InfrastructureManager._launch_group(sel_inf, ds, deploys_group_cloud_list[id(ds)], cloud_list, concrete_systems, radl, auth, deployed_vm, cancel_deployment), deploy_groups) pool.close() else: for ds in deploy_groups: InfrastructureManager._launch_group(sel_inf, ds, deploys_group_cloud_list[id(ds)], cloud_list, concrete_systems, radl, auth, deployed_vm, cancel_deployment) except Exception, e: # Please, avoid exception to arrive to this level, because some virtual # machine may lost. cancel_deployment.append(e)
def get_gce_cloud(): cloud_info = CloudInfo() cloud_info.type = "GCE" one_cloud = GCECloudConnector(cloud_info) return one_cloud
import unittest, time import logging, logging.config from IM.CloudInfo import CloudInfo from IM.auth import Authentication from IM.radl import radl_parse from IM.VirtualMachine import VirtualMachine from IM.VMRC import VMRC from IM.InfrastructureInfo import InfrastructureInfo TESTS_PATH = '/home/micafer/codigo/git_im/im/test' AUTH_FILE = TESTS_PATH + '/auth.dat' auth = Authentication(Authentication.read_auth_data(AUTH_FILE)) cloud_list = dict([ (c.id, c.getCloudConnector()) for c in CloudInfo.get_cloud_list(auth) ]) class TestConnectors(unittest.TestCase): """ Class to test the IM connectors """ vm_list = [] """ List of VMs launched in the test """ #connectors_to_test = "all" connectors_to_test = ["kub"] """ Specify the connectors to test: "all": All the connectors specified in the auth file or a list with the IDs""" @classmethod def setUpClass(cls):