def test_list_vm(self, suds_cli): client = MagicMock() service = MagicMock() vmrc_res = MagicMock() vmrc_res.name = "Image" vmrc_res.hypervisor = "qemu" vmrc_res.userPassword = "******" vmrc_res.userLogin = "******" vmrc_res.os = MagicMock() vmrc_res.os.name = "linux" vmrc_res.os.flavour = "ubuntu" vmrc_res.os.version = "14.04" vmrc_res.location = "one://server.com/1" service.list.return_value = [vmrc_res] client.service = service suds_cli.return_value = client vmrc = VMRC("http://host:8080/vmrc/vmrc", "user", "pass") res_radl = vmrc.list_vm() self.assertEqual(len(res_radl), 1) self.assertEqual(res_radl[0].getValue("disk.0.image.url"), "one://server.com/1") self.assertEqual( res_radl[0].getValue("disk.0.os.credentials.password"), "pass") self.assertEqual( res_radl[0].getValue("disk.0.os.credentials.username"), "user")
def test_search_vm(self, suds_cli): client = MagicMock() service = MagicMock() vmrc_res = MagicMock() vmrc_res.name = "Image" vmrc_res.hypervisor = "qemu" vmrc_res.userPassword = "******" vmrc_res.userLogin = "******" vmrc_res.os = MagicMock() vmrc_res.os.name = "linux" vmrc_res.os.flavour = "ubuntu" vmrc_res.os.version = "14.04" vmrc_res.location = "one://server.com/1" service.search.return_value = [vmrc_res] client.service = service suds_cli.return_value = client vmrc = VMRC("http://host:8080/vmrc/vmrc", "user", "pass") radl_data = """ network net () system test ( cpu.arch='x86_64' and cpu.count>=1 and memory.size>=512m and net_interface.0.connection = 'net' and disk.0.os.flavour='ubuntu' and disk.0.os.version>='12.04' and disk.applications contains (name = 'app' and version = '1.0') and soft 10 (disk.applications contains (name = 'otherapp' and version = '2.0')) ) deploy test 1 """ radl = radl_parse.parse_radl(radl_data) res_radl = vmrc.search_vm(radl.systems[0]) self.assertEqual(len(res_radl), 1) self.assertEqual(res_radl[0].getValue("disk.0.image.url"), "one://server.com/1") self.assertEqual( res_radl[0].getValue("disk.0.os.credentials.password"), "pass") self.assertEqual( res_radl[0].getValue("disk.0.os.credentials.username"), "user")
def test_list_vm(self, soapproxy): proxy = MagicMock() vmrc_res = MagicMock() vmrc_res.name = "Image" vmrc_res.hypervisor = "qemu" vmrc_res.userPassword = "******" vmrc_res.userLogin = "******" vmrc_res.os = MagicMock() vmrc_res.os.name = "linux" vmrc_res.os.flavour = "ubuntu" vmrc_res.os.version = "14.04" vmrc_res.location = "one://server.com/1" proxy.list.return_value = [vmrc_res] soapproxy.return_value = proxy vmrc = VMRC("http://host:8080/vmrc/vmrc", "user", "pass") res_radl = vmrc.list_vm() self.assertEqual(len(res_radl), 1) self.assertEqual(res_radl[0].getValue("disk.0.image.url"), "one://server.com/1") self.assertEqual(res_radl[0].getValue("disk.0.os.credentials.password"), "pass") self.assertEqual(res_radl[0].getValue("disk.0.os.credentials.username"), "user")
def test_search_vm(self, soapproxy): proxy = MagicMock() vmrc_res = MagicMock() vmrc_res.name = "Image" vmrc_res.hypervisor = "qemu" vmrc_res.userPassword = "******" vmrc_res.userLogin = "******" vmrc_res.os = MagicMock() vmrc_res.os.name = "linux" vmrc_res.os.flavour = "ubuntu" vmrc_res.os.version = "14.04" vmrc_res.location = "one://server.com/1" proxy.search.return_value = [vmrc_res] soapproxy.return_value = proxy vmrc = VMRC("http://host:8080/vmrc/vmrc", "user", "pass") radl_data = """ network net () system test ( cpu.arch='x86_64' and cpu.count>=1 and memory.size>=512m and net_interface.0.connection = 'net' and disk.0.os.flavour='ubuntu' and disk.0.os.version>='12.04' and disk.applications contains (name = 'app' and version = '1.0') and soft 10 (disk.applications contains (name = 'otherapp' and version = '2.0')) ) deploy test 1 """ radl = radl_parse.parse_radl(radl_data) res_radl = vmrc.search_vm(radl.systems[0]) self.assertEqual(len(res_radl), 1) self.assertEqual(res_radl[0].getValue("disk.0.image.url"), "one://server.com/1") self.assertEqual(res_radl[0].getValue("disk.0.os.credentials.password"), "pass") self.assertEqual(res_radl[0].getValue("disk.0.os.credentials.username"), "user")
def concrete_systems_with_vmrc(self, radl): # Get VMRC credentials vmrc_list = [] for vmrc_elem in auth.getAuthInfo('VMRC'): if ('host' in vmrc_elem and 'username' in vmrc_elem and 'password' in vmrc_elem): vmrc_list.append( VMRC(vmrc_elem['host'], vmrc_elem['username'], vmrc_elem['password'])) # Concrete systems using VMRC res = [] for vmrc in vmrc_list: vmrc_res = vmrc.search_vm(radl) res.extend([ radl.clone().applyFeatures(s0, conflict="other", missing="other") for s0 in vmrc_res ]) return res
def AddResource(inf_id, radl_data, auth, context=True, failed_clouds=[]): """ Add the resources in the RADL to the infrastructure. Args: - inf_id(str): infrastructure id. - radl(str): RADL description. - auth(Authentication): parsed authentication tokens. - context(bool): Flag to specify if the ctxt step will be made - failed_clouds(list of CloudInfo): A list of failed Cloud providers to avoid launching the VMs in them. Return(list of int): ids of the new virtual machine created. """ auth = InfrastructureManager.check_auth_data(auth) InfrastructureManager.logger.info( "Adding resources to inf: " + str(inf_id)) if isinstance(radl_data, RADL): radl = radl_data else: radl = radl_parse.parse_radl(radl_data) InfrastructureManager.logger.debug(radl) radl.check() sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth) # Update infrastructure RADL with this new RADL sel_inf.complete_radl(radl) # If any deploy is defined, only update definitions. if not radl.deploys: sel_inf.update_radl(radl, []) InfrastructureManager.logger.warn( "Infrastructure without any deploy. Exiting.") return [] for system in radl.systems: # Add apps requirements to the RADL apps_to_install = system.getApplications() for app_to_install in apps_to_install: for app_avail, _, _, _, requirements in Recipe.getInstallableApps(): if requirements and app_avail.isNewerThan(app_to_install): # This app must be installed and it has special # requirements try: requirements_radl = radl_parse.parse_radl( requirements).systems[0] system.applyFeatures( requirements_radl, conflict="other", missing="other") except Exception: InfrastructureManager.logger.exception( "Error in the requirements of the app: " + app_to_install.getValue("name") + ". Ignore them.") InfrastructureManager.logger.debug(requirements) break # Get VMRC credentials vmrc_list = [] for vmrc_elem in auth.getAuthInfo('VMRC'): if ('host' in vmrc_elem and 'username' in vmrc_elem and 'password' in vmrc_elem): vmrc_list.append(VMRC(vmrc_elem['host'], vmrc_elem['username'], vmrc_elem['password'])) # Concrete systems using VMRC # NOTE: consider not-fake deploys (vm_number > 0) systems_with_vmrc = {} for system_id in set([d.id for d in radl.deploys if d.vm_number > 0]): s = radl.get_system_by_name(system_id) if not s.getValue("disk.0.image.url") and len(vmrc_list) == 0: raise Exception( "No correct VMRC auth data provided nor image URL") # Remove the requested apps from the system s_without_apps = radl.get_system_by_name(system_id).clone() s_without_apps.delValue("disk.0.applications") # Set the default values for cpu, memory defaults = (Feature("cpu.count", ">=", Config.DEFAULT_VM_CPUS), Feature("memory.size", ">=", Config.DEFAULT_VM_MEMORY, Config.DEFAULT_VM_MEMORY_UNIT), Feature("cpu.arch", "=", Config.DEFAULT_VM_CPU_ARCH)) for f in defaults: if not s_without_apps.hasFeature(f.prop, check_softs=True): s_without_apps.addFeature(f) vmrc_res = [s0 for vmrc in vmrc_list for s0 in vmrc.search_vm(s)] # Check that now the image URL is in the RADL if not s.getValue("disk.0.image.url") and not vmrc_res: raise Exception( "No VMI obtained from VMRC to system: " + system_id) n = [s_without_apps.clone().applyFeatures(s0, conflict="other", missing="other") for s0 in vmrc_res] systems_with_vmrc[system_id] = n if n else [s_without_apps] # Concrete systems with cloud providers and select systems with the greatest score # in every cloud cloud_list = dict([(c.id, c.getCloudConnector()) for c in CloudInfo.get_cloud_list(auth) if c not in failed_clouds]) concrete_systems = {} for cloud_id, cloud in cloud_list.items(): for system_id, systems in systems_with_vmrc.items(): s1 = [InfrastructureManager._compute_score(s.clone().applyFeatures(s0, conflict="other", missing="other").concrete(), radl.get_system_by_name(system_id)) for s in systems for s0 in cloud.concreteSystem(s, auth)] # Store the concrete system with largest score concrete_systems.setdefault(cloud_id, {})[system_id] = ( max(s1, key=lambda x: x[1]) if s1 else (None, -1e9)) # Group virtual machines to deploy by network dependencies deploy_groups = InfrastructureManager._compute_deploy_groups(radl) InfrastructureManager.logger.debug("Groups of VMs with dependencies") InfrastructureManager.logger.debug(deploy_groups) # Sort by score the cloud providers # NOTE: consider fake deploys (vm_number == 0) deploys_group_cloud_list = {} for deploy_group in deploy_groups: suggested_cloud_ids = list( set([d.cloud_id for d in deploy_group if d.cloud_id])) if len(suggested_cloud_ids) > 1: raise Exception("Two deployments that have to be launched in the same cloud provider " "are asked to be deployed in different cloud providers: %s" % deploy_group) elif len(suggested_cloud_ids) == 1: if suggested_cloud_ids[0] not in cloud_list: InfrastructureManager.logger.debug("Cloud Provider list:") InfrastructureManager.logger.debug(cloud_list) raise Exception("No auth data for cloud with ID: %s" % suggested_cloud_ids[0]) else: cloud_list0 = [ (suggested_cloud_ids[0], cloud_list[suggested_cloud_ids[0]])] else: cloud_list0 = cloud_list.items() scored_clouds = [] for cloud_id, _ in cloud_list0: total = 0 for d in deploy_group: if d.vm_number: total += d.vm_number * concrete_systems[cloud_id][d.id][1] else: total += 1 scored_clouds.append((cloud_id, total)) ordered_cloud_list = [c.id for c in CloudInfo.get_cloud_list(auth)] # reverse the list to use the reverse order in the sort function ordered_cloud_list.reverse() # Order the clouds first by the score and then using the cloud # order in the auth data sorted_scored_clouds = sorted(scored_clouds, key=lambda x: ( x[1], ordered_cloud_list.index(x[0])), reverse=True) deploys_group_cloud_list[id(deploy_group)] = [ c[0] for c in sorted_scored_clouds] # Launch every group in the same cloud provider deployed_vm = {} cancel_deployment = [] try: if Config.MAX_SIMULTANEOUS_LAUNCHES > 1: pool = ThreadPool(processes=Config.MAX_SIMULTANEOUS_LAUNCHES) pool.map( lambda ds: InfrastructureManager._launch_group(sel_inf, ds, deploys_group_cloud_list[id(ds)], cloud_list, concrete_systems, radl, auth, deployed_vm, cancel_deployment), deploy_groups) pool.close() else: for ds in deploy_groups: InfrastructureManager._launch_group(sel_inf, ds, deploys_group_cloud_list[id(ds)], cloud_list, concrete_systems, radl, auth, deployed_vm, cancel_deployment) except Exception, e: # Please, avoid exception to arrive to this level, because some virtual # machine may lost. cancel_deployment.append(e)