def get_fogbow_cloud():
cloud_info = CloudInfo()
cloud_info.type = "FogBow"
cloud_info.server = "server.com"
cloud_info.port = 8182
cloud = FogBowCloudConnector(cloud_info)
return cloud
def get_one_cloud():
cloud_info = CloudInfo()
cloud_info.type = "OpenNebula"
cloud_info.server = "server.com"
cloud_info.port = 2633
one_cloud = OpenNebulaCloudConnector(cloud_info)
return one_cloud
def get_kube_cloud():
cloud_info = CloudInfo()
cloud_info.type = "Kubernetes"
cloud_info.protocol = "http"
cloud_info.server = "server.com"
cloud_info.port = 8080
cloud = KubernetesCloudConnector(cloud_info)
return cloud
def get_docker_cloud():
cloud_info = CloudInfo()
cloud_info.type = "Kubernetes"
cloud_info.protocol = "http"
cloud_info.server = "server.com"
cloud_info.port = 2375
cloud = DockerCloudConnector(cloud_info)
return cloud
def get_ost_cloud():
cloud_info = CloudInfo()
cloud_info.type = "OpenStack"
cloud_info.protocol = "https"
cloud_info.server = "server.com"
cloud_info.port = 5000
one_cloud = OpenStackCloudConnector(cloud_info)
return one_cloud
def get_occi_cloud():
cloud_info = CloudInfo()
cloud_info.type = "OCCI"
cloud_info.protocol = "https"
cloud_info.server = "server.com"
cloud_info.port = 11443
cloud = OCCICloudConnector(cloud_info)
return cloud
def gen_launch_res(self, inf, radl, requested_radl, num_vm, auth_data):
res = []
for _ in range(num_vm):
cloud = CloudInfo()
cloud.type = "Dummy"
vm = VirtualMachine(inf, "1234", cloud, radl, requested_radl)
vm.get_ssh = Mock(side_effect=self.get_dummy_ssh)
vm.state = VirtualMachine.RUNNING
res.append((True, vm))
return res
def gen_launch_res(self, inf, radl, requested_radl, num_vm, auth_data): res = [] for i in range(num_vm): cloud = CloudInfo() cloud.type = "DeployedNode" vm = VirtualMachine(inf, "1234", cloud, radl, requested_radl) # create the mock for the vm finalize function vm.finalize = Mock(return_value=(True, vm)) res.append((True, vm)) return res
def get_occi_cloud():
cloud_info = CloudInfo()
cloud_info.type = "OCCI"
cloud_info.protocol = "https"
cloud_info.server = "server.com"
cloud_info.port = 11443
inf = MagicMock()
inf.id = "1"
cloud = OCCICloudConnector(cloud_info, inf)
return cloud
def get_ost_cloud():
cloud_info = CloudInfo()
cloud_info.type = "OpenStack"
cloud_info.protocol = "https"
cloud_info.server = "server.com"
cloud_info.port = 5000
inf = MagicMock()
inf.id = "1"
one_cloud = OpenStackCloudConnector(cloud_info, inf)
return one_cloud
def get_kube_cloud():
cloud_info = CloudInfo()
cloud_info.type = "Kubernetes"
cloud_info.protocol = "http"
cloud_info.server = "server.com"
cloud_info.port = 8080
inf = MagicMock()
inf.id = "1"
cloud = KubernetesCloudConnector(cloud_info, inf)
return cloud
def get_docker_cloud():
cloud_info = CloudInfo()
cloud_info.type = "Docker"
cloud_info.protocol = "http"
cloud_info.server = "server.com"
cloud_info.port = 2375
inf = MagicMock()
inf.id = "1"
cloud = DockerCloudConnector(cloud_info, inf)
return cloud
import logging.config
sys.path.append("..")
from IM.CloudInfo import CloudInfo
from IM.auth import Authentication
from radl import radl_parse
from IM.VirtualMachine import VirtualMachine
from IM.VMRC import VMRC
from IM.InfrastructureInfo import InfrastructureInfo
TESTS_PATH = '/home/micafer/codigo/git_im/im/test'
AUTH_FILE = TESTS_PATH + '/auth.dat'
auth = Authentication(Authentication.read_auth_data(AUTH_FILE))
cloud_list = dict([(c.id, c.getCloudConnector())
for c in CloudInfo.get_cloud_list(auth)])
class TestConnectors(unittest.TestCase):
"""
Class to test the IM connectors
"""
vm_list = []
""" List of VMs launched in the test """
# connectors_to_test = "all"
connectors_to_test = ["fogbow"]
""" Specify the connectors to test: "all": All the connectors specified in the auth file or a list with the IDs"""
@classmethod
def setUpClass(cls):
def get_azure_cloud():
cloud_info = CloudInfo()
cloud_info.type = "Azure"
cloud = AzureCloudConnector(cloud_info)
return cloud
def get_lib_cloud():
cloud_info = CloudInfo()
cloud_info.type = "LibCloud"
cloud = LibCloudCloudConnector(cloud_info)
return cloud
def get_azure_cloud():
cloud_info = CloudInfo()
cloud_info.type = "AzureClassic"
cloud = AzureClassicCloudConnector(cloud_info)
return cloud
def get_ec2_cloud():
cloud_info = CloudInfo()
cloud_info.type = "EC2"
cloud = EC2CloudConnector(cloud_info)
return cloud
def AddResource(inf_id, radl_data, auth, context=True, failed_clouds=[]):
"""
Add the resources in the RADL to the infrastructure.
Args:
- inf_id(str): infrastructure id.
- radl(str): RADL description.
- auth(Authentication): parsed authentication tokens.
- context(bool): Flag to specify if the ctxt step will be made
- failed_clouds(list of CloudInfo): A list of failed Cloud providers to avoid launching the VMs in them.
Return(list of int): ids of the new virtual machine created.
"""
auth = InfrastructureManager.check_auth_data(auth)
InfrastructureManager.logger.info(
"Adding resources to inf: " + str(inf_id))
if isinstance(radl_data, RADL):
radl = radl_data
else:
radl = radl_parse.parse_radl(radl_data)
InfrastructureManager.logger.debug(radl)
radl.check()
sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth)
# Update infrastructure RADL with this new RADL
sel_inf.complete_radl(radl)
# If any deploy is defined, only update definitions.
if not radl.deploys:
sel_inf.update_radl(radl, [])
InfrastructureManager.logger.warn(
"Infrastructure without any deploy. Exiting.")
return []
for system in radl.systems:
# Add apps requirements to the RADL
apps_to_install = system.getApplications()
for app_to_install in apps_to_install:
for app_avail, _, _, _, requirements in Recipe.getInstallableApps():
if requirements and app_avail.isNewerThan(app_to_install):
# This app must be installed and it has special
# requirements
try:
requirements_radl = radl_parse.parse_radl(
requirements).systems[0]
system.applyFeatures(
requirements_radl, conflict="other", missing="other")
except Exception:
InfrastructureManager.logger.exception(
"Error in the requirements of the app: " +
app_to_install.getValue("name") +
". Ignore them.")
InfrastructureManager.logger.debug(requirements)
break
# Get VMRC credentials
vmrc_list = []
for vmrc_elem in auth.getAuthInfo('VMRC'):
if ('host' in vmrc_elem and 'username' in vmrc_elem and
'password' in vmrc_elem):
vmrc_list.append(VMRC(vmrc_elem['host'], vmrc_elem['username'],
vmrc_elem['password']))
# Concrete systems using VMRC
# NOTE: consider not-fake deploys (vm_number > 0)
systems_with_vmrc = {}
for system_id in set([d.id for d in radl.deploys if d.vm_number > 0]):
s = radl.get_system_by_name(system_id)
if not s.getValue("disk.0.image.url") and len(vmrc_list) == 0:
raise Exception(
"No correct VMRC auth data provided nor image URL")
# Remove the requested apps from the system
s_without_apps = radl.get_system_by_name(system_id).clone()
s_without_apps.delValue("disk.0.applications")
# Set the default values for cpu, memory
defaults = (Feature("cpu.count", ">=", Config.DEFAULT_VM_CPUS),
Feature("memory.size", ">=", Config.DEFAULT_VM_MEMORY,
Config.DEFAULT_VM_MEMORY_UNIT),
Feature("cpu.arch", "=", Config.DEFAULT_VM_CPU_ARCH))
for f in defaults:
if not s_without_apps.hasFeature(f.prop, check_softs=True):
s_without_apps.addFeature(f)
vmrc_res = [s0 for vmrc in vmrc_list for s0 in vmrc.search_vm(s)]
# Check that now the image URL is in the RADL
if not s.getValue("disk.0.image.url") and not vmrc_res:
raise Exception(
"No VMI obtained from VMRC to system: " + system_id)
n = [s_without_apps.clone().applyFeatures(s0, conflict="other", missing="other")
for s0 in vmrc_res]
systems_with_vmrc[system_id] = n if n else [s_without_apps]
# Concrete systems with cloud providers and select systems with the greatest score
# in every cloud
cloud_list = dict([(c.id, c.getCloudConnector())
for c in CloudInfo.get_cloud_list(auth) if c not in failed_clouds])
concrete_systems = {}
for cloud_id, cloud in cloud_list.items():
for system_id, systems in systems_with_vmrc.items():
s1 = [InfrastructureManager._compute_score(s.clone().applyFeatures(s0,
conflict="other",
missing="other").concrete(),
radl.get_system_by_name(system_id))
for s in systems for s0 in cloud.concreteSystem(s, auth)]
# Store the concrete system with largest score
concrete_systems.setdefault(cloud_id, {})[system_id] = (
max(s1, key=lambda x: x[1]) if s1 else (None, -1e9))
# Group virtual machines to deploy by network dependencies
deploy_groups = InfrastructureManager._compute_deploy_groups(radl)
InfrastructureManager.logger.debug("Groups of VMs with dependencies")
InfrastructureManager.logger.debug(deploy_groups)
# Sort by score the cloud providers
# NOTE: consider fake deploys (vm_number == 0)
deploys_group_cloud_list = {}
for deploy_group in deploy_groups:
suggested_cloud_ids = list(
set([d.cloud_id for d in deploy_group if d.cloud_id]))
if len(suggested_cloud_ids) > 1:
raise Exception("Two deployments that have to be launched in the same cloud provider "
"are asked to be deployed in different cloud providers: %s" % deploy_group)
elif len(suggested_cloud_ids) == 1:
if suggested_cloud_ids[0] not in cloud_list:
InfrastructureManager.logger.debug("Cloud Provider list:")
InfrastructureManager.logger.debug(cloud_list)
raise Exception("No auth data for cloud with ID: %s" % suggested_cloud_ids[0])
else:
cloud_list0 = [
(suggested_cloud_ids[0], cloud_list[suggested_cloud_ids[0]])]
else:
cloud_list0 = cloud_list.items()
scored_clouds = []
for cloud_id, _ in cloud_list0:
total = 0
for d in deploy_group:
if d.vm_number:
total += d.vm_number * concrete_systems[cloud_id][d.id][1]
else:
total += 1
scored_clouds.append((cloud_id, total))
ordered_cloud_list = [c.id for c in CloudInfo.get_cloud_list(auth)]
# reverse the list to use the reverse order in the sort function
ordered_cloud_list.reverse()
# Order the clouds first by the score and then using the cloud
# order in the auth data
sorted_scored_clouds = sorted(scored_clouds, key=lambda x: (
x[1], ordered_cloud_list.index(x[0])), reverse=True)
deploys_group_cloud_list[id(deploy_group)] = [
c[0] for c in sorted_scored_clouds]
# Launch every group in the same cloud provider
deployed_vm = {}
cancel_deployment = []
try:
if Config.MAX_SIMULTANEOUS_LAUNCHES > 1:
pool = ThreadPool(processes=Config.MAX_SIMULTANEOUS_LAUNCHES)
pool.map(
lambda ds: InfrastructureManager._launch_group(sel_inf, ds, deploys_group_cloud_list[id(ds)],
cloud_list, concrete_systems, radl, auth,
deployed_vm, cancel_deployment), deploy_groups)
pool.close()
else:
for ds in deploy_groups:
InfrastructureManager._launch_group(sel_inf, ds, deploys_group_cloud_list[id(ds)],
cloud_list, concrete_systems, radl,
auth, deployed_vm, cancel_deployment)
except Exception, e:
# Please, avoid exception to arrive to this level, because some virtual
# machine may lost.
cancel_deployment.append(e)
def get_gce_cloud():
cloud_info = CloudInfo()
cloud_info.type = "GCE"
one_cloud = GCECloudConnector(cloud_info)
return one_cloud
def get_gce_cloud():
cloud_info = CloudInfo()
cloud_info.type = "GCE"
one_cloud = GCECloudConnector(cloud_info)
return one_cloud
def get_lib_cloud():
cloud_info = CloudInfo()
cloud_info.type = "LibCloud"
cloud = LibCloudCloudConnector(cloud_info)
return cloud
import unittest, time
import logging, logging.config
from IM.CloudInfo import CloudInfo
from IM.auth import Authentication
from IM.radl import radl_parse
from IM.VirtualMachine import VirtualMachine
from IM.VMRC import VMRC
from IM.InfrastructureInfo import InfrastructureInfo
TESTS_PATH = '/home/micafer/codigo/git_im/im/test'
AUTH_FILE = TESTS_PATH + '/auth.dat'
auth = Authentication(Authentication.read_auth_data(AUTH_FILE))
cloud_list = dict([ (c.id, c.getCloudConnector()) for c in CloudInfo.get_cloud_list(auth) ])
class TestConnectors(unittest.TestCase):
"""
Class to test the IM connectors
"""
vm_list = []
""" List of VMs launched in the test """
#connectors_to_test = "all"
connectors_to_test = ["kub"]
""" Specify the connectors to test: "all": All the connectors specified in the auth file or a list with the IDs"""
@classmethod
def setUpClass(cls):
def get_ec2_cloud():
cloud_info = CloudInfo()
cloud_info.type = "EC2"
cloud = EC2CloudConnector(cloud_info)
return cloud
