def test_list_vm(self, suds_cli):
client = MagicMock()
service = MagicMock()
vmrc_res = MagicMock()
vmrc_res.name = "Image"
vmrc_res.hypervisor = "qemu"
vmrc_res.userPassword = "******"
vmrc_res.userLogin = "******"
vmrc_res.os = MagicMock()
vmrc_res.os.name = "linux"
vmrc_res.os.flavour = "ubuntu"
vmrc_res.os.version = "14.04"
vmrc_res.location = "one://server.com/1"
service.list.return_value = [vmrc_res]
client.service = service
suds_cli.return_value = client
vmrc = VMRC("http://host:8080/vmrc/vmrc", "user", "pass")
res_radl = vmrc.list_vm()
self.assertEqual(len(res_radl), 1)
self.assertEqual(res_radl[0].getValue("disk.0.image.url"),
"one://server.com/1")
self.assertEqual(
res_radl[0].getValue("disk.0.os.credentials.password"), "pass")
self.assertEqual(
res_radl[0].getValue("disk.0.os.credentials.username"), "user")
def test_search_vm(self, suds_cli):
client = MagicMock()
service = MagicMock()
vmrc_res = MagicMock()
vmrc_res.name = "Image"
vmrc_res.hypervisor = "qemu"
vmrc_res.userPassword = "******"
vmrc_res.userLogin = "******"
vmrc_res.os = MagicMock()
vmrc_res.os.name = "linux"
vmrc_res.os.flavour = "ubuntu"
vmrc_res.os.version = "14.04"
vmrc_res.location = "one://server.com/1"
service.search.return_value = [vmrc_res]
client.service = service
suds_cli.return_value = client
vmrc = VMRC("http://host:8080/vmrc/vmrc", "user", "pass")
radl_data = """
network net ()
system test (
cpu.arch='x86_64' and
cpu.count>=1 and
memory.size>=512m and
net_interface.0.connection = 'net' and
disk.0.os.flavour='ubuntu' and
disk.0.os.version>='12.04' and
disk.applications contains (name = 'app' and version = '1.0') and
soft 10 (disk.applications contains (name = 'otherapp' and version = '2.0'))
)
deploy test 1
"""
radl = radl_parse.parse_radl(radl_data)
res_radl = vmrc.search_vm(radl.systems[0])
self.assertEqual(len(res_radl), 1)
self.assertEqual(res_radl[0].getValue("disk.0.image.url"),
"one://server.com/1")
self.assertEqual(
res_radl[0].getValue("disk.0.os.credentials.password"), "pass")
self.assertEqual(
res_radl[0].getValue("disk.0.os.credentials.username"), "user")
def test_list_vm(self, soapproxy):
proxy = MagicMock()
vmrc_res = MagicMock()
vmrc_res.name = "Image"
vmrc_res.hypervisor = "qemu"
vmrc_res.userPassword = "******"
vmrc_res.userLogin = "******"
vmrc_res.os = MagicMock()
vmrc_res.os.name = "linux"
vmrc_res.os.flavour = "ubuntu"
vmrc_res.os.version = "14.04"
vmrc_res.location = "one://server.com/1"
proxy.list.return_value = [vmrc_res]
soapproxy.return_value = proxy
vmrc = VMRC("http://host:8080/vmrc/vmrc", "user", "pass")
res_radl = vmrc.list_vm()
self.assertEqual(len(res_radl), 1)
self.assertEqual(res_radl[0].getValue("disk.0.image.url"), "one://server.com/1")
self.assertEqual(res_radl[0].getValue("disk.0.os.credentials.password"), "pass")
self.assertEqual(res_radl[0].getValue("disk.0.os.credentials.username"), "user")
def test_search_vm(self, soapproxy):
proxy = MagicMock()
vmrc_res = MagicMock()
vmrc_res.name = "Image"
vmrc_res.hypervisor = "qemu"
vmrc_res.userPassword = "******"
vmrc_res.userLogin = "******"
vmrc_res.os = MagicMock()
vmrc_res.os.name = "linux"
vmrc_res.os.flavour = "ubuntu"
vmrc_res.os.version = "14.04"
vmrc_res.location = "one://server.com/1"
proxy.search.return_value = [vmrc_res]
soapproxy.return_value = proxy
vmrc = VMRC("http://host:8080/vmrc/vmrc", "user", "pass")
radl_data = """
network net ()
system test (
cpu.arch='x86_64' and
cpu.count>=1 and
memory.size>=512m and
net_interface.0.connection = 'net' and
disk.0.os.flavour='ubuntu' and
disk.0.os.version>='12.04' and
disk.applications contains (name = 'app' and version = '1.0') and
soft 10 (disk.applications contains (name = 'otherapp' and version = '2.0'))
)
deploy test 1
"""
radl = radl_parse.parse_radl(radl_data)
res_radl = vmrc.search_vm(radl.systems[0])
self.assertEqual(len(res_radl), 1)
self.assertEqual(res_radl[0].getValue("disk.0.image.url"), "one://server.com/1")
self.assertEqual(res_radl[0].getValue("disk.0.os.credentials.password"), "pass")
self.assertEqual(res_radl[0].getValue("disk.0.os.credentials.username"), "user")
def concrete_systems_with_vmrc(self, radl):
# Get VMRC credentials
vmrc_list = []
for vmrc_elem in auth.getAuthInfo('VMRC'):
if ('host' in vmrc_elem and 'username' in vmrc_elem
and 'password' in vmrc_elem):
vmrc_list.append(
VMRC(vmrc_elem['host'], vmrc_elem['username'],
vmrc_elem['password']))
# Concrete systems using VMRC
res = []
for vmrc in vmrc_list:
vmrc_res = vmrc.search_vm(radl)
res.extend([
radl.clone().applyFeatures(s0,
conflict="other",
missing="other") for s0 in vmrc_res
])
return res
def AddResource(inf_id, radl_data, auth, context=True, failed_clouds=[]):
"""
Add the resources in the RADL to the infrastructure.
Args:
- inf_id(str): infrastructure id.
- radl(str): RADL description.
- auth(Authentication): parsed authentication tokens.
- context(bool): Flag to specify if the ctxt step will be made
- failed_clouds(list of CloudInfo): A list of failed Cloud providers to avoid launching the VMs in them.
Return(list of int): ids of the new virtual machine created.
"""
auth = InfrastructureManager.check_auth_data(auth)
InfrastructureManager.logger.info(
"Adding resources to inf: " + str(inf_id))
if isinstance(radl_data, RADL):
radl = radl_data
else:
radl = radl_parse.parse_radl(radl_data)
InfrastructureManager.logger.debug(radl)
radl.check()
sel_inf = InfrastructureManager.get_infrastructure(inf_id, auth)
# Update infrastructure RADL with this new RADL
sel_inf.complete_radl(radl)
# If any deploy is defined, only update definitions.
if not radl.deploys:
sel_inf.update_radl(radl, [])
InfrastructureManager.logger.warn(
"Infrastructure without any deploy. Exiting.")
return []
for system in radl.systems:
# Add apps requirements to the RADL
apps_to_install = system.getApplications()
for app_to_install in apps_to_install:
for app_avail, _, _, _, requirements in Recipe.getInstallableApps():
if requirements and app_avail.isNewerThan(app_to_install):
# This app must be installed and it has special
# requirements
try:
requirements_radl = radl_parse.parse_radl(
requirements).systems[0]
system.applyFeatures(
requirements_radl, conflict="other", missing="other")
except Exception:
InfrastructureManager.logger.exception(
"Error in the requirements of the app: " +
app_to_install.getValue("name") +
". Ignore them.")
InfrastructureManager.logger.debug(requirements)
break
# Get VMRC credentials
vmrc_list = []
for vmrc_elem in auth.getAuthInfo('VMRC'):
if ('host' in vmrc_elem and 'username' in vmrc_elem and
'password' in vmrc_elem):
vmrc_list.append(VMRC(vmrc_elem['host'], vmrc_elem['username'],
vmrc_elem['password']))
# Concrete systems using VMRC
# NOTE: consider not-fake deploys (vm_number > 0)
systems_with_vmrc = {}
for system_id in set([d.id for d in radl.deploys if d.vm_number > 0]):
s = radl.get_system_by_name(system_id)
if not s.getValue("disk.0.image.url") and len(vmrc_list) == 0:
raise Exception(
"No correct VMRC auth data provided nor image URL")
# Remove the requested apps from the system
s_without_apps = radl.get_system_by_name(system_id).clone()
s_without_apps.delValue("disk.0.applications")
# Set the default values for cpu, memory
defaults = (Feature("cpu.count", ">=", Config.DEFAULT_VM_CPUS),
Feature("memory.size", ">=", Config.DEFAULT_VM_MEMORY,
Config.DEFAULT_VM_MEMORY_UNIT),
Feature("cpu.arch", "=", Config.DEFAULT_VM_CPU_ARCH))
for f in defaults:
if not s_without_apps.hasFeature(f.prop, check_softs=True):
s_without_apps.addFeature(f)
vmrc_res = [s0 for vmrc in vmrc_list for s0 in vmrc.search_vm(s)]
# Check that now the image URL is in the RADL
if not s.getValue("disk.0.image.url") and not vmrc_res:
raise Exception(
"No VMI obtained from VMRC to system: " + system_id)
n = [s_without_apps.clone().applyFeatures(s0, conflict="other", missing="other")
for s0 in vmrc_res]
systems_with_vmrc[system_id] = n if n else [s_without_apps]
# Concrete systems with cloud providers and select systems with the greatest score
# in every cloud
cloud_list = dict([(c.id, c.getCloudConnector())
for c in CloudInfo.get_cloud_list(auth) if c not in failed_clouds])
concrete_systems = {}
for cloud_id, cloud in cloud_list.items():
for system_id, systems in systems_with_vmrc.items():
s1 = [InfrastructureManager._compute_score(s.clone().applyFeatures(s0,
conflict="other",
missing="other").concrete(),
radl.get_system_by_name(system_id))
for s in systems for s0 in cloud.concreteSystem(s, auth)]
# Store the concrete system with largest score
concrete_systems.setdefault(cloud_id, {})[system_id] = (
max(s1, key=lambda x: x[1]) if s1 else (None, -1e9))
# Group virtual machines to deploy by network dependencies
deploy_groups = InfrastructureManager._compute_deploy_groups(radl)
InfrastructureManager.logger.debug("Groups of VMs with dependencies")
InfrastructureManager.logger.debug(deploy_groups)
# Sort by score the cloud providers
# NOTE: consider fake deploys (vm_number == 0)
deploys_group_cloud_list = {}
for deploy_group in deploy_groups:
suggested_cloud_ids = list(
set([d.cloud_id for d in deploy_group if d.cloud_id]))
if len(suggested_cloud_ids) > 1:
raise Exception("Two deployments that have to be launched in the same cloud provider "
"are asked to be deployed in different cloud providers: %s" % deploy_group)
elif len(suggested_cloud_ids) == 1:
if suggested_cloud_ids[0] not in cloud_list:
InfrastructureManager.logger.debug("Cloud Provider list:")
InfrastructureManager.logger.debug(cloud_list)
raise Exception("No auth data for cloud with ID: %s" % suggested_cloud_ids[0])
else:
cloud_list0 = [
(suggested_cloud_ids[0], cloud_list[suggested_cloud_ids[0]])]
else:
cloud_list0 = cloud_list.items()
scored_clouds = []
for cloud_id, _ in cloud_list0:
total = 0
for d in deploy_group:
if d.vm_number:
total += d.vm_number * concrete_systems[cloud_id][d.id][1]
else:
total += 1
scored_clouds.append((cloud_id, total))
ordered_cloud_list = [c.id for c in CloudInfo.get_cloud_list(auth)]
# reverse the list to use the reverse order in the sort function
ordered_cloud_list.reverse()
# Order the clouds first by the score and then using the cloud
# order in the auth data
sorted_scored_clouds = sorted(scored_clouds, key=lambda x: (
x[1], ordered_cloud_list.index(x[0])), reverse=True)
deploys_group_cloud_list[id(deploy_group)] = [
c[0] for c in sorted_scored_clouds]
# Launch every group in the same cloud provider
deployed_vm = {}
cancel_deployment = []
try:
if Config.MAX_SIMULTANEOUS_LAUNCHES > 1:
pool = ThreadPool(processes=Config.MAX_SIMULTANEOUS_LAUNCHES)
pool.map(
lambda ds: InfrastructureManager._launch_group(sel_inf, ds, deploys_group_cloud_list[id(ds)],
cloud_list, concrete_systems, radl, auth,
deployed_vm, cancel_deployment), deploy_groups)
pool.close()
else:
for ds in deploy_groups:
InfrastructureManager._launch_group(sel_inf, ds, deploys_group_cloud_list[id(ds)],
cloud_list, concrete_systems, radl,
auth, deployed_vm, cancel_deployment)
except Exception, e:
# Please, avoid exception to arrive to this level, because some virtual
# machine may lost.
cancel_deployment.append(e)
