def test_same_cluster(self):
nodes = [Host("foocluster-1"), Host("foocluster-2")]
self.resources["machines"][1]["cluster"] = "foocluster"
utils.concretize_nodes(self.resources, nodes)
self.assertItemsEqual(self.resources["machines"][0]["_c_nodes"],
["foocluster-1"])
self.assertItemsEqual(self.resources["machines"][1]["_c_nodes"], ["foocluster-2"])
def test_exact(self):
nodes = [Host("foocluster-1"), Host("barcluster-2")]
utils.concretize_nodes(self.resources, nodes)
self.assertCountEqual(self.resources["machines"][0]["_c_nodes"],
["foocluster-1"])
self.assertCountEqual(self.resources["machines"][1]["_c_nodes"],
["barcluster-2"])
def test_exact(self):
nodes = [Host("foocluster-1"), Host("foocluster-2")]
utils.concretize_nodes(self.resources, nodes)
self.assertCountEqual(self.resources["machines"][0]["_c_nodes"],
["foocluster-2"])
# Description with min are filled first
self.assertCountEqual(self.resources["machines"][1]["_c_nodes"],
["foocluster-1"])
def get_hosts_list(self, hosts_str):
"""Generate a list of hosts from the given string.
Args:
hosts_str (str): The following options are supported
- The path of the file containing the hosts to be used. Each host
should be in a different line. Repeated hosts are pruned.
Hint: in a running Grid5000 job, $OAR_NODEFILE should be used.
- A comma-separated list of site:job_id
- A comma-separated list of hosts.
- An oargrid_job_id
Return:
list of Host: The list of hosts.
"""
hosts = []
if os.path.isfile(hosts_str):
for line in open(hosts_str):
h = Host(line.rstrip())
if h not in hosts:
hosts.append(h)
elif ':' in hosts_str:
# We assume the string is a comma separated list of site:job_id
for job in hosts_str.split(','):
site, job_id = job.split(':')
hosts += get_oar_job_nodes(int(job_id), site)
elif "," in hosts_str:
# We assume the string is a comma separated list of hosts
for hstr in hosts_str.split(','):
h = Host(hstr.rstrip())
if h not in hosts:
hosts.append(h)
elif hosts_str.isdigit():
# If the file_name is a number, we assume this is a oargrid_job_id
hosts = get_oargrid_job_nodes(int(hosts_str))
else:
# If not any of the previous, we assume is a single-host cluster
# where the given input is the only host
hosts = [Host(hosts_str.rstrip())]
logger.debug('Hosts list: \n%s',
' '.join(style.host(host.address.split('.')[0])
for host in hosts))
return hosts
def _launch_kadeploy(self, max_tries=10, check_deploy=True):
"""Create an execo_g5k.Deployment object, launch the deployment and
return a tuple (deployed_hosts, undeployed_hosts)
"""
# if the provisioner has oar_job_ids and no config_file_path
# then the configs variable is not created
if not hasattr(self, 'configs'):
logger.info('The list of %s hosts: \n%s', len(self.hosts),
hosts_list(self.hosts, separator='\n'))
return
logger.info('Deploying %s hosts \n%s', len(self.hosts),
hosts_list(self.hosts, separator='\n'))
try:
deployment = Deployment(
hosts=[Host(canonical_host_name(host)) for host in self.hosts],
env_file=self.configs['custom_image'],
env_name=self.configs['cloud_provider_image'])
except ValueError:
logger.error(
"Please put in the config file either custom_image or cloud_provider_image."
)
exit()
# user=self.env_user,
# vlan=self.kavlan)
# Activate kadeploy output log if log level is debug
if logger.getEffectiveLevel() <= 10:
stdout = [sys.stdout]
stderr = [sys.stderr]
else:
stdout = None
stderr = None
# deploy() function will iterate through each frontend to run kadeploy
# so that the deployment hosts will be performed sequentially site after site
deployed_hosts, undeployed_hosts = deploy(
deployment,
stdout_handlers=stdout,
stderr_handlers=stderr,
num_tries=max_tries,
check_deployed_command=check_deploy)
deployed_hosts = list(deployed_hosts)
undeployed_hosts = list(undeployed_hosts)
# # Renaming hosts if a kavlan is used
# if self.kavlan:
# for i, host in enumerate(deployed_hosts):
# deployed_hosts[i] = get_kavlan_host_name(host, self.kavlan)
# for i, host in enumerate(undeployed_hosts):
# undeployed_hosts[i] = get_kavlan_host_name(host, self.kavlan)
logger.info('Deployed %s hosts successfully', len(deployed_hosts))
cr = '\n' if len(undeployed_hosts) > 0 else ''
logger.info('Failed %s hosts %s%s', len(undeployed_hosts), cr,
hosts_list(undeployed_hosts))
return deployed_hosts, undeployed_hosts
def configure_apt_proxy(vms):
"""Override apt proxy-guess with server as proxy"""
hosts_vms = {}
for vm in vms:
if not vm['host'] in hosts_vms:
hosts_vms[vm['host']] = []
hosts_vms[vm['host']].append(vm['ip'])
conf = []
for server, clients in hosts_vms.iteritems():
server = Host(server)
logger.detail('Configuring %s as APT proxy for %s',
style.host(server.address), ','.join(clients))
conf.append(
TaktukRemote(
' echo \'Acquire::http::Proxy \"http://' + server.address +
':9999" ; \' > /etc/apt/apt.conf.d/proxy-guess', clients))
ParallelActions(conf).run()
def test_not_order_dependent(self):
nodes = [Host("foocluster-1"), Host("foocluster-2"), Host("foocluster-3")]
self.resources["machines"][0]["nodes"] = 2
resources_1 = copy.deepcopy(self.resources)
utils.concretize_nodes(resources_1, nodes)
nodes = [Host("foocluster-2"), Host("foocluster-3"), Host("foocluster-1")]
resources_2 = copy.deepcopy(self.resources)
resources_2["machines"][0]["nodes"] = 2
utils.concretize_nodes(resources_2, nodes)
self.assertItemsEqual(resources_1["machines"][0]["_c_nodes"],
resources_2["machines"][0]["_c_nodes"])
def _mount_secondary_nics(desc, networks):
cluster = desc["cluster"]
# get only the secondary interfaces
nics = get_cluster_interfaces(cluster, lambda nic: not nic['mounted'])
idx = 0
desc["_c_nics"] = desc.get("_c_nics") or []
for network_id in desc.get("secondary_networks", []):
net = lookup_networks(network_id, networks)
if net["type"] == PROD:
# nothing to do
continue
nic_device, nic_name = nics[idx]
nodes_to_set = [Host(n) for n in desc["_c_nodes"]]
vlan_id = net["_c_network"]["vlan_id"]
logger.info("Put %s, %s in vlan id %s for nodes %s" %
(nic_device, nic_name, vlan_id, nodes_to_set))
api.set_nodes_vlan(net["site"], nodes_to_set, nic_device, vlan_id)
# recording the mapping, just in case
desc["_c_nics"].append((nic_name, get_roles_as_list(net)))
idx = idx + 1
def import_from_kaenv(self, env, remote=None):
"""import a kadeploy environment from a kadeploy database"""
env = re.match(
r"^(?P<name>[-_.\w]+)(?:@(?P<user>[_.\w]+))?(:?:(?P<version>[_.\w]+))?$",
env).groupdict("")
if env['user']:
env['user'] = "******" + env['user']
if env['version']:
env['version'] = " --env-version " + env['version']
kaenv_cmd = "kaenv3{user}{version} -p {name}".format(**env)
if remote:
remote = re.match(
r"^(?:(?P<user>[-_.\w]+)@)?(?P<address>[-_.\w]+)(?::(?P<port>\d{1,5}))?$",
remote).groupdict()
p = SshProcess(kaenv_cmd, Host(**remote))
else:
p = Process(kaenv_cmd, shell=True)
p.run()
self.desc = yaml.load(p.stdout)
return self
def _launch_kadeploy(self, max_tries=1, check_deploy=True):
"""Create a execo_g5k.Deployment object, launch the deployment and
return a tuple (deployed_hosts, undeployed_hosts)
"""
logger.info('Deploying %s hosts \n%s', len(self.hosts),
hosts_list(self.hosts))
deployment = Deployment(
hosts=[Host(canonical_host_name(host)) for host in self.hosts],
env_file=self.env_file,
env_name=self.env_name,
user=self.env_user,
vlan=self.kavlan)
# Activate kadeploy output log if log level is debug
if logger.getEffectiveLevel() <= 10:
stdout = [sys.stdout]
stderr = [sys.stderr]
else:
stdout = None
stderr = None
deployed_hosts, undeployed_hosts = deploy(
deployment,
stdout_handlers=stdout,
stderr_handlers=stderr,
num_tries=max_tries,
check_deployed_command=check_deploy)
deployed_hosts = list(deployed_hosts)
undeployed_hosts = list(undeployed_hosts)
# Renaming hosts if a kavlan is used
if self.kavlan:
for i, host in enumerate(deployed_hosts):
deployed_hosts[i] = get_kavlan_host_name(host, self.kavlan)
for i, host in enumerate(undeployed_hosts):
undeployed_hosts[i] = get_kavlan_host_name(host, self.kavlan)
logger.info('Deployed %s hosts \n%s', len(deployed_hosts),
hosts_list(deployed_hosts))
cr = '\n' if len(undeployed_hosts) > 0 else ''
logger.info('Failed %s hosts %s%s', len(undeployed_hosts), cr,
hosts_list(undeployed_hosts))
self._update_hosts_state(deployed_hosts, undeployed_hosts)
return deployed_hosts, undeployed_hosts
def test_one_missing(self):
nodes = [Host("foocluster-1")]
utils.concretize_nodes(self.resources, nodes)
self.assertCountEqual(self.resources["machines"][0]["_c_nodes"], [])
self.assertCountEqual(self.resources["machines"][1]["_c_nodes"],
["foocluster-1"])
def get_vms_slot(vms, elements, slots, excluded_elements=None):
"""Return a slot with enough RAM and CPU """
chosen_slot = None
mem = vms[0]['mem']
cpu = vms[0]['n_cpu']
req_ram = sum([vm['mem'] for vm in vms])
req_cpu = sum([vm['n_cpu'] for vm in vms]) / 3
logger.debug('RAM %s CPU %s', req_ram, req_cpu)
for element in excluded_elements:
if element in get_g5k_sites():
excluded_elements += [
cluster for cluster in get_site_clusters(element)
if cluster not in excluded_elements
]
if 'grid5000' in elements:
clusters = [
cluster for cluster in get_g5k_clusters()
if cluster not in excluded_elements
and get_cluster_site not in excluded_elements
]
else:
clusters = [
element for element in elements if element in get_g5k_clusters()
and element not in excluded_elements
]
for element in elements:
if element in get_g5k_sites():
clusters += [
cluster for cluster in get_site_clusters(element)
if cluster not in clusters
and cluster not in excluded_elements
]
for slot in slots:
hosts = []
for element in slot[2]:
if str(element) in clusters:
n_hosts = slot[2][element]
for i in range(n_hosts):
hosts.append(Host(str(element + '-1.' + \
get_cluster_site(element) + '.grid5000.fr')))
attr = get_CPU_RAM_FLOPS(hosts)['TOTAL']
if attr['CPU'] > req_cpu and attr['RAM'] > req_ram:
chosen_slot = slot
break
del hosts[:]
if chosen_slot is None:
return None, None
resources_needed = {}
resources_available = chosen_slot[2]
logger.debug('resources available' + pformat(resources_available))
iter_clusters = cycle(clusters)
while req_ram > 0 or req_cpu > 0:
cluster = iter_clusters.next()
if resources_available[cluster] == 0:
clusters.remove(cluster)
iter_clusters = cycle(clusters)
else:
host = cluster + '-1'
attr = get_CPU_RAM_FLOPS([host])
resources_available[cluster] -= 1
req_ram -= float(attr[host]['RAM'] / mem) * mem
req_cpu -= float(attr[host]['CPU'] / cpu) * cpu
if cluster not in resources_needed:
resources_needed[cluster] = 0
resources_needed[cluster] += 1
if 'kavlan' in elements:
resources_needed['kavlan'] = 1
logger.debug('resources needed' + pformat(resources_needed))
return chosen_slot[0], distribute_hosts(chosen_slot[2], resources_needed,
excluded_elements)