def add_host(self, host, data=None):
"""Add a host in the graph
:param host: a string corresponding to the node name
:param data: a dict containing the Grid'5000 host attributes"""
if isinstance(host, Host):
_host = get_host_shortname(host.address)
else:
_host = host
if data:
power = data['performance']['core_flops']
cores = data['architecture']['nb_cores']
else:
power = 0
cores = 0
if len(self.get_host_adapters(_host)) > 0:
logger.debug('Adding %s', style.host(_host))
self.add_node(_host, {'kind': 'node',
'power': power,
'cores': cores})
for eq in self.get_host_adapters(_host):
if eq['mounted']:
self.add_equip(eq['switch'], get_host_site(_host))
else:
logger.warning('Node %s has no valid network connection',
_host)
def add_host(self, host, data=None):
"""Add a host in the graph
:param host: a string corresponding to the node name
:param data: a dict containing the Grid'5000 host attributes"""
if isinstance(host, Host):
_host = get_host_shortname(host.address)
else:
_host = host
if data:
power = data['performance']['core_flops']
cores = data['architecture']['nb_cores']
else:
power = 0
cores = 0
if len(self.get_host_adapters(_host)) == 0:
logger.warning('Node %s has no valid network connection', _host)
logger.debug('Adding %s', style.host(_host))
self.add_node(_host, {'kind': 'node', 'power': power, 'cores': cores})
for eq in self.get_host_adapters(_host):
if eq['mounted']:
self.add_equip(eq['switch'], get_host_site(_host))
self._filter_equip_leaves()
def get_host_adapters(self, host):
"""Return the mountable network interfaces from a host"""
try:
if host in self.data['hosts']:
return [
m for m in self.data['hosts'][host]['network_adapters']
if 'switch' in m and not m['management'] and m['mountable']
and m['switch'] and m['interface'] == 'Ethernet'
]
except:
logger.warning("Wrong description for host %s" % style.host(host))
logger.debug("host's network_adapters = %s" %
(self.data['hosts'][host]['network_adapters'], ))
return []
def get_host_adapters(self, host):
"""Return the mountable network interfaces from a host"""
try:
if host in self.data['hosts']:
return [m for m in self.data['hosts'][host]['network_adapters']
if 'switch' in m
and not m['management']
and m['mountable']
and m['switch']
and m['interface'] == 'Ethernet']
except:
logger.warning("Wrong description for host %s" % style.host(host))
logger.debug("host's network_adapters = %s" % (self.data['hosts'][host]['network_adapters'],))
return []
def _is_cache_old_and_reachable(cache_dir):
"""Try to read the api_commit stored in the cache_dir and compare
it with latest commit, return True if remote commit is different
from cache commit"""
try:
with open(cache_dir + 'api_commit') as f:
local_commit = f.readline()
except:
logger.detail('No commit version found')
return True
try:
api_commit = get_resource_attributes('')['version']
except:
logger.warning('Unable to check API, reverting to cache')
return False
if local_commit != get_resource_attributes('')['version']:
logger.info('Cache is outdated, will retrieve the latest commit')
return True
else:
logger.detail('Already at the latest commit')
return False
def _is_cache_old_and_reachable(cache_dir):
"""Try to read the api_commit stored in the cache_dir and compare
it with latest commit, return True if remote commit is different
from cache commit"""
try:
with open(cache_dir + 'api_commit') as f:
local_commit = f.readline()
except:
logger.detail('No commit version found')
return True
try:
api_commit = get_resource_attributes('')['version']
except:
logger.warning('Unable to check API, reverting to cache')
return False
if local_commit != get_resource_attributes('')['version']:
logger.info('Cache is outdated, will retrieve the latest commit')
return True
else:
logger.detail('Already at the latest commit')
return False
def distribute_vms(vms, hosts, distribution='round-robin'):
"""Distribute the virtual machines on the hosts.
:param vms: a list of VMs dicts which host key will be updated
:param hosts: a list of hosts
:param distribution: a string defining the distribution type:
'round-robin', 'concentrated', 'n_by_hosts', 'random
"""
logger.debug('Initial virtual machines distribution \n%s',
"\n".join([vm['id'] + ": " + str(vm['host']) for vm in vms]))
if distribution in ['round-robin', 'concentrated', 'random']:
attr = get_CPU_RAM_FLOPS(hosts)
dist_hosts = hosts[:]
iter_hosts = cycle(dist_hosts)
host = iter_hosts.next()
for vm in vms:
remaining = attr[host].copy()
while remaining['RAM'] - vm['mem'] <= 0 \
or remaining['CPU'] - vm['n_cpu'] / 3 <= 0:
dist_hosts.remove(host)
if len(dist_hosts) == 0:
req_mem = sum([vm['mem'] for vm in vms])
req_cpu = sum([vm['n_cpu'] for vm in vms]) / 3
logger.error('Not enough ressources ! \n' + 'RAM'.rjust(20)
+ 'CPU'.rjust(10) + '\n' + 'Needed'.ljust(15)
+ '%s Mb'.ljust(15) + '%s \n' +
'Available'.ljust(15) + '%s Mb'.ljust(15)
+ '%s \n' + 'Maximum number of VM is %s',
req_mem, req_cpu, attr['TOTAL']['RAM'],
attr['TOTAL']['CPU'],
style.emph(str(get_max_vms(hosts, vm['mem']))))
exit()
iter_hosts = cycle(dist_hosts)
host = iter_hosts.next()
remaining = attr[host].copy()
vm['host'] = host
remaining['RAM'] -= vm['mem']
remaining['CPU'] -= vm['n_cpu'] / 3
attr[host] = remaining.copy()
if distribution == 'round-robin':
host = iter_hosts.next()
remaining = attr[host].copy()
if distribution == 'random':
for i in range(randint(0, len(dist_hosts))):
host = iter_hosts.next()
remaining = attr[host].copy()
elif distribution == 'n_by_hosts':
n_by_host = int(len(vms) / len(hosts))
i_vm = 0
for host in hosts:
for i in range(n_by_host):
vms[i_vm]['host'] = host
i_vm += 1
if len(vms) % len(hosts) != 0:
logger.warning('Reducing number of VMs to have %s by host',
style.emph(n_by_host))
vms[:] = vms[0:n_by_host * len(hosts)]
else:
logger.debug('No valid distribution given')
logger.debug('Final virtual machines distribution \n%s',
"\n".join([vm['id'] + ": " + str(vm['host']) for vm in vms]))
def treemap(gr,
nodes_legend=None,
edges_legend=None,
nodes_labels=None,
layout='neato',
compact=False):
"""Create a treemap of the topology and return a matplotlib figure
:param nodes_legend: a dict of dicts containing the parameter used to draw
the nodes, such as 'myelement': {'color': '#9CF7BC', 'shape': 'p',
'size': 200}
:param edges_legend: a dict of dicts containing the parameter used to draw
the edges, such as bandwidth: {'width': 0.2, 'color': '#666666'}
:param nodes_labels: a dict of dicts containing the font parameters for
the labels, such as 'myelement ': {'nodes': {}, 'font_size': 8,
'font_weight': 'bold', 'str_func': lambda n: n.split('.')[1].title()}
:param layout: the graphviz tool to be used to compute node position
:param compact: represent only on node for a cluster/cabinet
WARNING: This function use matplotlib.figure that by default requires a
DISPLAY. If you want use this on a headless host, you need to change the
matplotlib backend before to import execo_g5k.topology module.
"""
base_size = 2
_default_color = '#000000'
_default_shape = 'o'
_default_size = 100
_default_width = 0.8
_default_font_size = 10
_default_font_weight = 'normal'
def _default_str_func(n):
return n.split('.')[0]
def _default_nodes_legend():
"""Create a default legend for the nodes"""
return {
'renater': {
'color': '#9CF7BC',
'shape': 'p',
'size': 200
},
'router': {
'color': '#BFDFF2',
'shape': '8',
'size': 300,
'width': 0.5
},
'switch': {
'color': '#F5C9CD',
'shape': 's',
'size': 100,
'width': 0.2
},
'host': {
'color': '#F0F7BE',
'shape': 'o',
'size': 30,
'width': 0.2
},
'cluster': {
'color': '#F0F7BE',
'shape': 'd',
'size': 200,
'width': _default_width
},
'default': {
'color': _default_color,
'shape': _default_shape,
'size': _default_size
},
'linecard': {
'size': 10,
'shape': '^',
'color': 'w',
'width': 0.1
},
}
def _default_edges_legend():
"""Defines the width and color of the edges based on bandwidth"""
return {
100000000: {
'width': 0.2,
'color': '#666666'
},
1000000000: {
'width': 0.4,
'color': '#666666'
},
3000000000: {
'width': 0.6,
'color': '#333333'
},
10000000000: {
'width': 1.0,
'color': '#111111'
},
20000000000: {
'width': 2.0,
'color': '#111111'
},
30000000000: {
'width': 3.0,
'color': '#111111'
},
40000000000: {
'width': 4.0,
'color': '#111111'
},
'default': {
'width': _default_width,
'color': _default_color
}
}
def _default_nodes_labels(compact=False):
"""Defines the font labels"""
def _default_str_func(n):
return n.split('.')[0]
return {
'renater': {
'nodes': {},
'font_size': base_size * 4,
'font_weight': 'normal',
'str_func': _default_str_func
},
'router': {
'nodes': {},
'font_size': base_size * 4,
'font_weight': 'bold'
},
'switch': {
'nodes': {},
'font_size': base_size * 4,
'font_weight': 'normal'
},
'cluster': {
'nodes': {},
'font_size': base_size * 4,
'font_weight': 'normal'
},
'host': {
'nodes': {},
'font_size': base_size * 3,
'font_weight': 'normal'
},
'default': {
'nodes': {},
'font_size': _default_font_size,
'font_weight': _default_font_weight,
'str_func': _default_str_func
},
'linecard': {
'nodes': {},
'font_size': base_size * 3,
'str_func': _default_str_func
}
}
grdot = gr.copy()
# Setting legend and labels
_nodes_legend = _default_nodes_legend()
_edges_legend = _default_edges_legend()
_nodes_labels = _default_nodes_labels(compact)
if nodes_legend:
_nodes_legend.update(nodes_legend)
if edges_legend:
_edges_legend.update(edges_legend)
if nodes_labels:
_nodes_labels.update(nodes_labels)
if not compact:
elements = ['renater', 'router', 'switch', 'node', 'linecard']
else:
for site in grdot.get_sites():
for cluster, data in grdot.get_clusters().items():
for equip, radicals in data['equips'].items():
grdot.add_node(cluster + '\n' + radicals,
{'kind': 'cluster'})
grdot.add_edge(cluster + '\n' + radicals, equip,
{'bandwidth': data['bandwidth']})
grdot.remove_nodes_from(
[n[0] for n in grdot.nodes(True) if n[1]['kind'] == 'node'])
elements = ['renater', 'router', 'switch', 'cluster']
logger.debug('Legend and labels initialized')
# substitute kind host to kind node in all the graph, as node is a reserved dot word
for n in grdot.nodes_iter(True):
if n[1].get('kind') == 'node':
n[1]['kind'] = 'host'
# Initializing plot
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(111)
logger.debug('Defining positions')
try:
pos = graphviz_layout(grdot, prog=layout)
except:
logger.warning('Error in generating graphviz layout, will use ' +
'spring layout that does not scale well ...')
raise
pos = nx.spring_layout(grdot, iterations=100)
# Adding the nodes
for k in elements:
nodes = [
node[0] for node in grdot.nodes_iter(data=True)
if 'kind' in node[1] and node[1]['kind'] == k
]
if k not in _nodes_legend:
_nodes_legend[k] = _nodes_legend['default']
nodes = nx.draw_networkx_nodes(
grdot,
pos,
nodelist=nodes,
node_shape=_nodes_legend[k]['shape']
if 'shape' in _nodes_legend[k] else _default_shape,
node_color=_nodes_legend[k]['color']
if 'color' in _nodes_legend[k] else _default_color,
node_size=_nodes_legend[k]['size']
if 'size' in _nodes_legend[k] else _default_size,
linewidths=_nodes_legend[k]['width']
if 'width' in _nodes_legend[k] else _default_width)
# Adding the edges
for bandwidth, params in _edges_legend.items():
if bandwidth != 'other':
edges = [
(edge[0], edge[1]) for edge in grdot.edges_iter(data=True)
if 'bandwidth' in edge[2] and edge[2]['bandwidth'] == bandwidth
]
nx.draw_networkx_edges(
grdot,
pos,
edgelist=edges,
width=params['width'] if 'width' in params else _default_width,
edge_color=params['color']
if 'color' in params else _default_color)
edges = [(edge[0], edge[1]) for edge in grdot.edges_iter(data=True)
if edge[2]['bandwidth'] not in _edges_legend]
nx.draw_networkx_edges(grdot,
pos,
edgelist=edges,
width=_edges_legend['default']['width'],
edge_color=_edges_legend['default']['color'])
# Adding the labels
for node, data in grdot.nodes_iter(data=True):
if 'nodes' not in _nodes_labels[data['kind']]:
_nodes_labels[data['kind']]['nodes'] = {}
if data['kind'] in _nodes_labels:
_nodes_labels[data['kind']]['nodes'][node] = _nodes_labels[data['kind']]['str_func'](node) \
if 'str_func' in _nodes_labels[data['kind']] else _default_str_func(node)
else:
_nodes_labels['default']['nodes'][node] = _nodes_labels['default'][
'str_func'](node)
for data in _nodes_labels.values():
nx.draw_networkx_labels(
grdot,
pos,
labels=data['nodes'],
font_size=data['font_size']
if 'font_size' in data else _default_font_size,
font_weight=data['font_weight']
if 'font_weight' in data else _default_font_weight)
plt.axis('off')
plt.tight_layout()
title = 'Created by execo_g5k.topology \n%s\nAPI commit %s' % \
(grdot.graph['date'], grdot.graph['api_commit'])
plt.text(0.1, 0, title, transform=ax.transAxes)
return fig
def treemap(gr, nodes_legend=None, edges_legend=None, nodes_labels=None,
layout='neato', compact=False):
"""Create a treemap of the topology and return a matplotlib figure
:param nodes_legend: a dict of dicts containing the parameter used to draw
the nodes, such as 'myelement': {'color': '#9CF7BC', 'shape': 'p',
'size': 200}
:param edges_legend: a dict of dicts containing the parameter used to draw
the edges, such as bandwidth: {'width': 0.2, 'color': '#666666'}
:param nodes_labels: a dict of dicts containing the font parameters for
the labels, such as 'myelement ': {'nodes': {}, 'font_size': 8,
'font_weight': 'bold', 'str_func': lambda n: n.split('.')[1].title()}
:param layout: the graphviz tool to be used to compute node position
:param compact: represent only on node for a cluster/cabinet
WARNING: This function use matplotlib.figure that by default requires a
DISPLAY. If you want use this on a headless host, you need to change the
matplotlib backend before to import execo_g5k.topology module.
"""
base_size = 2
_default_color = '#000000'
_default_shape = 'o'
_default_size = 100
_default_width = 0.8
_default_font_size = 10
_default_font_weight = 'normal'
def _default_str_func(n):
return n.split('.')[0]
def _default_nodes_legend():
"""Create a default legend for the nodes"""
return {'renater':
{'color': '#9CF7BC', 'shape': 'p', 'size': 200},
'router':
{'color': '#BFDFF2', 'shape': '8', 'size': 300,
'width': 0.5},
'switch':
{'color': '#F5C9CD', 'shape': 's', 'size': 100,
'width': 0.2},
'node':
{'color': '#F0F7BE', 'shape': 'o', 'size': 30,
'width': 0.2},
'cluster':
{'color': '#F0F7BE', 'shape': 'd', 'size': 200,
'width': _default_width},
'default':
{'color': _default_color, 'shape': _default_shape,
'size': _default_size},
'linecard':
{'size': 10, 'shape': '^', 'color': 'w', 'width': 0.1},
}
def _default_edges_legend():
"""Defines the width and color of the edges based on bandwidth"""
return {100000000: {'width': 0.2, 'color': '#666666'},
1000000000: {'width': 0.4, 'color': '#666666'},
3000000000: {'width': 0.6, 'color': '#333333'},
10000000000: {'width': 1.0, 'color': '#111111'},
20000000000: {'width': 2.0, 'color': '#111111'},
30000000000: {'width': 3.0, 'color': '#111111'},
40000000000: {'width': 4.0, 'color': '#111111'},
'default': {'width': _default_width, 'color': _default_color}}
def _default_nodes_labels(compact=False):
"""Defines the font labels"""
def _default_str_func(n):
return n.split('.')[0]
return {'renater':
{'nodes': {},
'font_size': base_size * 4,
'font_weight': 'normal',
'str_func': lambda n: n.split('-')[1].title()},
'router':
{'nodes': {},
'font_size': base_size * 4,
'font_weight': 'bold'},
'switch':
{'nodes': {},
'font_size': base_size * 4,
'font_weight': 'normal'},
'cluster':
{'nodes': {},
'font_size': base_size * 4,
'font_weight': 'normal'},
'node':
{'nodes': {},
'font_size': base_size * 3,
'font_weight': 'normal'},
'default':
{'nodes': {},
'font_size': _default_font_size,
'font_weight': _default_font_weight,
'str_func': _default_str_func},
'linecard':
{'nodes': {},
'font_size': base_size * 3,
'str_func': lambda n: n.split('_')[1]}
}
# Setting legend and labels
_nodes_legend = _default_nodes_legend()
_edges_legend = _default_edges_legend()
_nodes_labels = _default_nodes_labels(compact)
if nodes_legend:
_nodes_legend.update(nodes_legend)
if edges_legend:
_edges_legend.update(edges_legend)
if nodes_labels:
_nodes_labels.update(nodes_labels)
if not compact:
elements = ['renater', 'router', 'switch', 'node', 'linecard']
else:
for site in gr.get_sites():
for cluster, data in gr.get_clusters().items():
for equip, radicals in data['equips'].items():
gr.add_node(cluster + '\n' + radicals,
{'kind': 'cluster'})
gr.add_edge(cluster + '\n' + radicals, equip,
{'bandwidth': data['bandwidth']})
gr.remove_nodes_from([n[0] for n in gr.nodes(True) if n[1]['kind'] == 'node'])
elements = ['renater', 'router', 'switch', 'cluster']
logger.debug('Legend and labels initialized')
# Initializing plot
fig = plt.figure(figsize=(10, 10))
ax = fig.add_subplot(111)
logger.debug('Defining positions')
try:
pos = nx.graphviz_layout(gr, prog=layout)
except:
logger.warning('Error in generating graphviz layout, will use ' +
'spring layout that does not scale well ...')
raise
pos = nx.spring_layout(gr, iterations=100)
# Adding the nodes
for k in elements:
nodes = [node[0] for node in gr.nodes_iter(data=True)
if 'kind' in node[1] and node[1]['kind'] == k]
if k not in _nodes_legend:
_nodes_legend[k] = _nodes_legend['default']
nodes = nx.draw_networkx_nodes(gr, pos, nodelist=nodes,
node_shape=_nodes_legend[k]['shape']
if 'shape' in _nodes_legend[k] else
_default_shape,
node_color=_nodes_legend[k]['color']
if 'color' in _nodes_legend[k] else
_default_color,
node_size=_nodes_legend[k]['size']
if 'size' in _nodes_legend[k] else
_default_size,
linewidths=_nodes_legend[k]['width']
if 'width' in _nodes_legend[k] else
_default_width)
# Adding the edges
for bandwidth, params in _edges_legend.items():
if bandwidth != 'other':
edges = [(edge[0], edge[1]) for edge in gr.edges_iter(data=True)
if 'bandwidth' in edge[2] and edge[2]['bandwidth'] == bandwidth]
nx.draw_networkx_edges(gr, pos, edgelist=edges,
width=params['width'] if 'width' in params
else _default_width,
edge_color=params['color'] if 'color' in params
else _default_color)
edges = [(edge[0], edge[1]) for edge in gr.edges_iter(data=True)
if edge[2]['bandwidth'] not in _edges_legend]
nx.draw_networkx_edges(gr, pos, edgelist=edges,
width=_edges_legend['default']['width'],
edge_color=_edges_legend['default']['color'])
# Adding the labels
for node, data in gr.nodes_iter(data=True):
if 'nodes' not in _nodes_labels[data['kind']]:
_nodes_labels[data['kind']]['nodes'] = {}
if data['kind'] in _nodes_labels:
_nodes_labels[data['kind']]['nodes'][node] = _nodes_labels[data['kind']]['str_func'](node) \
if 'str_func' in _nodes_labels[data['kind']] else _default_str_func(node)
else:
_nodes_labels['default']['nodes'][node] = _nodes_labels['default']['str_func'](node)
for data in _nodes_labels.values():
nx.draw_networkx_labels(gr, pos, labels=data['nodes'],
font_size=data['font_size']
if 'font_size' in data else _default_font_size,
font_weight=data['font_weight']
if 'font_weight' in data else _default_font_weight)
plt.axis('off')
plt.tight_layout()
title = 'Created by execo_g5k.topology \n%s\nAPI commit %s' % \
(gr.graph['date'], gr.graph['api_commit'])
plt.text(0.1, 0, title, transform=ax.transAxes)
return fig
def add_equip(self, equip, site):
"""Add a network equipment """
if equip not in self.data['network'][site]:
logger.warning('Equipment %s not described in API' % (equip,))
return
data = self.data['network'][site][equip]
logger.debug('Adding equipment %s', equip)
self.add_node(equip, kind=data['kind'],
backplane=data['backplane_bps'])
lc_data = data['linecards']
if data['kind'] == 'router':
router_bw = data['backplane_bps']
for i_lc, lc in enumerate([n for n in lc_data if 'ports' in n]):
lc_node = equip + '_lc' + str(i_lc)
lc_has_element = False
for port in sorted([p for p in lc['ports'] if 'uid' in p]):
kind = port['kind'] if 'kind' in port else lc['kind']
bandwidth = lc['rate'] if 'rate' not in port else port['rate']
if self.has_node(port['uid']):
if kind == 'node':
for e in self.get_host_adapters(port['uid']):
if e['switch'] == equip:
if ('port' in port and port['port'] == e['device']) or ('port' not in port and e['device'] == 'eth0'):
lc_has_element = True
key1 = lc_node + '_' + port['uid'] + '_' + e['device']
logger.debug('Adding link between %s and %s %s' % (
lc_node, port['uid'], e['device']))
self.add_edge(lc_node, port['uid'], key1,
bandwidth=bandwidth,
active=e['mounted'])
key2 = equip + '_' + lc_node
logger.debug('Adding link between %s and %s',
equip, lc_node)
self.add_edge(equip, lc_node, key2,
bandwidth=router_bw, active=True)
if kind == 'switch':
lc_has_element = True
key1 = lc_node + '_' + port['uid']
self.add_edge(lc_node, port['uid'], key1,
bandwidth=bandwidth, active=True)
key2 = equip + '_' + lc_node
self.add_edge(equip, lc_node, key2,
bandwidth=router_bw, active=True)
if 'renater' in port['uid']:
lc_has_element = True
self.add_node(port['uid'], kind='renater')
key1 = lc_node + ' ' + port['uid']
self.add_edge(lc_node, port['uid'], key1,
bandwidth=bandwidth, active=True)
key2 = equip + '_' + lc_node
self.add_edge(equip, lc_node, key2,
bandwidth=router_bw, active=True)
if lc_has_element:
logger.debug('Adding linecard %s', lc_node)
backplane = lc['backplane_bps'] if 'backplane_bps' \
in lc else data['backplane_bps']
self.add_node(lc_node, kind='linecard',
backplane=backplane)
else:
# some switch have two linecards ?? pat, sgraphene1 => REPORT BUG
for lc in [n for n in lc_data if 'ports' in n]:
for port in sorted([p for p in lc['ports'] if 'uid' in p]):
kind = port['kind'] if 'kind' in port else lc['kind']
bandwidth = lc['rate'] if 'rate' not in port else port['rate']
if self.has_node(port['uid']):
if kind == 'node':
for e in self.get_host_adapters(port['uid']):
if e['switch'] == equip:
key = equip + '_' + port['uid'] + '_' + e['device']
self.add_edge(equip, port['uid'], key,
bandwidth=bandwidth,
active=e['mounted'])
if kind == 'switch':
key = equip + '_' + port['uid']
self.add_edge(equip, port['uid'], key,
bandwidth=bandwidth, active=True)
if kind == 'router':
self.add_equip(port['uid'], site)
def distribute_vms(vms, hosts, distribution='round-robin'):
"""Distribute the virtual machines on the hosts.
:param vms: a list of VMs dicts which host key will be updated
:param hosts: a list of hosts
:param distribution: a string defining the distribution type:
'round-robin', 'concentrated', 'n_by_hosts', 'random
"""
logger.debug('Initial virtual machines distribution \n%s',
"\n".join([vm['id'] + ": " + str(vm['host']) for vm in vms]))
if distribution in ['round-robin', 'concentrated', 'random']:
attr = get_CPU_RAM_FLOPS(hosts)
dist_hosts = hosts[:]
iter_hosts = cycle(dist_hosts)
host = iter_hosts.next()
for vm in vms:
remaining = attr[host].copy()
while remaining['RAM'] - vm['mem'] <= 0 \
or remaining['CPU'] - vm['n_cpu'] / 3 <= 0:
dist_hosts.remove(host)
if len(dist_hosts) == 0:
req_mem = sum([vm['mem'] for vm in vms])
req_cpu = sum([vm['n_cpu'] for vm in vms]) / 3
logger.error(
'Not enough ressources ! \n' + 'RAM'.rjust(20) +
'CPU'.rjust(10) + '\n' + 'Needed'.ljust(15) +
'%s Mb'.ljust(15) + '%s \n' + 'Available'.ljust(15) +
'%s Mb'.ljust(15) + '%s \n' +
'Maximum number of VM is %s', req_mem, req_cpu,
attr['TOTAL']['RAM'], attr['TOTAL']['CPU'],
style.emph(str(get_max_vms(hosts, vm['mem']))))
exit()
iter_hosts = cycle(dist_hosts)
host = iter_hosts.next()
remaining = attr[host].copy()
vm['host'] = host
remaining['RAM'] -= vm['mem']
remaining['CPU'] -= vm['n_cpu'] / 3
attr[host] = remaining.copy()
if distribution == 'round-robin':
host = iter_hosts.next()
remaining = attr[host].copy()
if distribution == 'random':
for i in range(randint(0, len(dist_hosts))):
host = iter_hosts.next()
remaining = attr[host].copy()
elif distribution == 'n_by_hosts':
n_by_host = int(len(vms) / len(hosts))
i_vm = 0
for host in hosts:
for i in range(n_by_host):
vms[i_vm]['host'] = host
i_vm += 1
if len(vms) % len(hosts) != 0:
logger.warning('Reducing number of VMs to have %s by host',
style.emph(n_by_host))
vms[:] = vms[0:n_by_host * len(hosts)]
else:
logger.debug('No valid distribution given')
logger.debug('Final virtual machines distribution \n%s',
"\n".join([vm['id'] + ": " + str(vm['host']) for vm in vms]))
logger.setLevel(logging.INFO)
logger.debug('Options\n'+'\n'.join( [ set_style(option.ljust(20),'emph')+\
'= '+str(value).ljust(10) for option, value in vars(options).iteritems() if value is not None ]))
logger.info('%s', set_style('-- Find yoUr Nodes on g5K --', 'log_header'))
logger.info('From %s to %s', set_style(options.startdate, 'emph'),
set_style(options.enddate, 'emph'))
if options.resources is None:
options.resources = 'suno:2,sol:2,griffon:10,rennes:20'
logger.warning('No resources given, will use demo values ')
logger.info('Resources: %s', set_style(options.resources, 'emph'))
logger.info('Walltime: %s', set_style(options.walltime, 'emph'))
logger.info('Mode: %s', set_style(options.mode, 'emph'))
if prog is not None:
logger.info('Program: %s', set_style(prog, 'emph'))
if options.plots:
if 'grid5000.fr' in getfqdn():
options.plots = False
logger.warning('Plots are disabled on Grid5000 frontend until the migration to Wheezy')
resources = {}
for element in options.resources.split(','):
def add_equip(self, equip, site):
"""Add a network equipment """
if equip not in self.data['network'][site]:
logger.warning('Equipment %s not described in API' % (equip, ))
return
data = self.data['network'][site][equip]
logger.debug('Adding equipment %s', equip)
self.add_node(equip,
kind=data['kind'],
backplane=data['backplane_bps'])
lc_data = data['linecards']
if data['kind'] == 'router':
router_bw = data['backplane_bps']
for i_lc, lc in enumerate([n for n in lc_data if 'ports' in n]):
lc_node = equip + '_lc' + str(i_lc)
lc_has_element = False
for port in sorted([p for p in lc['ports'] if 'uid' in p]):
kind = port['kind'] if 'kind' in port else lc['kind']
bandwidth = lc['rate'] if 'rate' not in port else port[
'rate']
if self.has_node(port['uid']):
if kind == 'node':
for e in self.get_host_adapters(port['uid']):
if e['switch'] == equip:
if ('port' in port
and port['port'] == e['device']
) or ('port' not in port
and e['device'] == 'eth0'):
lc_has_element = True
key1 = lc_node + '_' + port[
'uid'] + '_' + e['device']
logger.debug(
'Adding link between %s and %s %s'
% (lc_node, port['uid'],
e['device']))
self.add_edge(lc_node,
port['uid'],
key1,
bandwidth=bandwidth,
active=e['mounted'])
key2 = equip + '_' + lc_node
logger.debug(
'Adding link between %s and %s',
equip, lc_node)
self.add_edge(equip,
lc_node,
key2,
bandwidth=router_bw,
active=True)
if kind == 'switch':
lc_has_element = True
key1 = lc_node + '_' + port['uid']
self.add_edge(lc_node,
port['uid'],
key1,
bandwidth=bandwidth,
active=True)
key2 = equip + '_' + lc_node
self.add_edge(equip,
lc_node,
key2,
bandwidth=router_bw,
active=True)
if 'renater' in port['uid']:
lc_has_element = True
self.add_node(port['uid'], kind='renater')
key1 = lc_node + ' ' + port['uid']
self.add_edge(lc_node,
port['uid'],
key1,
bandwidth=bandwidth,
active=True)
key2 = equip + '_' + lc_node
self.add_edge(equip,
lc_node,
key2,
bandwidth=router_bw,
active=True)
if lc_has_element:
logger.debug('Adding linecard %s', lc_node)
backplane = lc['backplane_bps'] if 'backplane_bps' \
in lc else data['backplane_bps']
self.add_node(lc_node,
kind='linecard',
backplane=backplane)
else:
# some switch have two linecards ?? pat, sgraphene1 => REPORT BUG
for lc in [n for n in lc_data if 'ports' in n]:
for port in sorted([p for p in lc['ports'] if 'uid' in p]):
kind = port['kind'] if 'kind' in port else lc['kind']
bandwidth = lc['rate'] if 'rate' not in port else port[
'rate']
if self.has_node(port['uid']):
if kind == 'node':
for e in self.get_host_adapters(port['uid']):
if e['switch'] == equip:
key = equip + '_' + port['uid'] + '_' + e[
'device']
self.add_edge(equip,
port['uid'],
key,
bandwidth=bandwidth,
active=e['mounted'])
if kind == 'switch':
key = equip + '_' + port['uid']
self.add_edge(equip,
port['uid'],
key,
bandwidth=bandwidth,
active=True)
if kind == 'router':
self.add_equip(port['uid'], site)