def _get_vlans_API(site):
"""Retrieve the list of VLAN of a site from the 3.0 Grid'5000 API"""
equips = get_resource_attributes('/sites/' + site + '/network_equipments/')
vlans = []
for equip in equips['items']:
if 'vlans' in equip and len(equip['vlans']) > 2:
for params in equip['vlans'].values():
if type( params ) == type({}) and 'name' in params \
and int(params['name'].split('-')[1])>3:
# > 3 because vlans 1, 2, 3 are not routed
vlans.append(params['name'])
return vlans
def get_kavlan_network(kavlan, site):
"""Retrieve the network parameters for a given kavlan from the API"""
network, mask_size = None, None
equips = get_resource_attributes('/sites/' + site + '/network_equipments/')
for equip in equips['items']:
if 'vlans' in equip and equip['kind'] == "router":
all_vlans = equip['vlans']
break
for info in all_vlans.values():
if type(info) == type({}) and 'name' in info \
and info['name'] == 'kavlan-' + str(kavlan):
network, _, mask_size = info['addresses'][0].partition('/',)
return network, mask_size
def get_kavlan_network(kavlan, site):
"""Retrieve the network parameters for a given kavlan from the API"""
network, mask_size = None, None
equips = get_resource_attributes('/sites/' + site + '/network_equipments/')
for equip in equips['items']:
if 'vlans' in equip and equip['kind'] == "router":
all_vlans = equip['vlans']
break
for info in all_vlans.values():
if type(info) == type({}) and 'name' in info \
and info['name'] == 'kavlan-' + str(kavlan):
network, _, mask_size = info['addresses'][0].partition('/', )
return network, mask_size
def get_kavlan_network(kavlan, site):
"""Retrieve the network parameters for a given kavlan from the API"""
logger.debug(str(kavlan) + ' on site ' + site)
network, mask_size = None, None
equips = get_resource_attributes('/sites/' + site + '/network_equipments/')
for equip in equips['items']:
if 'vlans' in equip and len(equip['vlans']) > 2:
all_vlans = equip['vlans']
break
for info in all_vlans.itervalues():
if isinstance(info, dict) and 'name' in info \
and info['name'] == 'kavlan-' + str(kavlan):
network, _, mask_size = info['addresses'][0].partition('/', )
logger.debug('network=%s, mask_size=%s', network, mask_size)
return network, mask_size
def get_kavlan_network(kavlan, site):
"""Retrieve the network parameters for a given kavlan from the API"""
logger.debug(str(kavlan) + ' on site ' + site)
network, mask_size = None, None
equips = get_resource_attributes('/sites/' + site + '/network_equipments/')
for equip in equips['items']:
if 'vlans' in equip and len(equip['vlans']) > 2:
all_vlans = equip['vlans']
break
for info in all_vlans.itervalues():
if isinstance(info, dict) and 'name' in info \
and info['name'] == 'kavlan-' + str(kavlan):
network, _, mask_size = info['addresses'][0].partition('/',)
logger.debug('network=%s, mask_size=%s', network, mask_size)
return network, mask_size
def get_nodes(job):
"""Returns nodes assigned to a job."""
site = socket.getfqdn().split('.')
site = site[1] if len(site) >= 2 else site[0]
path = '/sites/' + site + '/jobs/' + job
job_properties = get_resource_attributes(path)
nodes = job_properties['assigned_nodes']
try:
started_at = job_properties['started_at']
except KeyError:
started_at = 'Undefined'
try:
stopped_at = job_properties['stopped_at']
except KeyError:
stopped_at = 'Undefined'
return flask.jsonify({'job': int(job),
'started_at': started_at,
'stopped_at': stopped_at,
'nodes': nodes})
def get_nodes(job, metric):
"""Returns nodes assigned to a job."""
site = socket.getfqdn().split('.')
site = site[1] if len(site) >= 2 else site[0]
path = '/sites/' + site + '/jobs/' + job
job_properties = get_resource_attributes(path)
nodes = job_properties['assigned_nodes']
if "network" in metric:
probes = []
for node in nodes:
selectedProbe = site + '.' + node.split('.')[0]
all_probes = flask.request.probes_network
for probe in all_probes:
try:
if selectedProbe == filter_network_interface(probe):
probes.append("%s.%s.grid5000.fr" % (probe.split(".")[1],probe.split(".")[0]))
except:
continue
nodes = probes
else:
probes = []
for node in nodes:
selectedProbe = site + '.' + node.split('.')[0]
all_probes = flask.request.probes_power
for probe in all_probes:
try:
if selectedProbe == filter_network_interface(probe):
probes.append("%s.%s.grid5000.fr" % (probe.split(".")[1],probe.split(".")[0]))
except:
continue
nodes = probes
try:
started_at = job_properties['started_at']
except KeyError:
started_at = 'Undefined'
try:
stopped_at = job_properties['stopped_at']
except KeyError:
stopped_at = 'Undefined'
return flask.jsonify({'job': int(job),
'started_at': started_at,
'stopped_at': stopped_at,
'nodes': nodes})
def _get_site_planning_API(site, site_planning, ignore_besteffort):
try:
alive_nodes = set([
str(node['network_address']) for node in get_resource_attributes(
'/sites/' + site +
'/internal/oarapi/resources/details.json?limit=2^30')['items']
if node['type'] == 'default' and node['state'] != 'Dead'
and node['maintenance'] != 'YES'
])
for host in alive_nodes:
host_cluster = get_host_cluster(str(host))
if host_cluster in site_planning:
site_planning[host_cluster].update(
{host: {
'busy': [],
'free': []
}})
if 'vlans' in site_planning:
site_planning['vlans'] = {}
for vlan in _get_vlans_API(site):
site_planning['vlans'][vlan] = {'busy': [], 'free': []}
# STORAGE AND SUBNETS MISSING
# Retrieving jobs
site_jobs = get_resource_attributes(
'/sites/' + site +
'/jobs?limit=1073741824&state=waiting,launching,running')['items']
jobs_links = [ link['href'] for job in site_jobs for link in job['links'] \
if link['rel'] == 'self' and (ignore_besteffort == False or job['queue'] != 'besteffort') ]
threads = []
for link in jobs_links:
t = Thread(target=_get_job_link_attr_API,
args=('/' + str(link).split('/', 2)[2], ))
t.broken = False
t.attr = None
t.ex = None
threads.append(t)
t.start()
for t in threads:
t.join()
if t.broken:
raise t.ex
attr = t.attr
try:
start_time = attr['started_at'] if attr[
'started_at'] != 0 else attr['scheduled_at']
end_time = start_time + attr['walltime']
except:
continue
start_time, end_time = _fix_job(start_time, end_time)
nodes = attr['assigned_nodes']
for node in nodes:
cluster = node.split('.', 1)[0].split('-')[0]
if cluster in site_planning and node in site_planning[cluster]:
site_planning[cluster][node]['busy'].append(
(start_time, end_time))
if 'vlans' in site_planning and 'vlans' in attr['resources_by_type'] \
and int(attr['resources_by_type']['vlans'][0]) > 3:
kavname = 'kavlan-' + str(
attr['resources_by_type']['vlans'][0])
site_planning['vlans'][kavname]['busy'].append(
(start_time, end_time))
if 'subnets' in site_planning and 'subnets' in attr[
'resources_by_type']:
for subnet in attr['resources_by_type']['subnets']:
if subnet not in site_planning['subnets']:
site_planning['subnets'][subnet] = {
'busy': [],
'free': []
}
site_planning['subnets'][subnet]['busy'].append(
(start_time, end_time))
# STORAGE IS MISSING
except Exception as e:
logger.warn(
'error connecting to oar database / getting planning from ' + site)
logger.detail("exception:\n" + format_exc())
currentThread().broken = True
def _get_job_link_attr_API(p):
try:
currentThread().attr = get_resource_attributes(p)
except Exception as e:
currentThread().broken = True
currentThread().ex = e
