def _get_vlans_API(site): """Retrieve the list of VLAN of a site from the 3.0 Grid'5000 API""" equips = get_resource_attributes('/sites/' + site + '/network_equipments/') vlans = [] for equip in equips['items']: if 'vlans' in equip and len(equip['vlans']) > 2: for params in equip['vlans'].values(): if type( params ) == type({}) and 'name' in params \ and int(params['name'].split('-')[1])>3: # > 3 because vlans 1, 2, 3 are not routed vlans.append(params['name']) return vlans
def get_kavlan_network(kavlan, site): """Retrieve the network parameters for a given kavlan from the API""" network, mask_size = None, None equips = get_resource_attributes('/sites/' + site + '/network_equipments/') for equip in equips['items']: if 'vlans' in equip and equip['kind'] == "router": all_vlans = equip['vlans'] break for info in all_vlans.values(): if type(info) == type({}) and 'name' in info \ and info['name'] == 'kavlan-' + str(kavlan): network, _, mask_size = info['addresses'][0].partition('/',) return network, mask_size
def get_kavlan_network(kavlan, site): """Retrieve the network parameters for a given kavlan from the API""" network, mask_size = None, None equips = get_resource_attributes('/sites/' + site + '/network_equipments/') for equip in equips['items']: if 'vlans' in equip and equip['kind'] == "router": all_vlans = equip['vlans'] break for info in all_vlans.values(): if type(info) == type({}) and 'name' in info \ and info['name'] == 'kavlan-' + str(kavlan): network, _, mask_size = info['addresses'][0].partition('/', ) return network, mask_size
def get_kavlan_network(kavlan, site): """Retrieve the network parameters for a given kavlan from the API""" logger.debug(str(kavlan) + ' on site ' + site) network, mask_size = None, None equips = get_resource_attributes('/sites/' + site + '/network_equipments/') for equip in equips['items']: if 'vlans' in equip and len(equip['vlans']) > 2: all_vlans = equip['vlans'] break for info in all_vlans.itervalues(): if isinstance(info, dict) and 'name' in info \ and info['name'] == 'kavlan-' + str(kavlan): network, _, mask_size = info['addresses'][0].partition('/', ) logger.debug('network=%s, mask_size=%s', network, mask_size) return network, mask_size
def get_kavlan_network(kavlan, site): """Retrieve the network parameters for a given kavlan from the API""" logger.debug(str(kavlan) + ' on site ' + site) network, mask_size = None, None equips = get_resource_attributes('/sites/' + site + '/network_equipments/') for equip in equips['items']: if 'vlans' in equip and len(equip['vlans']) > 2: all_vlans = equip['vlans'] break for info in all_vlans.itervalues(): if isinstance(info, dict) and 'name' in info \ and info['name'] == 'kavlan-' + str(kavlan): network, _, mask_size = info['addresses'][0].partition('/',) logger.debug('network=%s, mask_size=%s', network, mask_size) return network, mask_size
def get_nodes(job): """Returns nodes assigned to a job.""" site = socket.getfqdn().split('.') site = site[1] if len(site) >= 2 else site[0] path = '/sites/' + site + '/jobs/' + job job_properties = get_resource_attributes(path) nodes = job_properties['assigned_nodes'] try: started_at = job_properties['started_at'] except KeyError: started_at = 'Undefined' try: stopped_at = job_properties['stopped_at'] except KeyError: stopped_at = 'Undefined' return flask.jsonify({'job': int(job), 'started_at': started_at, 'stopped_at': stopped_at, 'nodes': nodes})
def get_nodes(job, metric): """Returns nodes assigned to a job.""" site = socket.getfqdn().split('.') site = site[1] if len(site) >= 2 else site[0] path = '/sites/' + site + '/jobs/' + job job_properties = get_resource_attributes(path) nodes = job_properties['assigned_nodes'] if "network" in metric: probes = [] for node in nodes: selectedProbe = site + '.' + node.split('.')[0] all_probes = flask.request.probes_network for probe in all_probes: try: if selectedProbe == filter_network_interface(probe): probes.append("%s.%s.grid5000.fr" % (probe.split(".")[1],probe.split(".")[0])) except: continue nodes = probes else: probes = [] for node in nodes: selectedProbe = site + '.' + node.split('.')[0] all_probes = flask.request.probes_power for probe in all_probes: try: if selectedProbe == filter_network_interface(probe): probes.append("%s.%s.grid5000.fr" % (probe.split(".")[1],probe.split(".")[0])) except: continue nodes = probes try: started_at = job_properties['started_at'] except KeyError: started_at = 'Undefined' try: stopped_at = job_properties['stopped_at'] except KeyError: stopped_at = 'Undefined' return flask.jsonify({'job': int(job), 'started_at': started_at, 'stopped_at': stopped_at, 'nodes': nodes})
def _get_site_planning_API(site, site_planning, ignore_besteffort): try: alive_nodes = set([ str(node['network_address']) for node in get_resource_attributes( '/sites/' + site + '/internal/oarapi/resources/details.json?limit=2^30')['items'] if node['type'] == 'default' and node['state'] != 'Dead' and node['maintenance'] != 'YES' ]) for host in alive_nodes: host_cluster = get_host_cluster(str(host)) if host_cluster in site_planning: site_planning[host_cluster].update( {host: { 'busy': [], 'free': [] }}) if 'vlans' in site_planning: site_planning['vlans'] = {} for vlan in _get_vlans_API(site): site_planning['vlans'][vlan] = {'busy': [], 'free': []} # STORAGE AND SUBNETS MISSING # Retrieving jobs site_jobs = get_resource_attributes( '/sites/' + site + '/jobs?limit=1073741824&state=waiting,launching,running')['items'] jobs_links = [ link['href'] for job in site_jobs for link in job['links'] \ if link['rel'] == 'self' and (ignore_besteffort == False or job['queue'] != 'besteffort') ] threads = [] for link in jobs_links: t = Thread(target=_get_job_link_attr_API, args=('/' + str(link).split('/', 2)[2], )) t.broken = False t.attr = None t.ex = None threads.append(t) t.start() for t in threads: t.join() if t.broken: raise t.ex attr = t.attr try: start_time = attr['started_at'] if attr[ 'started_at'] != 0 else attr['scheduled_at'] end_time = start_time + attr['walltime'] except: continue start_time, end_time = _fix_job(start_time, end_time) nodes = attr['assigned_nodes'] for node in nodes: cluster = node.split('.', 1)[0].split('-')[0] if cluster in site_planning and node in site_planning[cluster]: site_planning[cluster][node]['busy'].append( (start_time, end_time)) if 'vlans' in site_planning and 'vlans' in attr['resources_by_type'] \ and int(attr['resources_by_type']['vlans'][0]) > 3: kavname = 'kavlan-' + str( attr['resources_by_type']['vlans'][0]) site_planning['vlans'][kavname]['busy'].append( (start_time, end_time)) if 'subnets' in site_planning and 'subnets' in attr[ 'resources_by_type']: for subnet in attr['resources_by_type']['subnets']: if subnet not in site_planning['subnets']: site_planning['subnets'][subnet] = { 'busy': [], 'free': [] } site_planning['subnets'][subnet]['busy'].append( (start_time, end_time)) # STORAGE IS MISSING except Exception as e: logger.warn( 'error connecting to oar database / getting planning from ' + site) logger.detail("exception:\n" + format_exc()) currentThread().broken = True
def _get_job_link_attr_API(p): try: currentThread().attr = get_resource_attributes(p) except Exception as e: currentThread().broken = True currentThread().ex = e