def _actions_hosts(self, action): hosts_ok, hosts_ko = [], [] for p in action.processes: if p.ok: hosts_ok.append(p.host) else: logger.warn('%s is KO', p.host) hosts_ko.append(p.host) hosts_ok, hosts_ko = list(set(hosts_ok)), list(set(hosts_ko)) self._update_hosts_state(hosts_ok, hosts_ko)
def _actions_hosts(self, action): hosts_ok, hosts_ko = [], [] for p in action.processes: if p.ok: hosts_ok.append(p.host) else: logger.warn('%s is KO', p.host) hosts_ko.append(p.host) hosts_ok, hosts_ko = list(set(hosts_ok)), list(set(hosts_ko)) self._update_hosts_state(hosts_ok, hosts_ko)
def _get_target_lc_and_port(self, equip, linecard_index, port_index, site): # given the port <linecard_index>:<port_index> on equip/site # find (linecard_index, port_index) to which is it connected to on the target equipment # by looking at the (optional) port specification and by searching in the target equipment network description, and comparing / complementing this informations # returns the tuple (linecard_index, port_index) port = self.data['network'][site][equip]['linecards'][linecard_index][ 'ports'][port_index] uid = _parse_port_uid(port['uid']) if 'port' in port: target_lc1, target_port1 = _parse_port_port(port['port']) if target_lc1 is None: logger.warn( 'unable to get from the port spec %s the target linecard of link between %s(%s:%s %s) and %s' % (port['port'], equip, linecard_index, port_index, port, uid)) target_lc2, target_port2 = self._get_target_linecard_and_port_from_api( equip, linecard_index, port_index, site) if target_lc2 is None: logger.warn( 'unable to get from %s the target linecard of link between %s(%s:%s %s) and %s' % (uid, equip, linecard_index, port_index, port, uid)) if (target_lc1 != target_lc2) or (target_port1 != target_port2): if not target_lc1 is None: target_lc, target_port = target_lc1, target_port1 else: target_lc, target_port = target_lc2, target_port2 logger.warn( 'mismatch between the linecards of link between %s(%s:%s %s) and %s: %s:%s vs %s:%s. Using the "less unlikely one" %s:%s' % (equip, linecard_index, port_index, port, uid, target_lc1, target_port1, target_lc2, target_port2, target_lc, target_port)) return (target_lc, target_port) else: return self._get_target_linecard_and_port_from_api( equip, linecard_index, port_index, site)
def _parse_port_uid(uid): prefix, _, uid = uid.rpartition(' ') if prefix: logger.warn('uid %s prefixed with %s' % (uid, prefix)) return uid
def add_equip(self, equip, site): """Add a network equipment """ if equip not in self.data['network'][site]: logger.warn('Equipment %s not described in API' % (equip, )) return data = self.data['network'][site][equip] if self.has_node(equip): recurse = False else: logger.debug('Adding equipment %s', equip) self.add_node(equip, kind=data['kind'], backplane=data['backplane_bps']) recurse = True lc_data = data['linecards'] multiple_linecards = self._equip_uses_multiple_linecards(equip, site) equip_bw = data['backplane_bps'] for i_lc, lc in enumerate(lc_data): lc_node = _get_linecard_name(equip, i_lc) if 'ports' in lc: for i_port, port in enumerate(lc['ports']): if 'uid' in port: uid = _parse_port_uid(port['uid']) if not self._is_in_api(site, uid): do_once( (site, uid), logger.warn, 'unable to get kind of %s in %s, is it in g5k api?' % (uid, site)) continue kind = port.get('kind') kind2 = self._get_node_kind(site, uid) if not kind: kind = kind2 if kind != 'node': do_once( (equip, i_lc, i_port), logger.warn, 'missing kind in port %s:%s %s of %s, using %s from %s' % (i_lc, i_port, port, equip, kind, uid)) elif not kind2: logger.warn('missing kind in %s' % (uid, )) elif kind != kind2: logger.warn( 'mismatching kind %s in port %s:%s %s of %s and kind %s from %s. Using %s' % (kind, i_lc, i_port, port, equip, kind2, uid, kind2)) kind = kind2 if not kind: logger.error('unable to find kind of %s' % (uid, )) port_bw = lc['rate'] if 'rate' not in port else port[ 'rate'] if kind == 'virtual': # in this situation, we don't know what # kind is the target equipment, we need to # discover it if uid in self.data['network'][site]: pass elif uid in self.data['hosts']: kind = 'virtual-node' logger.warn( 'virtual link from %s(%s:%s %s) to node %s' % (equip, i_lc, i_port, port, uid)) else: pass if self.has_node(uid): if kind in ['node', 'virtual-node']: for e in self.get_host_adapters(uid): if e['switch'] == equip: if multiple_linecards: self._checked_add_linecard( lc_node, lc.get('backplane_bps', data['backplane_bps'])) self._checked_add_edge( equip, lc_node, _unique_link_key( equip, lc_node), bandwidth=equip_bw, active=True) self._checked_add_edge( lc_node, uid, _unique_link_key( lc_node, uid + '-' + e['device']), bandwidth=port_bw, active=e['mounted']) else: self._checked_add_edge( equip, uid, _unique_link_key( equip, uid + '-' + e['device']), bandwidth=min( port_bw, equip_bw), active=e['mounted']) elif kind in ['switch', 'router'] and recurse: if multiple_linecards: self._checked_add_linecard( lc_node, lc.get('backplane_bps', data['backplane_bps'])) self._checked_add_edge(equip, lc_node, _unique_link_key( equip, lc_node), bandwidth=equip_bw, active=True) target_lc, target_port = self._get_target_lc_and_port( equip, i_lc, i_port, site) if not target_lc is None: if self._equip_uses_multiple_linecards( uid, site): self._checked_add_edge( lc_node, _get_linecard_name( uid, target_lc), _unique_link_key( lc_node, _get_linecard_name( uid, target_lc)), bandwidth=port_bw, active=True) else: self._checked_add_edge( lc_node, uid, _unique_link_key(lc_node, uid), bandwidth=port_bw, active=True) else: logger.error( 'unable to find the target linecard of link between %s(%s:%s %s) and %s. Skipping this link!' % (equip, i_lc, i_port, port, uid)) else: target_lc, target_port = self._get_target_lc_and_port( equip, i_lc, i_port, site) if not target_lc is None: if self._equip_uses_multiple_linecards( uid, site): self._checked_add_edge( equip, _get_linecard_name( uid, target_lc), _unique_link_key( equip, _get_linecard_name( uid, target_lc)), bandwidth=min( port_bw, equip_bw), active=True) else: self._checked_add_edge( equip, uid, _unique_link_key(equip, uid), bandwidth=min( port_bw, equip_bw), active=True) else: logger.error( 'unable to find the target linecard of link between %s(%s:%s %s) and %s. Skipping this link!' % (equip, i_lc, i_port, port, uid)) if 'renater' in uid: # if uid != 'renater-' + site: # logger.error('renater node in %s has name %s which is not of the form renater-%s. Forcing to renater-%s' % (site, uid, site, site)) # uid = 'renater-' + site self.add_node(uid, kind='renater') if multiple_linecards: self._checked_add_linecard( lc_node, lc.get('backplane_bps', data['backplane_bps'])) self._checked_add_edge(equip, lc_node, _unique_link_key( equip, lc_node), bandwidth=equip_bw, active=True) self._checked_add_edge(lc_node, uid, _unique_link_key( lc_node, uid), bandwidth=port_bw, active=True) else: self._checked_add_edge( equip, uid, _unique_link_key(equip, uid), bandwidth=min(port_bw, equip_bw), active=True) elif kind in ['switch', 'router']: if multiple_linecards: self._checked_add_linecard( lc_node, lc.get('backplane_bps', data['backplane_bps'])) self._checked_add_edge(equip, lc_node, _unique_link_key( equip, lc_node), bandwidth=equip_bw, active=True) if recurse: self.add_equip(uid, site)
def _get_target_linecard_and_port_from_api(self, equip, linecard_index, port_index, site): # given the port <linecard_index>:<port_index> on equip/site # find (linecard_index, port_index) to which it is connected to on the target equipment # by searching in the target equipment network description # returns the tuple (linecard_index, port_index) port = self.data['network'][site][equip]['linecards'][linecard_index][ 'ports'][port_index] uid = _parse_port_uid(port['uid']) if uid not in self.data['network'][site]: raise Exception( 'trying to find a linecard of equipment %s which is not in the network description of %s' % (uid, site)) possible_targets = [] for i_lc, lc in enumerate( self.data['network'][site][uid]['linecards']): if 'ports' in lc: for i_p, p in enumerate(lc['ports']): if 'uid' in p and _parse_port_uid(p['uid']) == equip: possible_targets.append((i_lc, i_p)) targets = [] if len(possible_targets) > 1: # need to disambiguate for target in possible_targets: target_port_data = self.data['network'][site][uid][ 'linecards'][target[0]]['ports'][target[1]] if 'port' in target_port_data: target_lc, target_port = _parse_port_port( target_port_data['port']) if target_lc is None or target_port is None: logger.warn( 'unable to parse port spec %s of port %s:%s on %s of link from between %s(%s:%s %s)' % (target_port_data['port'], target[0], target[1], uid, equip, linecard_index, port_index, port)) if target_lc == linecard_index and target_port == port_index: targets.append(target) else: logger.warn( 'no "port" entry in api network/%s/%s/linecards[%s]/ports[%s]' % (site, uid, target[0], target[1])) else: targets = possible_targets if len(targets) == 0: logger.warn( 'unable to find the target linecard on %s of link from %s(%s:%s %s)' % (uid, equip, linecard_index, port_index, port)) if len(possible_targets) > 0: logger.warn( 'there are %s candidates %s, use the first possible one: %s' % (len(possible_targets), possible_targets, possible_targets[0])) return possible_targets[0] else: return (None, None) if len(targets) > 1: logger.warn( 'unable to disambiguate between multiple links to %s from %s(%s:%s %s). candidate linecards:ports are %s. Using the first possible one %s' % (uid, equip, linecard_index, port_index, port, targets, targets[0])) return targets[0] return targets[0]
def _get_site_planning_PGSQL(site, site_planning, ignore_besteffort): try: with G5kAutoPortForwarder( site, 'oardb.' + site + '.grid5000.fr', g5k_configuration['oar_pgsql_ro_port']) as (host, port): conn = psycopg2.connect( host=host, port=port, user=g5k_configuration['oar_pgsql_ro_user'], password=g5k_configuration['oar_pgsql_ro_password'], database=g5k_configuration['oar_pgsql_ro_db']) try: cur = conn.cursor() # Retrieving alive resources sql = """SELECT DISTINCT R.type, R.network_address, R.vlan, R.subnet_address FROM resources R WHERE state <> 'Dead' AND R.maintenance <> 'YES';""" cur.execute(sql) for data in cur.fetchall(): if data[0] == "default": cluster = get_host_cluster(data[1]) if cluster in site_planning: site_planning[cluster][data[1]] = { 'busy': [], 'free': [] } if data[0] in ['kavlan', 'kavlan-global'] \ and 'vlans' in site_planning: site_planning['vlans']['kavlan-' + data[2]] = { 'busy': [], 'free': [] } if data[0] == "subnet" and 'subnet' in site_planning: site_planning['subnets'][data[3]] = { 'busy': [], 'free': [] } sql = ( """SELECT J.job_id, J.state, GJP.start_time AS start_time, GJP.start_time+MJD.moldable_walltime, array_agg(DISTINCT R.network_address) AS hosts, array_agg(DISTINCT R.vlan) AS vlan, array_agg(DISTINCT R.subnet_address) AS subnets FROM jobs J LEFT JOIN moldable_job_descriptions MJD ON MJD.moldable_job_id=J.job_id LEFT JOIN gantt_jobs_predictions GJP ON GJP.moldable_job_id=MJD.moldable_id INNER JOIN gantt_jobs_resources AR ON AR.moldable_job_id=MJD.moldable_id LEFT JOIN resources R ON AR.resource_id=R.resource_id WHERE ( J.state='Launching' OR J.state='Running' OR J.state='Waiting') """ + (""" AND queue_name<>'besteffort'""" if ignore_besteffort else """""") + """GROUP BY J.job_id, GJP.start_time, MJD.moldable_walltime ORDER BY J.start_time""") # CONVERT(SUBSTRING_INDEX(SUBSTRING_INDEX(R.network_address,'.',1),'-',-1), SIGNED)""" cur.execute(sql) for job in cur.fetchall(): start_time = job[2] end_time = job[3] start_time, end_time = _fix_job(start_time, end_time) if len(job[4]) > 0: for host in job[4]: if host != '': cluster = get_host_cluster(host) if cluster in site_planning: if host in site_planning[cluster]: site_planning[cluster][host][ 'busy'].append( (start_time, end_time)) if job[5][0] and 'vlans' in site_planning: for vlan in job[5]: if isinstance(vlan, str) and int(vlan) > 3: # only routed vlan site_planning['vlans']['kavlan-' + vlan]['busy'].append( (start_time, end_time)) if len(job[6]) > 0 and 'subnet' in site_planning: for subnet in job[6]: site_planning['subnets'][subnet]['busy'].append( (start_time, end_time)) finally: conn.close() except Exception as e: logger.warn( 'error connecting to oar database / getting planning from ' + site) logger.detail("exception:\n" + format_exc()) currentThread().broken = True
def _get_site_planning_API(site, site_planning, ignore_besteffort): try: alive_nodes = set([ str(node['network_address']) for node in get_resource_attributes( '/sites/' + site + '/internal/oarapi/resources/details.json?limit=2^30')['items'] if node['type'] == 'default' and node['state'] != 'Dead' and node['maintenance'] != 'YES' ]) for host in alive_nodes: host_cluster = get_host_cluster(str(host)) if host_cluster in site_planning: site_planning[host_cluster].update( {host: { 'busy': [], 'free': [] }}) if 'vlans' in site_planning: site_planning['vlans'] = {} for vlan in _get_vlans_API(site): site_planning['vlans'][vlan] = {'busy': [], 'free': []} # STORAGE AND SUBNETS MISSING # Retrieving jobs site_jobs = get_resource_attributes( '/sites/' + site + '/jobs?limit=1073741824&state=waiting,launching,running')['items'] jobs_links = [ link['href'] for job in site_jobs for link in job['links'] \ if link['rel'] == 'self' and (ignore_besteffort == False or job['queue'] != 'besteffort') ] threads = [] for link in jobs_links: t = Thread(target=_get_job_link_attr_API, args=('/' + str(link).split('/', 2)[2], )) t.broken = False t.attr = None t.ex = None threads.append(t) t.start() for t in threads: t.join() if t.broken: raise t.ex attr = t.attr try: start_time = attr['started_at'] if attr[ 'started_at'] != 0 else attr['scheduled_at'] end_time = start_time + attr['walltime'] except: continue start_time, end_time = _fix_job(start_time, end_time) nodes = attr['assigned_nodes'] for node in nodes: cluster = node.split('.', 1)[0].split('-')[0] if cluster in site_planning and node in site_planning[cluster]: site_planning[cluster][node]['busy'].append( (start_time, end_time)) if 'vlans' in site_planning and 'vlans' in attr['resources_by_type'] \ and int(attr['resources_by_type']['vlans'][0]) > 3: kavname = 'kavlan-' + str( attr['resources_by_type']['vlans'][0]) site_planning['vlans'][kavname]['busy'].append( (start_time, end_time)) if 'subnets' in site_planning and 'subnets' in attr[ 'resources_by_type']: for subnet in attr['resources_by_type']['subnets']: if subnet not in site_planning['subnets']: site_planning['subnets'][subnet] = { 'busy': [], 'free': [] } site_planning['subnets'][subnet]['busy'].append( (start_time, end_time)) # STORAGE IS MISSING except Exception as e: logger.warn( 'error connecting to oar database / getting planning from ' + site) logger.detail("exception:\n" + format_exc()) currentThread().broken = True