def concretize_networks(resources, vlans, subnets): # avoid any non-determinism s_vlans = sorted(vlans, key=lambda v: (v["site"], v["vlan_id"])) s_subnets = sorted(subnets, key=lambda s: (s["site"], s["ip_prefix"])) pools = mk_pools(s_vlans, lambda n: (n["site"], to_vlan_type(n["vlan_id"]))) pools_subnets = mk_pools( s_subnets, lambda n: (n["site"], to_subnet_type(n["ip_prefix"]))) for desc in resources["networks"]: site = desc["site"] site_info = ex5.get_resource_attributes('/sites/%s' % site) n_type = desc["type"] if n_type == PROD: desc["_c_network"] = {"site": site, "vlan_id": None} desc["_c_network"].update(site_info["kavlans"]["default"]) elif n_type in KAVLAN_TYPE: networks = pick_things(pools, (site, n_type), 1) if len(networks) < 1: raise MissingNetworkError(site, n_type) # concretize the network desc["_c_network"] = networks[0] vlan_id = desc["_c_network"]["vlan_id"] desc["_c_network"].update(site_info["kavlans"][str(vlan_id)]) elif n_type in SUBNET_TYPE: networks = pick_things(pools_subnets, (site, n_type), 1) if len(networks) < 1: raise MissingNetworkError(site, n_type) desc["_c_network"] = networks[0] return resources
def get_clusters_interfaces(clusters, extra_cond=lambda nic: True): """ Returns for each cluster the available cluster interfaces Args: clusters (str): list of the clusters extra_cond (lambda): extra predicate to filter network card retrieved from the API. E.g lambda nic: not nic['mounted'] will retrieve all the usable network cards that are not mounted by default. Returns: dict of cluster with their associated nic names Examples: .. code-block:: python # pseudo code actual = get_clusters_interfaces(["paravance"]) expected = {"paravance": ["eth0", "eth1"]} assertDictEquals(expected, actual) """ utils.get_clusters_interfaces(clusters, extra_cond=extra_cond) interfaces = {} for cluster in clusters: site = EX5.get_cluster_site(cluster) nics = EX5.get_resource_attributes("/sites/%s/clusters/%s/nodes" % (site, cluster)) nics = nics['items'][0]['network_adapters'] nics = [ nic['device'] for nic in nics if nic['mountable'] and nic['interface'] == 'Ethernet' and not nic['management'] and extra_cond(nic) ] nics = sorted(nics) interfaces.setdefault(cluster, nics) return interfaces
def get_cluster_interfaces(cluster, extra_cond=lambda nic: True): site = ex5.get_cluster_site(cluster) nics = ex5.get_resource_attributes( "/sites/%s/clusters/%s/nodes" % (site, cluster)) nics = nics['items'][0]['network_adapters'] nics = [nic['device'] for nic in nics if nic['mountable'] and nic['interface'] == 'Ethernet' and not nic['management'] and extra_cond(nic)] nics = sorted(nics) return nics
def get_values(site, nodes): while not threading.current_thread().stopped() # Get new values from the API start = time.time() - 3600 url = "sites/%s/metrics/power/timeseries/?from=%d&to%d&only=%s" % (site, start, end, nodes) print("url: " + url) data = EX5.get_resource_attributes(url) f = open('data.json', 'w') print("Result:") pp(data) pp(data, stream=f)
def _mount_cluster_nics(self, conf, cluster, nodes, kavlan_nodes, vlans): """Get the NIC devices of the reserved cluster. :param nodes: List of hostnames unmodified by the vlan """ provider_conf = conf['provider'] # XXX: this only works if all nodes are on the same cluster, # or if nodes from different clusters have the same devices site = EX5.get_cluster_site(cluster) nics = EX5.get_resource_attributes( "/sites/%s/clusters/%s/nodes" % (site, cluster) )['items'][0]['network_adapters'] interfaces = [nic['device'] for nic in nics if nic['mountable'] and nic['interface'] == 'Ethernet'] network_interface = str(interfaces[0]) external_interface = None if len(interfaces) > 1 and not provider_conf['single_interface']: external_interface = str(interfaces[1]) _, vlan = self._get_primary_vlan(vlans) api.set_nodes_vlan(site, map(lambda d: EX.Host(d), nodes), external_interface, vlan) self._exec_command_on_nodes( kavlan_nodes, "ifconfig %s up && dhclient -nw %s" % ( external_interface, external_interface), 'mounting secondary interface') else: # TODO(msimonin) fix the network in this case as well. external_interface = 'veth0' if provider_conf['single_interface']: logging.warning("Forcing the use of a one network interface") else: logging.warning("%s has only one NIC. The same interface " "will be used for network_interface and " "neutron_external_interface." % conf['resources'].keys()[0]) self._exec_command_on_nodes( kavlan_nodes, 'ip link show veth0 || ip link add type veth peer', 'Creating a veth') return (network_interface, external_interface)
def get_kavlan_network(kavlan, site): """Retrieve the network parameters for a given kavlan from the API""" network, mask_size = None, None equips = EX5.get_resource_attributes('/sites/' + site + '/network_equipments/') for equip in equips['items']: if 'vlans' in equip and len(equip['vlans']) > 2: all_vlans = equip['vlans'] for info in all_vlans.itervalues(): if type(info) == type({}) and 'name' in info \ and info['name'] == 'kavlan-' + str(kavlan): network, _, mask_size = info['addresses'][0].partition('/',) EX.logger.debug('network=%s, mask_size=%s', network, mask_size) print network, mask_size return network, mask_size
def get_cluster_interfaces(cluster, extra_cond=lambda nic: True): site = ex5.get_cluster_site(cluster) nics = ex5.get_resource_attributes("/sites/%s/clusters/%s/nodes" % (site, cluster)) nics = nics['items'][0]['network_adapters'] # NOTE(msimonin): Since 05/18 nics on g5k nodes have predictable names but # the api description keep the legacy name (device key) and the new # predictable name (key name). The legacy names is still used for api # request to the vlan endpoint This should be fixed in # https://intranet.grid5000.fr/bugzilla/show_bug.cgi?id=9272 # When its fixed we should be able to only use the new predictable name. nics = [(nic['device'], nic['name']) for nic in nics if nic['mountable'] and nic['interface'] == 'Ethernet' and not nic['management'] and extra_cond(nic)] nics = sorted(nics) return nics
def _get_energy(self, bench_file, start, end): """Get the power consumption metrics for Kwapi This call writes a single JSON file with the metrics of all the nodes.""" # TODO get metrics from service nodes nodes = [] for n in self.config['os-computes']: try: nodes.append(re.search(r"(\w+\-\d+)\-\w+\-\d+", n).group(1)) except AttributeError: nodes.append( re.search(r"(\w+\-\d+)\.\w+\.grid5000\.fr", n).group(1)) for role, n in self.config['os-services'].items(): try: nodes.append(re.search(r"(\w+\-\d+)\-\w+\-\d+", n).group(1)) except AttributeError: nodes.append( re.search(r"(\w+\-\d+)\.\w+\.grid5000\.fr", n).group(1)) url = "/sites/%s/metrics/power/timeseries/?from=%d&to=%d&only=%s" % ( self.site, start, end, ','.join(nodes)) # This call to the API must be authenticated i = 0 while True: data = EX5.get_resource_attributes(url) i = i + 1 timestamps = data['items'][0]['timestamps'] time.sleep(0.2) if len(timestamps) > 0: break logger.info("%d attempt(s) for %s" % (i, url)) # Write the metrics from the API energy_path = os.path.join(self.result_dir, 'energy', os.path.splitext(bench_file)[0] + '.json') json_data = json.dumps(data, indent=3) with open(energy_path, 'w') as f: f.write(json_data) logger.info("Wrote " + f.name)
def _get_network(self, vlans): """Gets the network representation. Prerequisite : - a vlan must be reserved """ site, vlan_id = self._get_primary_vlan(vlans) # Get g5k networks. According to the documentation, this # network is a `/18`. site_info = EX5.get_resource_attributes('/sites/%s' % site) net = site_info['kavlans'][str(vlan_id)] logging.info("cidr : %s" % net['network']) # On the network, the first IP are reserved to g5k machines. # For a routed vlan I don't know exactly how many ip are # reserved. However, the specification is clear about global # vlan: "A global VLAN is a /18 subnet (16382 IP addresses). # It is split -- contiguously -- so that every site gets one # /23 (510 ip) in the global VLAN address space". There are 12 # site. This means that we could take ip from 13th subnetwork. # Lets consider the strategy is the same for routed vlan. See, # https://www.grid5000.fr/mediawiki/index.php/Grid5000:Network#KaVLAN # # First, split network in /23 this leads to 32 subnetworks. subnets = IPNetwork(net['network']).subnet(23) # Then, (i) drops the 12 first subnetworks because they are # dedicated to g5k machines, and (ii) drops the last one # because some of ips are used for specific stuff such as # gateway, kavlan server... subnets = islice(subnets, 13, 31) # Finally, compute the range of available ips ips = IPSet(subnets).iprange() return { 'cidr': str(net['network']), 'start': str(IPAddress(ips.first)), 'end': str(IPAddress(ips.last)), 'gateway': str(net['gateway']), 'dns': '131.254.203.235' }
def concretize_networks(resources, vlans): s_vlans = sorted(vlans, key=lambda v: (v["site"], v["vlan_id"])) pools = mk_pools(s_vlans, lambda n: (n["site"], to_vlan_type(n["vlan_id"]))) for desc in resources["networks"]: site = desc["site"] site_info = ex5.get_resource_attributes('/sites/%s' % site) n_type = desc["type"] if n_type == PROD: desc["_c_network"] = {"site": site, "vlan_id": None} desc["_c_network"].update(site_info["kavlans"]["default"]) else: networks = pick_things(pools, (site, n_type), 1) if len(networks) < 1: raise MissingNetworkError(site, n_type) # concretize the network desc["_c_network"] = networks[0] vlan_id = desc["_c_network"]["vlan_id"] desc["_c_network"].update(site_info["kavlans"][str(vlan_id)])
def _get_cluster_nics(self, cluster): site = EX5.get_cluster_site(cluster) nics = EX5.get_resource_attributes( '/sites/%s/clusters/%s/nodes' % (site, cluster))['items'][0]['network_adapters'] return [nic['device'] for nic in nics if nic['mountable']]
def retrieve_measurements(metric): """Returns measurements.""" if not metric in metrics: flask.abort(404) headers = flask.request.headers hostname = socket.getfqdn().split('.') site = hostname[1] if len(hostname) >= 2 else hostname[0] args = flask.request.args probes = None if 'job_id' in args: job_info = get_resource_attributes('sites/' + site + '/jobs/' + args['job_id']) start_time = job_info['started_at'] end_time = start_time + job_info['walltime'] nodes = list(set(job_info['resources_by_type']['cores'])) probes=[] for node in nodes: selectedProbe = site + '.' + node.split('.')[0] all_probes = [] if metric == 'power': all_probes = flask.request.storePower.get_probes_names() elif metric == 'network_in': all_probes = flask.request.storeNetworkIn.get_probes_names() else : all_probes = flask.request.storeNetworkOut.get_probes_names() for probe in all_probes: try: if selectedProbe == filter_network_interface(probe): probes.append(probe) except: continue elif 'only' in args: probes = [site + '.' + node for node in args['only'].split(',')] try: start_time = int(args.get('from', time.time() - 24 * 3600)) except: start_time = int(time.time() - 24 * 3600) try: end_time = int(args.get('to', time.time())) except: end_time = int(time.time()) else: if metric == 'power': probes = flask.request.storePower.get_probes_names() elif metric == 'network_in': probes = flask.request.storeNetworkIn.get_probes_names() else : probes = flask.request.storeNetworkOut.get_probes_names() start_time = time.time() end_time = time.time() message = {'total': len(probes), 'offset': 0, 'links': [ { "rel": "self", "href": _get_api_path(headers) + site, "type": "application/vnd.fr.grid5000.api.Collection+json;level=1" }, { "rel": "parent", "href": _get_api_path(headers) + "sites/" + site , "type": "application/vnd.fr.grid5000.api.Metric+json;level=1" } ], "items": [], } for probe in probes: try: site, probe = probe.split(".") except: LOG.error("Fail to parse %s" % probe) message['items'].append({"uid": probe, "to": end_time, "from": start_time, "resolution": 1, "type": "timeseries", "values": [], "timestamps": [], "links": [ { "rel": "self", "href": _get_api_path(headers) + "sites/" + site + "/metrics/" + metric + "/timeseries/" + probe, "type": "application/vnd.fr.grid5000.api.Timeseries+json;level=1" }, { "rel": "parent", "href": _get_api_path(headers) + "sites/" + site + "/metrics/" + metric, "type": "application/vnd.fr.grid5000.api.Metric+json;level=1" } ]}) response = flask.jsonify(message) response.headers.add('Access-Control-Allow-Origin', '*') if "grid5000" in flask.request.headers.get("Accept").lower(): response.headers['Content-Type'] = 'application/vnd.fr.grid5000.api.Collection+json;level=1' else: response.headers['Content-Type'] = 'application/json' return response if probes: message = {'total': len(probes), 'offset': 0, 'links': [ { "rel": "self", "href": _get_api_path(headers) + site, "type": "application/vnd.fr.grid5000.api.Collection+json;level=1" }, { "rel": "parent", "href": _get_api_path(headers) + "sites/" + site , "type": "application/vnd.fr.grid5000.api.Metric+json;level=1" } ], "items": [], } store = None try: if metric == 'power': store = flask.request.storePower elif metric == 'network_in': store = flask.request.storeNetworkIn else : store = flask.request.storeNetworkOut except: LOG.error("fail to retrieve store") flask.abort(404) items = store.select_probes_datas(probes, start_time, end_time) for item in items.values(): message['items'].append({"uid": item["uid"], "to": item["to"], "from": item["from"], "resolution": 1, "type": "timeseries", "values": item["values"], "timestamps": item.get("timestamps", None), "links": [ { "rel": "self", "href": _get_api_path(headers) + "sites/" + site + "/metrics/" + metric + "/timeseries/" + item["uid"], "type": "application/vnd.fr.grid5000.api.Timeseries+json;level=1" }, { "rel": "parent", "href": _get_api_path(headers) + "sites/" + site + "/metrics/" + metric, "type": "application/vnd.fr.grid5000.api.Metric+json;level=1" } ]}) response = flask.jsonify(message) response.headers.add('Access-Control-Allow-Origin', '*') if "grid5000" in flask.request.headers.get("Accept").lower(): response.headers['Content-Type'] = 'application/vnd.fr.grid5000.api.Collection+json;level=1' else: response.headers['Content-Type'] = 'application/json' return response else: LOG.error("Empty probe list") flask.abort(404)
def get_cluster_nics(self, cluster): site = EX5.get_cluster_site(cluster) nics = EX5.get_resource_attributes('/sites/%s/clusters/%s/nodes' % (site, cluster))['items'][0]['network_adapters'] return [nic['device'] for nic in nics if nic['mountable']]